From 802d19ea0d57d42c7c5df205d30b4fe4d61c8c93 Mon Sep 17 00:00:00 2001 From: Shreyas8612 Date: Thu, 17 Sep 2026 02:02:05 +0100 Subject: [PATCH 1/2] emulator: add an RTL-measured matrix accumulate latency model Every matrix-matrix accumulate (M_MM/M_TMM/M_BMM/M_BTMM) charged SYSTOLIC_PROCESSING_OVERHEAD + MLEN cycles, which cannot express the BLEN-proportional cost of the RTL matrix pipeline, so the emulator could not be run with the measured per-op timing. Add TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL ("mlen" | "rtl_blen") with a --matrix-latency-model CLI override. "rtl_blen" charges 3 * BLEN + 11 cycles per accumulate: Verilator runs of the RTL measured 23 cycles at BLEN=4 and 35 at BLEN=8 (accumulates are serialized, each draining before the next starts), which fixes the slope and the intercept. The key is optional and defaults to "mlen", so existing settings files and cycle counts are unchanged; matrix-vector ops and write-outs keep their charges. The option and the measurements are documented in the readme. Verified with cargo fmt, cargo clippy --workspace --all-targets -D warnings and cargo test --workspace: new tests pin both formulas at BLEN=4/8 with MLEN=64, check that switching the model changes only the accumulate charge with identical results, and cover the settings default/override resolution. The ATen linear testbench passes on the default path, and re-running its program with --matrix-latency-model rtl_blen leaves the VRAM contents identical while the cycle count moves. --- plena_settings.toml | 8 ++ transactional_emulator/readme.md | 11 ++ transactional_emulator/src/cli.rs | 9 ++ transactional_emulator/src/load_config.rs | 144 ++++++++++++++++++- transactional_emulator/src/matrix_machine.rs | 111 ++++++++++++-- transactional_emulator/src/runner.rs | 8 +- transactional_emulator/src/runtime_config.rs | 2 + 7 files changed, 278 insertions(+), 15 deletions(-) diff --git a/plena_settings.toml b/plena_settings.toml index 05fddad90..5b4e889da 100644 --- a/plena_settings.toml +++ b/plena_settings.toml @@ -209,6 +209,14 @@ dc_lib_dis = 1 [TRANSACTIONAL.CONFIG.DC_EN] value = 1 +[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL] +# Cycles charged per matrix-matrix accumulate (M_MM/M_TMM/M_BMM/M_BTMM): +# "mlen" = SYSTOLIC_PROCESSING_OVERHEAD + MLEN (default) +# "rtl_blen" = 3 * BLEN + 11, measured on the RTL matrix pipeline with +# Verilator (23 cycles at BLEN=4, 35 at BLEN=8) +# Absent means "mlen". CLI override: --matrix-latency-model. +value = "mlen" + [TRANSACTIONAL.CONFIG.MAX_LOOP_INSTRUCTIONS] value = 100000 diff --git a/transactional_emulator/readme.md b/transactional_emulator/readme.md index e6c5eb713..4c757d4cb 100644 --- a/transactional_emulator/readme.md +++ b/transactional_emulator/readme.md @@ -50,6 +50,17 @@ The simulator integrates **Ramulator 2** for High-Bandwidth Memory (HBM) modelin Writes a (BLEN, BLEN) accumulator matrix (`m_accum`) to the Vector SRAM. This operation loads a (BLEN, VLEN) matrix from HBM and uses a mask to write to the Vector SRAM. +### Accumulate latency model + +`TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL` in `plena_settings.toml` (command-line override `--matrix-latency-model`) selects the cycles charged for each matrix-matrix accumulate (`M_MM`, `M_TMM`, `M_BMM`, `M_BTMM`): + +| Value | Cycles per accumulate | Notes | +|---|---|---| +| `"mlen"` (default) | `SYSTOLIC_PROCESSING_OVERHEAD + MLEN` | The historical charge; `SYSTOLIC_PROCESSING_OVERHEAD` comes from `TRANSACTIONAL.LATENCY`. | +| `"rtl_blen"` | `3 * BLEN + 11` | Measured on the RTL matrix pipeline with Verilator: 23 cycles per accumulate at BLEN=4 and 35 at BLEN=8. Matrix ops are serialized on the RTL (each accumulate drains before the next starts) and the MLEN-deep reduction is spread across parallel sub-arrays, so the cost scales with BLEN rather than MLEN. | + +The key is optional: a settings file without it charges the `mlen` model, so existing runs are unchanged. Matrix-vector ops and write-outs are not affected by the setting. + ## Notes diff --git a/transactional_emulator/src/cli.rs b/transactional_emulator/src/cli.rs index 4f2de4650..c9b534207 100644 --- a/transactional_emulator/src/cli.rs +++ b/transactional_emulator/src/cli.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; pub(crate) use clap::Parser; use clap::ValueEnum; +use crate::load_config::MatrixLatencyModel; + /// Log level filter for the tracing subscriber. /// /// When passed via `--log-level`, this fully overrides the `RUST_LOG` @@ -165,6 +167,13 @@ pub(crate) struct Opts { /// the default ../plena_settings.toml lookup. pub(crate) settings: Option, + #[arg(long, value_enum)] + /// Override the cycles charged per matrix-matrix accumulate (M_MM, M_TMM, + /// M_BMM, M_BTMM); default: MATRIX_LATENCY_MODEL in plena_settings.toml, + /// mlen. `mlen` charges SYSTOLIC_PROCESSING_OVERHEAD + MLEN cycles; + /// `rtl_blen` charges the RTL-measured 3 * BLEN + 11 cycles. + pub(crate) matrix_latency_model: Option, + #[arg(long)] /// Optional generated ASM source used to derive PC-to-stage labels for a /// runtime stage profile. This is diagnostic only; normal runs omit it. diff --git a/transactional_emulator/src/load_config.rs b/transactional_emulator/src/load_config.rs index 5b73badbe..81955b800 100644 --- a/transactional_emulator/src/load_config.rs +++ b/transactional_emulator/src/load_config.rs @@ -1,6 +1,6 @@ // load_config.rs use serde::{Deserialize, Serialize}; -use std::{env, fs, sync::LazyLock}; +use std::{env, fmt, fs, str::FromStr, sync::LazyLock}; // Import the types from your main module use quantize::{DataType, FpType, IntType, MxDataType}; @@ -15,6 +15,53 @@ pub struct ConfigValueUsize { pub value: usize, } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ConfigValueString { + pub value: String, +} + +/// Cycles charged per matrix-matrix accumulate +/// (`TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum MatrixLatencyModel { + /// `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`: the historical charge. + #[default] + Mlen, + /// `3 * BLEN + 11`: measured on the RTL matrix pipeline with Verilator + /// (23 cycles at BLEN=4, 35 at BLEN=8). + #[value(name = "rtl_blen")] + RtlBlen, +} + +impl MatrixLatencyModel { + pub const fn as_str(self) -> &'static str { + match self { + Self::Mlen => "mlen", + Self::RtlBlen => "rtl_blen", + } + } +} + +impl FromStr for MatrixLatencyModel { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "mlen" => Ok(Self::Mlen), + "rtl_blen" => Ok(Self::RtlBlen), + other => Err(format!( + "unsupported matrix latency model {other:?} (expected \"mlen\" or \"rtl_blen\")" + )), + } + } +} + +impl fmt::Display for MatrixLatencyModel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct LatencyValue { pub dc_lib_en: u32, @@ -108,6 +155,23 @@ pub struct ConfigSection { pub dc_en: ConfigValue, #[serde(rename = "MAX_LOOP_INSTRUCTIONS")] pub max_loop_instructions: ConfigValueUsize, + /// Per-accumulate latency model for matrix-matrix ops, "mlen" or + /// "rtl_blen". Optional so settings files that predate the key keep + /// working; absent means mlen. + #[serde(rename = "MATRIX_LATENCY_MODEL", default)] + pub matrix_latency_model: Option, +} + +impl ConfigSection { + pub fn matrix_latency_model(&self) -> MatrixLatencyModel { + match &self.matrix_latency_model { + None => MatrixLatencyModel::default(), + Some(model) => model + .value + .parse() + .unwrap_or_else(|err| panic!("TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL: {err}")), + } + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -181,6 +245,7 @@ impl Default for AcceleratorConfig { hbm_v_writeback_amount: ConfigValue { value: 16 }, dc_en: ConfigValue { value: 1 }, max_loop_instructions: ConfigValueUsize { value: 10000 }, + matrix_latency_model: None, }, precision: PrecisionSection { matrix_sram_type: MxDataTypeConfig { @@ -527,6 +592,10 @@ pub fn systolic_processing_overhead() -> u32 { get_dc_lib_value(&CONFIG.latency.systolic_processing_overhead) } +pub fn matrix_latency_model() -> MatrixLatencyModel { + CONFIG.config.matrix_latency_model() +} + // pub fn vector_ps_cycles() -> u32 { // get_dc_lib_value(&CONFIG.latency.vector_ps_cycles) // } @@ -702,6 +771,79 @@ mod tests { assert_eq!(cfg.config.max_loop_instructions.value, 10000); } + #[test] + fn test_matrix_latency_model_parses_case_insensitively() { + assert_eq!( + "mlen".parse::().unwrap(), + MatrixLatencyModel::Mlen + ); + assert_eq!( + "RTL_BLEN".parse::().unwrap(), + MatrixLatencyModel::RtlBlen + ); + assert!("blen".parse::().is_err()); + assert_eq!(MatrixLatencyModel::RtlBlen.to_string(), "rtl_blen"); + } + + #[test] + fn test_default_matrix_latency_model_is_mlen() { + let cfg = AcceleratorConfig::default(); + assert!(cfg.config.matrix_latency_model.is_none()); + assert_eq!(cfg.config.matrix_latency_model(), MatrixLatencyModel::Mlen); + } + + /// Drop the `[header]` table (up to the next table) from a TOML text. + fn without_table(text: &str, header: &str) -> String { + let mut out = String::new(); + let mut skipping = false; + for line in text.lines() { + if line.trim() == header { + skipping = true; + continue; + } + if skipping && line.trim_start().starts_with('[') { + skipping = false; + } + if !skipping { + out.push_str(line); + out.push('\n'); + } + } + out + } + + #[test] + fn test_settings_file_matrix_latency_model() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../plena_settings.toml"); + let text = fs::read_to_string(path).unwrap(); + + // The checked-in value is the historical charge. + let settings: PlenaSettings = toml::from_str(&text).unwrap(); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::Mlen + ); + + // A settings file without the key means the same thing. + let stripped = without_table(&text, "[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]"); + assert!(!stripped.contains("MATRIX_LATENCY_MODEL")); + let settings: PlenaSettings = toml::from_str(&stripped).unwrap(); + assert!(settings.transactional.config.matrix_latency_model.is_none()); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::Mlen + ); + + let overridden = format!( + "{stripped}\n[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]\nvalue = \"rtl_blen\"\n" + ); + let settings: PlenaSettings = toml::from_str(&overridden).unwrap(); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::RtlBlen + ); + } + #[test] fn test_default_precision_types_convert() { let cfg = AcceleratorConfig::default(); diff --git a/transactional_emulator/src/matrix_machine.rs b/transactional_emulator/src/matrix_machine.rs index ce10045e7..ffb0e7d55 100644 --- a/transactional_emulator/src/matrix_machine.rs +++ b/transactional_emulator/src/matrix_machine.rs @@ -18,12 +18,45 @@ use quantize::QuantTensor; use sram::{MatrixSram, VectorSram, assert_multiple_of, multiple_and_offset}; use tch::{IndexOp, Tensor}; +use crate::load_config::MatrixLatencyModel; use crate::matrix_core::{MatrixCore, MatrixCoreProfile}; -use crate::runtime_config::SYSTOLIC_PROCESSING_OVERHEAD; +use crate::runtime_config::{MATRIX_LATENCY_MODEL, SYSTOLIC_PROCESSING_OVERHEAD}; /// Tensor allocation options used for every accumulator buffer (f32 on CPU). const ACCUM_OPTS: (tch::Kind, tch::Device) = (tch::Kind::Float, tch::Device::Cpu); +/// Slope of the RTL-measured matrix-matrix accumulate cost, in cycles per +/// BLEN row. +const RTL_ACCUMULATE_CYCLES_PER_ROW: u32 = 3; +/// Fixed part of the RTL-measured matrix-matrix accumulate cost, in cycles. +const RTL_ACCUMULATE_FIXED_CYCLES: u32 = 11; + +/// Cycles charged for one matrix-matrix accumulate (`M_MM`, `M_TMM`, `M_BMM`, +/// `M_BTMM`) under `model`. +/// +/// * [`MatrixLatencyModel::Mlen`]: `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`, the +/// historical charge. +/// * [`MatrixLatencyModel::RtlBlen`]: `3 * BLEN + 11`. Verilator runs of the +/// RTL matrix pipeline measured 23 cycles per accumulate at BLEN=4 and 35 at +/// BLEN=8 (matrix ops are serialized: each accumulate drains before the next +/// starts); those two points fix the slope at 3 cycles per BLEN row and the +/// fixed overhead at 11 cycles. MLEN does not enter because the MLEN-deep +/// reduction is spread across parallel sub-arrays, and the +/// `SYSTOLIC_PROCESSING_OVERHEAD` knob is part of the `mlen` model only. +pub(crate) fn matrix_accumulate_cycles( + model: MatrixLatencyModel, + blen: u32, + mlen: u32, + systolic_overhead: u32, +) -> u32 { + match model { + MatrixLatencyModel::Mlen => systolic_overhead + mlen, + MatrixLatencyModel::RtlBlen => { + RTL_ACCUMULATE_CYCLES_PER_ROW * blen + RTL_ACCUMULATE_FIXED_CYCLES + } + } +} + /// Executes matrix opcodes by reading tiles/vectors from `mram`/`vram`, /// running matmul in f32, and accumulating into per-shape buffers that are /// later flushed via the `*_wo` ops. @@ -39,6 +72,7 @@ pub(crate) struct MatrixMachine { blen: u32, broadcast_amount: u32, core: MatrixCore, + latency_model: MatrixLatencyModel, } impl MatrixMachine { @@ -90,6 +124,7 @@ impl MatrixMachine { blen, broadcast_amount, core: MatrixCore::new(core_profile), + latency_model: *MATRIX_LATENCY_MODEL, } } @@ -97,10 +132,28 @@ impl MatrixMachine { self.core.profile() } + pub(crate) fn latency_model(&self) -> MatrixLatencyModel { + self.latency_model + } + + pub(crate) fn set_latency_model(&mut self, model: MatrixLatencyModel) { + self.latency_model = model; + } + fn core(&self) -> MatrixCore { self.core } + /// Cycles charged for one matrix-matrix accumulate under the active model. + fn accumulate_cycles(&self) -> u32 { + matrix_accumulate_cycles( + self.latency_model, + self.blen, + self.mlen, + *SYSTOLIC_PROCESSING_OVERHEAD, + ) + } + pub(crate) async fn mm(&mut self, m_addr: u32, v_addr: u32) { let (mat_base, mat_offset) = multiple_and_offset(m_addr, self.mlen * self.mlen); // Row-granular projection ABI: the M_MM column stride is `blen * mlen` @@ -122,9 +175,7 @@ impl MatrixMachine { mat_offset as i64..(mat_offset as i64 + self.blen as i64), )); let mut tensors = Vec::with_capacity(self.blen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.blen { tensors.push( self.vram @@ -163,9 +214,7 @@ impl MatrixMachine { )); let mut tensors = Vec::with_capacity(self.mlen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.mlen { tensors.push( self.vram @@ -281,9 +330,7 @@ impl MatrixMachine { )); let mut tensors = Vec::with_capacity(self.mlen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; // B, S, H, D for i in 0..self.mlen { tensors.push( @@ -403,9 +450,7 @@ impl MatrixMachine { .transpose(-1, -2) .i((.., mat_offset as i64..(mat_offset + self.blen) as i64)); let mut tensors = Vec::with_capacity(self.blen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.blen { tensors.push( self.vram @@ -569,9 +614,24 @@ mod tests { use tch::Tensor; use super::*; + use crate::load_config::MatrixLatencyModel; use crate::matrix_core::MatrixCoreProfile; use crate::runtime_config::SYSTOLIC_PROCESSING_OVERHEAD; + #[test] + fn accumulate_cycles_pin_both_models() { + use MatrixLatencyModel::{Mlen, RtlBlen}; + // Historical charge: overhead + MLEN, independent of BLEN. + assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 0), 64); + assert_eq!(matrix_accumulate_cycles(Mlen, 8, 64, 0), 64); + assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 5), 69); + // RTL measurement: 23 cycles at BLEN=4 and 35 at BLEN=8, independent + // of MLEN and of the overhead knob. + assert_eq!(matrix_accumulate_cycles(RtlBlen, 4, 64, 0), 23); + assert_eq!(matrix_accumulate_cycles(RtlBlen, 8, 64, 0), 35); + assert_eq!(matrix_accumulate_cycles(RtlBlen, 8, 1024, 5), 35); + } + fn bf16_plain() -> MxDataType { MxDataType::Plain(DataType::Fp(FpType::BF16)) } @@ -652,6 +712,31 @@ mod tests { assert!(b1.equal(&Tensor::from_slice(&[5.0f32, 6.0, 0.0, 0.0]))); } + #[tokio::test] + async fn rtl_blen_model_changes_only_the_accumulate_charge() { + let executor = Executor::new(); + let (mut machine, vram, out) = make_machine(8).await; + machine.set_latency_model(MatrixLatencyModel::RtlBlen); + + executor.spawn(async move { + machine.mm(0, 0).await; + machine.mm_wo(out, 1).await; + }); + executor.enter(Instant::ETERNITY).await; + + // BLEN = 2: 3 * 2 + 11 cycles for the accumulate, then the write-out's + // one cycle as before. + let expected_cycles = 3 * 2 + 11 + 1; + assert_eq!( + executor.now(), + Instant::INIT + Duration::from_nanos(expected_cycles) + ); + let a0 = vram.read(8).await.as_tensor().shallow_clone(); + let a1 = vram.read(12).await.as_tensor().shallow_clone(); + assert!(a0.equal(&Tensor::from_slice(&[1.0f32, 2.0, 0.0, 0.0]))); + assert!(a1.equal(&Tensor::from_slice(&[5.0f32, 6.0, 0.0, 0.0]))); + } + #[tokio::test] async fn mm_row_granular_offset_selects_column_block() { // Row-granular M_MM ABI (compiler e852c88): the within-tile matrix offset is diff --git a/transactional_emulator/src/runner.rs b/transactional_emulator/src/runner.rs index 04a129c16..fb7ead921 100644 --- a/transactional_emulator/src/runner.rs +++ b/transactional_emulator/src/runner.rs @@ -118,10 +118,16 @@ pub(crate) async fn run_from_cli() { *VECTOR_SRAM_TYPE, )); // Vector SRAM - let m_machine = MatrixMachine::new(mram, vram.clone(), *MLEN, *HLEN, *BLEN, *BROADCAST_AMOUNT); + let mut m_machine = + MatrixMachine::new(mram, vram.clone(), *MLEN, *HLEN, *BLEN, *BROADCAST_AMOUNT); + // `--matrix-latency-model` wins over MATRIX_LATENCY_MODEL in plena_settings.toml. + if let Some(model) = opts.matrix_latency_model { + m_machine.set_latency_model(model); + } let big_core = m_machine.core_profile(); let tail_core = MatrixCoreProfile::tail_4x256(); tracing::info!( + matrix_latency_model = %m_machine.latency_model(), big_core = big_core.name, big_rows = big_core.rows, big_cols = big_core.cols, diff --git a/transactional_emulator/src/runtime_config.rs b/transactional_emulator/src/runtime_config.rs index ff3450ef1..c11b3f0aa 100644 --- a/transactional_emulator/src/runtime_config.rs +++ b/transactional_emulator/src/runtime_config.rs @@ -9,6 +9,8 @@ pub(crate) const PERIOD: Duration = Duration::from_nanos(1); pub(crate) static SYSTOLIC_PROCESSING_OVERHEAD: LazyLock = LazyLock::new(systolic_processing_overhead); +pub(crate) static MATRIX_LATENCY_MODEL: LazyLock = + LazyLock::new(matrix_latency_model); pub(crate) static VECTOR_ADD_CYCLES: LazyLock = LazyLock::new(vector_add_cycles); pub(crate) static VECTOR_MUL_CYCLES: LazyLock = LazyLock::new(vector_mul_cycles); pub(crate) static VECTOR_EXP_CYCLES: LazyLock = LazyLock::new(vector_exp_cycles); From 4c3ed118d7a944ace4aa291c1060c198f993de1c Mon Sep 17 00:00:00 2001 From: Shreyas8612 Date: Tue, 6 Oct 2026 12:52:58 +0100 Subject: [PATCH 2/2] emulator: rename the rtl_blen latency model to rtl_measured and cite the measurement The option name rtl_blen read like a different BLEN. It is the RTL-measured accumulate model and uses the normal TRANSACTIONAL.CONFIG.BLEN, so call it rtl_measured (MatrixLatencyModel::RtlMeasured). Re-measured on PLENA_RTL 783ee48 with Verilator 5.034 (SimTop, DMA-free kernels of N back-to-back accumulates, cost = slope in N): 23 cycles at BLEN=4 for MLEN 8, 16, 32 and 64, 35 at BLEN=8 for MLEN 16 and 32, and 59 at BLEN=16 for MLEN 32, identical for M_MM, M_TMM, M_BMM and M_BTMM. That is exactly 3 * BLEN + 11 at three BLEN values, independent of MLEN. The doc comments, settings comment and readme now cite this measurement, and the unit test pins the BLEN=16 point. cargo fmt --check, clippy -D warnings and cargo test --workspace pass. --- plena_settings.toml | 7 ++-- transactional_emulator/readme.md | 2 +- transactional_emulator/src/cli.rs | 2 +- transactional_emulator/src/load_config.rs | 28 +++++++------- transactional_emulator/src/matrix_machine.rs | 40 +++++++++++--------- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/plena_settings.toml b/plena_settings.toml index 5b4e889da..6c86c32c8 100644 --- a/plena_settings.toml +++ b/plena_settings.toml @@ -211,9 +211,10 @@ value = 1 [TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL] # Cycles charged per matrix-matrix accumulate (M_MM/M_TMM/M_BMM/M_BTMM): -# "mlen" = SYSTOLIC_PROCESSING_OVERHEAD + MLEN (default) -# "rtl_blen" = 3 * BLEN + 11, measured on the RTL matrix pipeline with -# Verilator (23 cycles at BLEN=4, 35 at BLEN=8) +# "mlen" = SYSTOLIC_PROCESSING_OVERHEAD + MLEN (default) +# "rtl_measured" = 3 * BLEN + 11, measured on the PLENA_RTL matrix machine +# with Verilator (23, 35, 59 cycles at BLEN 4, 8, 16; +# independent of MLEN) # Absent means "mlen". CLI override: --matrix-latency-model. value = "mlen" diff --git a/transactional_emulator/readme.md b/transactional_emulator/readme.md index 4c757d4cb..056a7ffd8 100644 --- a/transactional_emulator/readme.md +++ b/transactional_emulator/readme.md @@ -57,7 +57,7 @@ Writes a (BLEN, BLEN) accumulator matrix (`m_accum`) to the Vector SRAM. This op | Value | Cycles per accumulate | Notes | |---|---|---| | `"mlen"` (default) | `SYSTOLIC_PROCESSING_OVERHEAD + MLEN` | The historical charge; `SYSTOLIC_PROCESSING_OVERHEAD` comes from `TRANSACTIONAL.LATENCY`. | -| `"rtl_blen"` | `3 * BLEN + 11` | Measured on the RTL matrix pipeline with Verilator: 23 cycles per accumulate at BLEN=4 and 35 at BLEN=8. Matrix ops are serialized on the RTL (each accumulate drains before the next starts) and the MLEN-deep reduction is spread across parallel sub-arrays, so the cost scales with BLEN rather than MLEN. | +| `"rtl_measured"` | `3 * BLEN + 11` | Issue-to-issue cost of back-to-back accumulates on the PLENA_RTL matrix machine (`mxint_systolic_mcu`, KLEN = BLEN), measured with Verilator 5.034 on SimTop at PLENA_RTL 783ee48: 23 cycles at BLEN=4 (MLEN 8, 16, 32, 64), 35 at BLEN=8 (MLEN 16, 32) and 59 at BLEN=16 (MLEN 32), identical for `M_MM`, `M_TMM`, `M_BMM` and `M_BTMM`. The RTL drains each accumulate before starting the next and splits the MLEN-deep reduction across MLEN / BLEN parallel mini-arrays, so the cost depends on BLEN only. | The key is optional: a settings file without it charges the `mlen` model, so existing runs are unchanged. Matrix-vector ops and write-outs are not affected by the setting. diff --git a/transactional_emulator/src/cli.rs b/transactional_emulator/src/cli.rs index c9b534207..a8487bac5 100644 --- a/transactional_emulator/src/cli.rs +++ b/transactional_emulator/src/cli.rs @@ -171,7 +171,7 @@ pub(crate) struct Opts { /// Override the cycles charged per matrix-matrix accumulate (M_MM, M_TMM, /// M_BMM, M_BTMM); default: MATRIX_LATENCY_MODEL in plena_settings.toml, /// mlen. `mlen` charges SYSTOLIC_PROCESSING_OVERHEAD + MLEN cycles; - /// `rtl_blen` charges the RTL-measured 3 * BLEN + 11 cycles. + /// `rtl_measured` charges the RTL-measured 3 * BLEN + 11 cycles. pub(crate) matrix_latency_model: Option, #[arg(long)] diff --git a/transactional_emulator/src/load_config.rs b/transactional_emulator/src/load_config.rs index 81955b800..4dbb1edc3 100644 --- a/transactional_emulator/src/load_config.rs +++ b/transactional_emulator/src/load_config.rs @@ -27,17 +27,19 @@ pub enum MatrixLatencyModel { /// `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`: the historical charge. #[default] Mlen, - /// `3 * BLEN + 11`: measured on the RTL matrix pipeline with Verilator - /// (23 cycles at BLEN=4, 35 at BLEN=8). - #[value(name = "rtl_blen")] - RtlBlen, + /// `3 * BLEN + 11`: the accumulate cost measured on the PLENA_RTL matrix + /// machine with Verilator (23, 35 and 59 cycles at BLEN 4, 8 and 16; MLEN + /// does not enter). "RTL" names where the numbers come from; BLEN itself + /// is the normal `TRANSACTIONAL.CONFIG.BLEN`. + #[value(name = "rtl_measured")] + RtlMeasured, } impl MatrixLatencyModel { pub const fn as_str(self) -> &'static str { match self { Self::Mlen => "mlen", - Self::RtlBlen => "rtl_blen", + Self::RtlMeasured => "rtl_measured", } } } @@ -48,9 +50,9 @@ impl FromStr for MatrixLatencyModel { fn from_str(s: &str) -> Result { match s.trim().to_ascii_lowercase().as_str() { "mlen" => Ok(Self::Mlen), - "rtl_blen" => Ok(Self::RtlBlen), + "rtl_measured" => Ok(Self::RtlMeasured), other => Err(format!( - "unsupported matrix latency model {other:?} (expected \"mlen\" or \"rtl_blen\")" + "unsupported matrix latency model {other:?} (expected \"mlen\" or \"rtl_measured\")" )), } } @@ -156,7 +158,7 @@ pub struct ConfigSection { #[serde(rename = "MAX_LOOP_INSTRUCTIONS")] pub max_loop_instructions: ConfigValueUsize, /// Per-accumulate latency model for matrix-matrix ops, "mlen" or - /// "rtl_blen". Optional so settings files that predate the key keep + /// "rtl_measured". Optional so settings files that predate the key keep /// working; absent means mlen. #[serde(rename = "MATRIX_LATENCY_MODEL", default)] pub matrix_latency_model: Option, @@ -778,11 +780,11 @@ mod tests { MatrixLatencyModel::Mlen ); assert_eq!( - "RTL_BLEN".parse::().unwrap(), - MatrixLatencyModel::RtlBlen + "RTL_MEASURED".parse::().unwrap(), + MatrixLatencyModel::RtlMeasured ); assert!("blen".parse::().is_err()); - assert_eq!(MatrixLatencyModel::RtlBlen.to_string(), "rtl_blen"); + assert_eq!(MatrixLatencyModel::RtlMeasured.to_string(), "rtl_measured"); } #[test] @@ -835,12 +837,12 @@ mod tests { ); let overridden = format!( - "{stripped}\n[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]\nvalue = \"rtl_blen\"\n" + "{stripped}\n[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]\nvalue = \"rtl_measured\"\n" ); let settings: PlenaSettings = toml::from_str(&overridden).unwrap(); assert_eq!( settings.transactional.config.matrix_latency_model(), - MatrixLatencyModel::RtlBlen + MatrixLatencyModel::RtlMeasured ); } diff --git a/transactional_emulator/src/matrix_machine.rs b/transactional_emulator/src/matrix_machine.rs index ffb0e7d55..c6f423605 100644 --- a/transactional_emulator/src/matrix_machine.rs +++ b/transactional_emulator/src/matrix_machine.rs @@ -26,9 +26,9 @@ use crate::runtime_config::{MATRIX_LATENCY_MODEL, SYSTOLIC_PROCESSING_OVERHEAD}; const ACCUM_OPTS: (tch::Kind, tch::Device) = (tch::Kind::Float, tch::Device::Cpu); /// Slope of the RTL-measured matrix-matrix accumulate cost, in cycles per -/// BLEN row. +/// unit of BLEN (see [`matrix_accumulate_cycles`] for the measurement). const RTL_ACCUMULATE_CYCLES_PER_ROW: u32 = 3; -/// Fixed part of the RTL-measured matrix-matrix accumulate cost, in cycles. +/// Intercept of the RTL-measured matrix-matrix accumulate cost, in cycles. const RTL_ACCUMULATE_FIXED_CYCLES: u32 = 11; /// Cycles charged for one matrix-matrix accumulate (`M_MM`, `M_TMM`, `M_BMM`, @@ -36,13 +36,15 @@ const RTL_ACCUMULATE_FIXED_CYCLES: u32 = 11; /// /// * [`MatrixLatencyModel::Mlen`]: `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`, the /// historical charge. -/// * [`MatrixLatencyModel::RtlBlen`]: `3 * BLEN + 11`. Verilator runs of the -/// RTL matrix pipeline measured 23 cycles per accumulate at BLEN=4 and 35 at -/// BLEN=8 (matrix ops are serialized: each accumulate drains before the next -/// starts); those two points fix the slope at 3 cycles per BLEN row and the -/// fixed overhead at 11 cycles. MLEN does not enter because the MLEN-deep -/// reduction is spread across parallel sub-arrays, and the -/// `SYSTOLIC_PROCESSING_OVERHEAD` knob is part of the `mlen` model only. +/// * [`MatrixLatencyModel::RtlMeasured`]: `3 * BLEN + 11`, the issue-to-issue +/// cost of back-to-back accumulates on the PLENA_RTL matrix machine +/// (`matrix_machine.sv` over `mxint_systolic_mcu.sv`, KLEN = BLEN), measured +/// with Verilator 5.034 on SimTop at PLENA_RTL 783ee48: 23 cycles at BLEN=4 +/// (MLEN 8, 16, 32, 64), 35 at BLEN=8 (MLEN 16, 32) and 59 at BLEN=16 +/// (MLEN 32), identical for all four opcodes. The RTL drains each accumulate +/// before starting the next, and the MLEN-deep reduction is split across +/// MLEN / BLEN parallel mini-arrays, so the cost depends on BLEN only. The +/// `SYSTOLIC_PROCESSING_OVERHEAD` knob belongs to the `mlen` model. pub(crate) fn matrix_accumulate_cycles( model: MatrixLatencyModel, blen: u32, @@ -51,7 +53,7 @@ pub(crate) fn matrix_accumulate_cycles( ) -> u32 { match model { MatrixLatencyModel::Mlen => systolic_overhead + mlen, - MatrixLatencyModel::RtlBlen => { + MatrixLatencyModel::RtlMeasured => { RTL_ACCUMULATE_CYCLES_PER_ROW * blen + RTL_ACCUMULATE_FIXED_CYCLES } } @@ -620,16 +622,18 @@ mod tests { #[test] fn accumulate_cycles_pin_both_models() { - use MatrixLatencyModel::{Mlen, RtlBlen}; + use MatrixLatencyModel::{Mlen, RtlMeasured}; // Historical charge: overhead + MLEN, independent of BLEN. assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 0), 64); assert_eq!(matrix_accumulate_cycles(Mlen, 8, 64, 0), 64); assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 5), 69); - // RTL measurement: 23 cycles at BLEN=4 and 35 at BLEN=8, independent - // of MLEN and of the overhead knob. - assert_eq!(matrix_accumulate_cycles(RtlBlen, 4, 64, 0), 23); - assert_eq!(matrix_accumulate_cycles(RtlBlen, 8, 64, 0), 35); - assert_eq!(matrix_accumulate_cycles(RtlBlen, 8, 1024, 5), 35); + // RTL measurement: 23, 35 and 59 cycles at BLEN 4, 8 and 16, + // independent of MLEN and of the overhead knob. + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 4, 8, 0), 23); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 4, 64, 0), 23); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 8, 64, 0), 35); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 16, 32, 0), 59); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 8, 1024, 5), 35); } fn bf16_plain() -> MxDataType { @@ -713,10 +717,10 @@ mod tests { } #[tokio::test] - async fn rtl_blen_model_changes_only_the_accumulate_charge() { + async fn rtl_measured_model_changes_only_the_accumulate_charge() { let executor = Executor::new(); let (mut machine, vram, out) = make_machine(8).await; - machine.set_latency_model(MatrixLatencyModel::RtlBlen); + machine.set_latency_model(MatrixLatencyModel::RtlMeasured); executor.spawn(async move { machine.mm(0, 0).await;