diff --git a/plena_settings.toml b/plena_settings.toml index 05fddad90..6c86c32c8 100644 --- a/plena_settings.toml +++ b/plena_settings.toml @@ -209,6 +209,15 @@ dc_lib_dis = 1 [TRANSACTIONAL.CONFIG.DC_EN] value = 1 +[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL] +# Cycles charged per matrix-matrix accumulate (M_MM/M_TMM/M_BMM/M_BTMM): +# "mlen" = SYSTOLIC_PROCESSING_OVERHEAD + MLEN (default) +# "rtl_measured" = 3 * BLEN + 11, measured on the PLENA_RTL matrix machine +# with Verilator (23, 35, 59 cycles at BLEN 4, 8, 16; +# independent of MLEN) +# Absent means "mlen". CLI override: --matrix-latency-model. +value = "mlen" + [TRANSACTIONAL.CONFIG.MAX_LOOP_INSTRUCTIONS] value = 100000 diff --git a/transactional_emulator/readme.md b/transactional_emulator/readme.md index e6c5eb713..056a7ffd8 100644 --- a/transactional_emulator/readme.md +++ b/transactional_emulator/readme.md @@ -50,6 +50,17 @@ The simulator integrates **Ramulator 2** for High-Bandwidth Memory (HBM) modelin Writes a (BLEN, BLEN) accumulator matrix (`m_accum`) to the Vector SRAM. This operation loads a (BLEN, VLEN) matrix from HBM and uses a mask to write to the Vector SRAM. +### Accumulate latency model + +`TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL` in `plena_settings.toml` (command-line override `--matrix-latency-model`) selects the cycles charged for each matrix-matrix accumulate (`M_MM`, `M_TMM`, `M_BMM`, `M_BTMM`): + +| Value | Cycles per accumulate | Notes | +|---|---|---| +| `"mlen"` (default) | `SYSTOLIC_PROCESSING_OVERHEAD + MLEN` | The historical charge; `SYSTOLIC_PROCESSING_OVERHEAD` comes from `TRANSACTIONAL.LATENCY`. | +| `"rtl_measured"` | `3 * BLEN + 11` | Issue-to-issue cost of back-to-back accumulates on the PLENA_RTL matrix machine (`mxint_systolic_mcu`, KLEN = BLEN), measured with Verilator 5.034 on SimTop at PLENA_RTL 783ee48: 23 cycles at BLEN=4 (MLEN 8, 16, 32, 64), 35 at BLEN=8 (MLEN 16, 32) and 59 at BLEN=16 (MLEN 32), identical for `M_MM`, `M_TMM`, `M_BMM` and `M_BTMM`. The RTL drains each accumulate before starting the next and splits the MLEN-deep reduction across MLEN / BLEN parallel mini-arrays, so the cost depends on BLEN only. | + +The key is optional: a settings file without it charges the `mlen` model, so existing runs are unchanged. Matrix-vector ops and write-outs are not affected by the setting. + ## Notes diff --git a/transactional_emulator/src/cli.rs b/transactional_emulator/src/cli.rs index 4f2de4650..a8487bac5 100644 --- a/transactional_emulator/src/cli.rs +++ b/transactional_emulator/src/cli.rs @@ -4,6 +4,8 @@ use std::path::{Path, PathBuf}; pub(crate) use clap::Parser; use clap::ValueEnum; +use crate::load_config::MatrixLatencyModel; + /// Log level filter for the tracing subscriber. /// /// When passed via `--log-level`, this fully overrides the `RUST_LOG` @@ -165,6 +167,13 @@ pub(crate) struct Opts { /// the default ../plena_settings.toml lookup. pub(crate) settings: Option, + #[arg(long, value_enum)] + /// Override the cycles charged per matrix-matrix accumulate (M_MM, M_TMM, + /// M_BMM, M_BTMM); default: MATRIX_LATENCY_MODEL in plena_settings.toml, + /// mlen. `mlen` charges SYSTOLIC_PROCESSING_OVERHEAD + MLEN cycles; + /// `rtl_measured` charges the RTL-measured 3 * BLEN + 11 cycles. + pub(crate) matrix_latency_model: Option, + #[arg(long)] /// Optional generated ASM source used to derive PC-to-stage labels for a /// runtime stage profile. This is diagnostic only; normal runs omit it. diff --git a/transactional_emulator/src/load_config.rs b/transactional_emulator/src/load_config.rs index 5b73badbe..4dbb1edc3 100644 --- a/transactional_emulator/src/load_config.rs +++ b/transactional_emulator/src/load_config.rs @@ -1,6 +1,6 @@ // load_config.rs use serde::{Deserialize, Serialize}; -use std::{env, fs, sync::LazyLock}; +use std::{env, fmt, fs, str::FromStr, sync::LazyLock}; // Import the types from your main module use quantize::{DataType, FpType, IntType, MxDataType}; @@ -15,6 +15,55 @@ pub struct ConfigValueUsize { pub value: usize, } +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ConfigValueString { + pub value: String, +} + +/// Cycles charged per matrix-matrix accumulate +/// (`TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum MatrixLatencyModel { + /// `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`: the historical charge. + #[default] + Mlen, + /// `3 * BLEN + 11`: the accumulate cost measured on the PLENA_RTL matrix + /// machine with Verilator (23, 35 and 59 cycles at BLEN 4, 8 and 16; MLEN + /// does not enter). "RTL" names where the numbers come from; BLEN itself + /// is the normal `TRANSACTIONAL.CONFIG.BLEN`. + #[value(name = "rtl_measured")] + RtlMeasured, +} + +impl MatrixLatencyModel { + pub const fn as_str(self) -> &'static str { + match self { + Self::Mlen => "mlen", + Self::RtlMeasured => "rtl_measured", + } + } +} + +impl FromStr for MatrixLatencyModel { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "mlen" => Ok(Self::Mlen), + "rtl_measured" => Ok(Self::RtlMeasured), + other => Err(format!( + "unsupported matrix latency model {other:?} (expected \"mlen\" or \"rtl_measured\")" + )), + } + } +} + +impl fmt::Display for MatrixLatencyModel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct LatencyValue { pub dc_lib_en: u32, @@ -108,6 +157,23 @@ pub struct ConfigSection { pub dc_en: ConfigValue, #[serde(rename = "MAX_LOOP_INSTRUCTIONS")] pub max_loop_instructions: ConfigValueUsize, + /// Per-accumulate latency model for matrix-matrix ops, "mlen" or + /// "rtl_measured". Optional so settings files that predate the key keep + /// working; absent means mlen. + #[serde(rename = "MATRIX_LATENCY_MODEL", default)] + pub matrix_latency_model: Option, +} + +impl ConfigSection { + pub fn matrix_latency_model(&self) -> MatrixLatencyModel { + match &self.matrix_latency_model { + None => MatrixLatencyModel::default(), + Some(model) => model + .value + .parse() + .unwrap_or_else(|err| panic!("TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL: {err}")), + } + } } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -181,6 +247,7 @@ impl Default for AcceleratorConfig { hbm_v_writeback_amount: ConfigValue { value: 16 }, dc_en: ConfigValue { value: 1 }, max_loop_instructions: ConfigValueUsize { value: 10000 }, + matrix_latency_model: None, }, precision: PrecisionSection { matrix_sram_type: MxDataTypeConfig { @@ -527,6 +594,10 @@ pub fn systolic_processing_overhead() -> u32 { get_dc_lib_value(&CONFIG.latency.systolic_processing_overhead) } +pub fn matrix_latency_model() -> MatrixLatencyModel { + CONFIG.config.matrix_latency_model() +} + // pub fn vector_ps_cycles() -> u32 { // get_dc_lib_value(&CONFIG.latency.vector_ps_cycles) // } @@ -702,6 +773,79 @@ mod tests { assert_eq!(cfg.config.max_loop_instructions.value, 10000); } + #[test] + fn test_matrix_latency_model_parses_case_insensitively() { + assert_eq!( + "mlen".parse::().unwrap(), + MatrixLatencyModel::Mlen + ); + assert_eq!( + "RTL_MEASURED".parse::().unwrap(), + MatrixLatencyModel::RtlMeasured + ); + assert!("blen".parse::().is_err()); + assert_eq!(MatrixLatencyModel::RtlMeasured.to_string(), "rtl_measured"); + } + + #[test] + fn test_default_matrix_latency_model_is_mlen() { + let cfg = AcceleratorConfig::default(); + assert!(cfg.config.matrix_latency_model.is_none()); + assert_eq!(cfg.config.matrix_latency_model(), MatrixLatencyModel::Mlen); + } + + /// Drop the `[header]` table (up to the next table) from a TOML text. + fn without_table(text: &str, header: &str) -> String { + let mut out = String::new(); + let mut skipping = false; + for line in text.lines() { + if line.trim() == header { + skipping = true; + continue; + } + if skipping && line.trim_start().starts_with('[') { + skipping = false; + } + if !skipping { + out.push_str(line); + out.push('\n'); + } + } + out + } + + #[test] + fn test_settings_file_matrix_latency_model() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/../plena_settings.toml"); + let text = fs::read_to_string(path).unwrap(); + + // The checked-in value is the historical charge. + let settings: PlenaSettings = toml::from_str(&text).unwrap(); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::Mlen + ); + + // A settings file without the key means the same thing. + let stripped = without_table(&text, "[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]"); + assert!(!stripped.contains("MATRIX_LATENCY_MODEL")); + let settings: PlenaSettings = toml::from_str(&stripped).unwrap(); + assert!(settings.transactional.config.matrix_latency_model.is_none()); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::Mlen + ); + + let overridden = format!( + "{stripped}\n[TRANSACTIONAL.CONFIG.MATRIX_LATENCY_MODEL]\nvalue = \"rtl_measured\"\n" + ); + let settings: PlenaSettings = toml::from_str(&overridden).unwrap(); + assert_eq!( + settings.transactional.config.matrix_latency_model(), + MatrixLatencyModel::RtlMeasured + ); + } + #[test] fn test_default_precision_types_convert() { let cfg = AcceleratorConfig::default(); diff --git a/transactional_emulator/src/matrix_machine.rs b/transactional_emulator/src/matrix_machine.rs index ce10045e7..c6f423605 100644 --- a/transactional_emulator/src/matrix_machine.rs +++ b/transactional_emulator/src/matrix_machine.rs @@ -18,12 +18,47 @@ use quantize::QuantTensor; use sram::{MatrixSram, VectorSram, assert_multiple_of, multiple_and_offset}; use tch::{IndexOp, Tensor}; +use crate::load_config::MatrixLatencyModel; use crate::matrix_core::{MatrixCore, MatrixCoreProfile}; -use crate::runtime_config::SYSTOLIC_PROCESSING_OVERHEAD; +use crate::runtime_config::{MATRIX_LATENCY_MODEL, SYSTOLIC_PROCESSING_OVERHEAD}; /// Tensor allocation options used for every accumulator buffer (f32 on CPU). const ACCUM_OPTS: (tch::Kind, tch::Device) = (tch::Kind::Float, tch::Device::Cpu); +/// Slope of the RTL-measured matrix-matrix accumulate cost, in cycles per +/// unit of BLEN (see [`matrix_accumulate_cycles`] for the measurement). +const RTL_ACCUMULATE_CYCLES_PER_ROW: u32 = 3; +/// Intercept of the RTL-measured matrix-matrix accumulate cost, in cycles. +const RTL_ACCUMULATE_FIXED_CYCLES: u32 = 11; + +/// Cycles charged for one matrix-matrix accumulate (`M_MM`, `M_TMM`, `M_BMM`, +/// `M_BTMM`) under `model`. +/// +/// * [`MatrixLatencyModel::Mlen`]: `SYSTOLIC_PROCESSING_OVERHEAD + MLEN`, the +/// historical charge. +/// * [`MatrixLatencyModel::RtlMeasured`]: `3 * BLEN + 11`, the issue-to-issue +/// cost of back-to-back accumulates on the PLENA_RTL matrix machine +/// (`matrix_machine.sv` over `mxint_systolic_mcu.sv`, KLEN = BLEN), measured +/// with Verilator 5.034 on SimTop at PLENA_RTL 783ee48: 23 cycles at BLEN=4 +/// (MLEN 8, 16, 32, 64), 35 at BLEN=8 (MLEN 16, 32) and 59 at BLEN=16 +/// (MLEN 32), identical for all four opcodes. The RTL drains each accumulate +/// before starting the next, and the MLEN-deep reduction is split across +/// MLEN / BLEN parallel mini-arrays, so the cost depends on BLEN only. The +/// `SYSTOLIC_PROCESSING_OVERHEAD` knob belongs to the `mlen` model. +pub(crate) fn matrix_accumulate_cycles( + model: MatrixLatencyModel, + blen: u32, + mlen: u32, + systolic_overhead: u32, +) -> u32 { + match model { + MatrixLatencyModel::Mlen => systolic_overhead + mlen, + MatrixLatencyModel::RtlMeasured => { + RTL_ACCUMULATE_CYCLES_PER_ROW * blen + RTL_ACCUMULATE_FIXED_CYCLES + } + } +} + /// Executes matrix opcodes by reading tiles/vectors from `mram`/`vram`, /// running matmul in f32, and accumulating into per-shape buffers that are /// later flushed via the `*_wo` ops. @@ -39,6 +74,7 @@ pub(crate) struct MatrixMachine { blen: u32, broadcast_amount: u32, core: MatrixCore, + latency_model: MatrixLatencyModel, } impl MatrixMachine { @@ -90,6 +126,7 @@ impl MatrixMachine { blen, broadcast_amount, core: MatrixCore::new(core_profile), + latency_model: *MATRIX_LATENCY_MODEL, } } @@ -97,10 +134,28 @@ impl MatrixMachine { self.core.profile() } + pub(crate) fn latency_model(&self) -> MatrixLatencyModel { + self.latency_model + } + + pub(crate) fn set_latency_model(&mut self, model: MatrixLatencyModel) { + self.latency_model = model; + } + fn core(&self) -> MatrixCore { self.core } + /// Cycles charged for one matrix-matrix accumulate under the active model. + fn accumulate_cycles(&self) -> u32 { + matrix_accumulate_cycles( + self.latency_model, + self.blen, + self.mlen, + *SYSTOLIC_PROCESSING_OVERHEAD, + ) + } + pub(crate) async fn mm(&mut self, m_addr: u32, v_addr: u32) { let (mat_base, mat_offset) = multiple_and_offset(m_addr, self.mlen * self.mlen); // Row-granular projection ABI: the M_MM column stride is `blen * mlen` @@ -122,9 +177,7 @@ impl MatrixMachine { mat_offset as i64..(mat_offset as i64 + self.blen as i64), )); let mut tensors = Vec::with_capacity(self.blen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.blen { tensors.push( self.vram @@ -163,9 +216,7 @@ impl MatrixMachine { )); let mut tensors = Vec::with_capacity(self.mlen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.mlen { tensors.push( self.vram @@ -281,9 +332,7 @@ impl MatrixMachine { )); let mut tensors = Vec::with_capacity(self.mlen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; // B, S, H, D for i in 0..self.mlen { tensors.push( @@ -403,9 +452,7 @@ impl MatrixMachine { .transpose(-1, -2) .i((.., mat_offset as i64..(mat_offset + self.blen) as i64)); let mut tensors = Vec::with_capacity(self.blen as usize); - self.core() - .compute(*SYSTOLIC_PROCESSING_OVERHEAD + self.mlen) - .await; + self.core().compute(self.accumulate_cycles()).await; for i in 0..self.blen { tensors.push( self.vram @@ -569,9 +616,26 @@ mod tests { use tch::Tensor; use super::*; + use crate::load_config::MatrixLatencyModel; use crate::matrix_core::MatrixCoreProfile; use crate::runtime_config::SYSTOLIC_PROCESSING_OVERHEAD; + #[test] + fn accumulate_cycles_pin_both_models() { + use MatrixLatencyModel::{Mlen, RtlMeasured}; + // Historical charge: overhead + MLEN, independent of BLEN. + assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 0), 64); + assert_eq!(matrix_accumulate_cycles(Mlen, 8, 64, 0), 64); + assert_eq!(matrix_accumulate_cycles(Mlen, 4, 64, 5), 69); + // RTL measurement: 23, 35 and 59 cycles at BLEN 4, 8 and 16, + // independent of MLEN and of the overhead knob. + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 4, 8, 0), 23); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 4, 64, 0), 23); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 8, 64, 0), 35); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 16, 32, 0), 59); + assert_eq!(matrix_accumulate_cycles(RtlMeasured, 8, 1024, 5), 35); + } + fn bf16_plain() -> MxDataType { MxDataType::Plain(DataType::Fp(FpType::BF16)) } @@ -652,6 +716,31 @@ mod tests { assert!(b1.equal(&Tensor::from_slice(&[5.0f32, 6.0, 0.0, 0.0]))); } + #[tokio::test] + async fn rtl_measured_model_changes_only_the_accumulate_charge() { + let executor = Executor::new(); + let (mut machine, vram, out) = make_machine(8).await; + machine.set_latency_model(MatrixLatencyModel::RtlMeasured); + + executor.spawn(async move { + machine.mm(0, 0).await; + machine.mm_wo(out, 1).await; + }); + executor.enter(Instant::ETERNITY).await; + + // BLEN = 2: 3 * 2 + 11 cycles for the accumulate, then the write-out's + // one cycle as before. + let expected_cycles = 3 * 2 + 11 + 1; + assert_eq!( + executor.now(), + Instant::INIT + Duration::from_nanos(expected_cycles) + ); + let a0 = vram.read(8).await.as_tensor().shallow_clone(); + let a1 = vram.read(12).await.as_tensor().shallow_clone(); + assert!(a0.equal(&Tensor::from_slice(&[1.0f32, 2.0, 0.0, 0.0]))); + assert!(a1.equal(&Tensor::from_slice(&[5.0f32, 6.0, 0.0, 0.0]))); + } + #[tokio::test] async fn mm_row_granular_offset_selects_column_block() { // Row-granular M_MM ABI (compiler e852c88): the within-tile matrix offset is diff --git a/transactional_emulator/src/runner.rs b/transactional_emulator/src/runner.rs index 04a129c16..fb7ead921 100644 --- a/transactional_emulator/src/runner.rs +++ b/transactional_emulator/src/runner.rs @@ -118,10 +118,16 @@ pub(crate) async fn run_from_cli() { *VECTOR_SRAM_TYPE, )); // Vector SRAM - let m_machine = MatrixMachine::new(mram, vram.clone(), *MLEN, *HLEN, *BLEN, *BROADCAST_AMOUNT); + let mut m_machine = + MatrixMachine::new(mram, vram.clone(), *MLEN, *HLEN, *BLEN, *BROADCAST_AMOUNT); + // `--matrix-latency-model` wins over MATRIX_LATENCY_MODEL in plena_settings.toml. + if let Some(model) = opts.matrix_latency_model { + m_machine.set_latency_model(model); + } let big_core = m_machine.core_profile(); let tail_core = MatrixCoreProfile::tail_4x256(); tracing::info!( + matrix_latency_model = %m_machine.latency_model(), big_core = big_core.name, big_rows = big_core.rows, big_cols = big_core.cols, diff --git a/transactional_emulator/src/runtime_config.rs b/transactional_emulator/src/runtime_config.rs index ff3450ef1..c11b3f0aa 100644 --- a/transactional_emulator/src/runtime_config.rs +++ b/transactional_emulator/src/runtime_config.rs @@ -9,6 +9,8 @@ pub(crate) const PERIOD: Duration = Duration::from_nanos(1); pub(crate) static SYSTOLIC_PROCESSING_OVERHEAD: LazyLock = LazyLock::new(systolic_processing_overhead); +pub(crate) static MATRIX_LATENCY_MODEL: LazyLock = + LazyLock::new(matrix_latency_model); pub(crate) static VECTOR_ADD_CYCLES: LazyLock = LazyLock::new(vector_add_cycles); pub(crate) static VECTOR_MUL_CYCLES: LazyLock = LazyLock::new(vector_mul_cycles); pub(crate) static VECTOR_EXP_CYCLES: LazyLock = LazyLock::new(vector_exp_cycles);