From c49bff2265205bd61b6cc1ad6e72ea2c96d563f7 Mon Sep 17 00:00:00 2001 From: "Felix C. A. Auer" <10127354+FelixCAAuer@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:38:30 +0200 Subject: [PATCH] Add information about CPU, RAM etc. to .sqlite result files --- CLAUDE.md | 2 ++ README.md | 1 + SQLiteWriter.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 8bac2c1..bf5884e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ See `README.md` for usage, key concepts, and data structure. - Sequential reads happen first (`Global_Parameters`, `Global_Scenarios`, `Power_Parameters`), because subsequent file selection depends on `pEnable*` flags. - All remaining files are read in parallel via `ThreadPoolExecutor` — order of assignment is non-deterministic, so no file read may depend on another parallel read. +- `Power_VRESProfiles.xlsx` and `Power_Inflows.xlsx` are **optional**: a missing file leaves the attribute set to `None` (it still exists, so `hasattr` is not enough). Every consumer (scaling, clustering, `filter_zone`, `merge_generators`, `to_full_hourly_model`, `write_caseStudy`) must check `is not None`; the LEGO `vres`/`storage` modules do the same, and `vres` raises if a VRES generator has neither a profile nor an inflow. - `dPower_WeightsRP` is **computed** from `dPower_Hindex` (counting occurrences per `rp`); if a `Power_WeightsRP.xlsx` file also exists, it is read and compared — a mismatch triggers a warning but uses the **file** value, not the computed one. - `merge_single_node_buses()` preserves the `z` (zone) column as a sorted unique union string of all merged zones (e.g. `"R1_R2"`). This is documented in root `CLAUDE.md` as well. - `merge_generators()` collapses all generators sharing the same `(tec, i)` into one representative generator with ID `"{i}_{tec}"`. It must be called after construction (scaling is not required). VRESProfiles and Inflows are merged **before** VRES so that the original per-generator `MaxProd` weights are available for the capacity-factor weighted average. Generators in VRESProfiles/Inflows that have no matching entry in `dPower_VRES` (left-join miss) are grouped under `(tec=NaN, i=NaN)` — filter to a single scenario first to avoid mixing scenarios in the groupby. @@ -36,6 +37,7 @@ See `README.md` for usage, key concepts, and data structure. - `model_to_sqlite()` automatically calls `add_objective_decomposition_to_sqlite()` and `add_dual_values_to_sqlite()` — these do not need to be called separately. - `add_run_parameters_to_sqlite()` stores all run configuration in the `run_parameters` table. Evaluation scripts should read from this table (more reliable than filename parsing). +- `add_solver_statistics_to_sqlite()` adds the machine columns from `machine_info()` (CPU model: Windows registry `ProcessorNameString`, Linux `/proc/cpuinfo`; cores/affinity via `psutil`) and the resource columns from `lego.solver_attributes` (`Threads`, `MaxMemUsed` — Gurobi path only) and `process_memory_gb()` (shared with `research/MK/Markov.py`; Windows `peak_wset`, Linux `ru_maxrss` in kB). `None` values are dropped, so the columns may be missing (non-Gurobi solvers, old files). `process_peak_rss_gb` is the process-lifetime peak, so it covers model building and all earlier solves of the same process. - Pyomo component types not handled by the writer emit a `printer.warning()` and are skipped silently — add new `case` branches if new Pyomo types need to be stored. ### Utilities diff --git a/README.md b/README.md index 9670220..ac4cbc2 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ add_run_parameters_to_sqlite("output/results.sqlite", zoi="R1", dc_buffer=2) - `model_to_sqlite()` exports all Pyomo variables, parameters, and sets; automatically appends objective decomposition and dual values. - `add_run_parameters_to_sqlite()` creates a `run_parameters` table — evaluation scripts should read from this table rather than parsing filenames. +- `add_solver_statistics_to_sqlite()` creates a `solver_statistics` table: status, termination condition, solver time, bounds, model size, work units and MIP gap, plus the machine and resources of the solve — `host`, `cpu_model`, `cpu_physical_cores`, `cpu_logical_cores`, `cpus_available` (CPU affinity, e.g. a Slurm allocation), `threads` (Gurobi threads; `0`/automatic is stored as the logical core count), `solver_max_mem_gb` (Gurobi's peak memory) and `process_peak_rss_gb` (peak RAM of the whole Python process). Works on Windows and Linux. ## Caller (Parallel Job Runner) diff --git a/SQLiteWriter.py b/SQLiteWriter.py index 7c6d672..a83dca0 100644 --- a/SQLiteWriter.py +++ b/SQLiteWriter.py @@ -1,6 +1,8 @@ from __future__ import annotations import os +import platform +import socket import sqlite3 from typing import TYPE_CHECKING @@ -70,6 +72,59 @@ def model_to_sqlite(model: pyo.base.ConcreteModel, filename: str) -> None: pass +def process_memory_gb() -> tuple[float | None, float | None]: + """(current RSS, peak RSS) of this process in GB. Peak is the lifetime high-water mark of the process.""" + rss = peak = None + try: + import psutil + mem = psutil.Process().memory_info() + rss = mem.rss / 1024 ** 3 + if getattr(mem, 'peak_wset', None) is not None: # Windows + peak = mem.peak_wset / 1024 ** 3 + except Exception: + pass + if peak is None: + try: + import resource # Linux: ru_maxrss in kB + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 ** 2 + except Exception: + pass + return rss, peak + + +def _cpu_model() -> str | None: + """CPU model name as the OS reports it (e.g. '13th Gen Intel(R) Core(TM) i9-13900').""" + try: + if platform.system() == 'Windows': + import winreg + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'HARDWARE\DESCRIPTION\System\CentralProcessor\0') as key: + return str(winreg.QueryValueEx(key, 'ProcessorNameString')[0]).strip() + if platform.system() == 'Linux': + with open('/proc/cpuinfo') as f: + for line in f: + if line.startswith('model name'): + return line.split(':', 1)[1].strip() + except Exception: + pass + return platform.processor() or None + + +def machine_info() -> dict: + """Host, CPU model and core counts of the machine running this process (missing values omitted). + `cpus_available` is the process's CPU affinity - under Slurm the cores of the allocation, not of the node.""" + info = {'host': socket.gethostname(), 'cpu_model': _cpu_model()} + try: + import psutil + info['cpu_physical_cores'] = psutil.cpu_count(logical=False) + info['cpu_logical_cores'] = psutil.cpu_count(logical=True) + info['cpus_available'] = len(psutil.Process().cpu_affinity()) + except Exception: + info.setdefault('cpu_logical_cores', os.cpu_count()) + if hasattr(os, 'sched_getaffinity'): # Linux + info['cpus_available'] = len(os.sched_getaffinity(0)) + return {k: v for k, v in info.items() if v is not None} + + def add_solver_statistics_to_sqlite(filename: str, lego: LEGO) -> None: """ Add solver statistics to an existing SQLite database, reading from a LEGO instance. @@ -135,6 +190,17 @@ def add_solver_statistics_to_sqlite(filename: str, lego: LEGO) -> None: else: printer.information("MIP gap not available: not provided and bounds not found in results") + # Machine and resources of the solve (work units / solver times are only comparable on the same CPU and threads) + stats.update(machine_info()) + solver_attributes = getattr(lego, 'solver_attributes', {}) or {} + threads = solver_attributes.get('Threads') + if threads is not None: # Gurobi's Threads parameter, 0 (automatic) = all logical cores + stats['threads'] = int(threads) if threads > 0 else stats.get('cpu_logical_cores') + if solver_attributes.get('MaxMemUsed') is not None: + stats['solver_max_mem_gb'] = float(solver_attributes['MaxMemUsed']) + stats['process_peak_rss_gb'] = process_memory_gb()[1] + stats = {k: v for k, v in stats.items() if v is not None} + # Create a DataFrame with solver statistics if stats: df = pd.DataFrame([stats])