From ff67cd516fcea45503fe1964b2943d6a17fd15f2 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:53:00 -0400 Subject: [PATCH 01/11] Rename old columns: opcn3_temp, opcn3_rh and operating_state --- quantaq_cli/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/quantaq_cli/schema.py b/quantaq_cli/schema.py index 7db7038..506e509 100644 --- a/quantaq_cli/schema.py +++ b/quantaq_cli/schema.py @@ -126,6 +126,8 @@ "opcn3_pm1": "opc_pm1", "opcn3_pm25": "opc_pm25", "opcn3_pm10": "opc_pm10", + "opcn3_temp": "opc_temp", + "opcn3_rh": "opc_rh", "sample_period": "opc_sample_period", "sample_flow": "opc_sample_flow", "laser_status": "opc_laser_status", @@ -151,6 +153,9 @@ # --- RHT columns --- "temp": "sample_temp", "rh": "sample_rh", + + # --- Device / metadata columns --- + "operating_state": "dd_operating_state" } # Prefixes for unstandardized column names From aada12ee2f3e1bf7ac556718e35e15b19cd6a274 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:53:20 -0400 Subject: [PATCH 02/11] Add a 2-sec tolerance for dominant tdiffs to infer data source by sampling interval --- quantaq_cli/utilities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/quantaq_cli/utilities.py b/quantaq_cli/utilities.py index 6f5536d..438d16c 100644 --- a/quantaq_cli/utilities.py +++ b/quantaq_cli/utilities.py @@ -42,9 +42,10 @@ def infer_data_source(df, tscol=None): dominant_tdiff = tdiff_counts.idxmax() - mostly_1min = dominant_tdiff == 60.0 - mostly_5sec = dominant_tdiff == 5.0 - mostly_10sec = dominant_tdiff == 10.0 + tolerance = 2.0 + mostly_1min = abs(dominant_tdiff - 60.0) <= tolerance + mostly_5sec = abs(dominant_tdiff - 5.0) <= tolerance + mostly_10sec = abs(dominant_tdiff - 10.0) <= tolerance if mostly_1min: logger.info(f"Reading mostly {dominant_tdiff}s data --> inferring database") From 43458cfa27c1a60521edae39d785d0289b56558e Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:56:15 -0400 Subject: [PATCH 03/11] Bump pre-release version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index df524be..be73d95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "quantaq-cli" -version = "1.0.0rc3" +version = "1.0.0rc4" description = "The QuantAQ Python Toolkit and CLI" authors = ["David H Hagan "] license = "Apache-2.0" From a8d7a34ddb1cf5917a9b6790ac27190e39584855 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:38:31 -0400 Subject: [PATCH 04/11] Fix: neph_bin0/opc_bin0 ratio check should occur when neph_bin0>=10, not opc_bin0>=0 --- quantaq_cli/variables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quantaq_cli/variables.py b/quantaq_cli/variables.py index 7d92b96..aac610f 100644 --- a/quantaq_cli/variables.py +++ b/quantaq_cli/variables.py @@ -155,7 +155,7 @@ # Check 2: ratio between the OPC and nephelometer is within spec Multiple( criteria=( - Single(column="opc_bin0", op=">=", value=10.0), + Single(column="neph_bin0", op=">=", value=10.0), Ratio(column_numerator="neph_bin0", column_denominator="opc_bin0", op=">", @@ -183,7 +183,7 @@ # Check 5: ratio between the OPC and nephelometer is within spec Multiple( criteria=( - Single(column="opc_bin0", op=">=", value=10.0), + Single(column="neph_bin0", op=">=", value=10.0), Ratio(column_numerator="neph_bin0", column_denominator="opc_bin0", op=">", From 5c71fe02d0ba6934e49166a6f2c09c10296ddc24 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:25:24 -0400 Subject: [PATCH 05/11] Convert fw and dd_measurement_state/dd_operating_state to nullable pandas int since older data might be missing these column values --- quantaq_cli/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quantaq_cli/schema.py b/quantaq_cli/schema.py index 506e509..f548978 100644 --- a/quantaq_cli/schema.py +++ b/quantaq_cli/schema.py @@ -110,12 +110,12 @@ ('vbat', np.float64), # --- Device / metadata columns --- - ('fw', np.int64), + ('fw', 'Int64'), # nullable pandas int ('flag', np.int64), ('connection_status', np.int16), ('iteration', np.int16), - ('dd_measurement_state', np.int64), - ('dd_operating_state', np.int64), + ('dd_measurement_state', 'Int64'), # nullable pandas int + ('dd_operating_state', 'Int64'), # nullable pandas int ] STATIC_COLUMN_RENAMES = { From 22027afccc1b6bc77d761bc0c8f29e92fae12e47 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:29:36 -0400 Subject: [PATCH 06/11] Convert nullable ints to float64 --- quantaq_cli/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quantaq_cli/schema.py b/quantaq_cli/schema.py index f548978..9bf0e51 100644 --- a/quantaq_cli/schema.py +++ b/quantaq_cli/schema.py @@ -110,12 +110,12 @@ ('vbat', np.float64), # --- Device / metadata columns --- - ('fw', 'Int64'), # nullable pandas int + ('fw', np.float64), # needs to be nullable for older data ('flag', np.int64), ('connection_status', np.int16), ('iteration', np.int16), - ('dd_measurement_state', 'Int64'), # nullable pandas int - ('dd_operating_state', 'Int64'), # nullable pandas int + ('dd_measurement_state', np.float64), # needs to be nullable for older data + ('dd_operating_state', np.float64), # needs to be nullable for older data ] STATIC_COLUMN_RENAMES = { From 78c2047b36e4602aa9870c60137cd6334afe5e1d Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:29:21 -0400 Subject: [PATCH 07/11] Add vectorized flag-aware resampling to speed things up --- quantaq_cli/toolkit/resample.py | 91 +++++++++++++-------------------- 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/quantaq_cli/toolkit/resample.py b/quantaq_cli/toolkit/resample.py index 9f5a6b8..6a93ed7 100644 --- a/quantaq_cli/toolkit/resample.py +++ b/quantaq_cli/toolkit/resample.py @@ -51,61 +51,52 @@ def _components_from_polar( df[v_col] = df[ws_col] * np.cos(wd_rad) return df -def _aggregate_group(group, agg): - """Aggregate a single resample bin using flag-aware row selection. +def _flag_aware_resample(df, rule, keys, agg): + """Vectorized flag-aware resampling. - This attempts to mirror the averaging logic in the firmware: - If the bin has at least one good row (flag == 0), only those rows - are aggregated and the resulting flag is 0. If there are no good rows (every - row in the bin is flagged), all rows are aggregated instead, and the resulting flag is the - bitwise OR of every flag value that was present in the bin. + For each resample bin: if any row has flag == 0, aggregate only those + "clean" rows and set the output flag to 0. Otherwise aggregate all rows + in the bin and set the output flag to the bitwise OR of every flag value + present. Args: - group (pd.DataFrame): All rows falling in one resample bin + df (pd.DataFrame): the input dataframe to resample + rule: Any pandas offset alias, e.g. ``"1min"``, ``"1h"``, ``"1D"``. + keys (list[str]): Column(s) to group by before resampling (e.g. + ``["sn"]``), so each device/location is resampled independently. + Pass an empty list to resample the whole frame as one series + of bins. agg (dict): Mapping of column name -> aggregation method, defined inside - resample_dataframe. - - Returns: - pd.Series: One aggregated row for this bin. + resample_dataframe. """ - if 'flag' not in group.columns: - error = ValueError("No 'flag' column found in dataframe! Cannot implement" - "flag-aware resampling. Consider calling flag_dataframe() first.") + + if "flag" not in df.columns: + error = ValueError( + "No 'flag' column found in dataframe! Cannot implement " + "flag-aware resampling. Consider calling flag_dataframe() first." + ) logger.error(error) raise error - # resample can produce empty bins so we check len(group) - if len(group): - clean_mask = group['flag'] == 0 # true for every good row + def _resampler(frame): + return frame.groupby(keys).resample(rule) if keys else frame.resample(rule) - # there's at least 1 good row --> aggregate only the good rows, and set the new flag to 0 - if clean_mask.any(): - subset = group.loc[clean_mask] - group_flag = 0 + clean_agg = _resampler(df[df["flag"] == 0]).agg(agg) - # there are no good rows --> aggregate all rows, and combine flags using bitwise OR - else: - subset = group - group_flag = int(np.bitwise_or.reduce(group['flag'].to_numpy())) - else: - # empty bin (produced by resampe) --> all columns will be nan, include the flag - subset = group - group_flag = np.nan - - # hack for naming collision issue with "first" and "last" agg methods - # (TypeError: NDFrame.first() missing 1 required positional argument: 'offset') - values = {} - for col, how in agg.items(): - if how == "first": - values[col] = subset[col].iloc[0] if len(subset) else np.nan - elif how == "last": - values[col] = subset[col].iloc[-1] if len(subset) else np.nan - else: - values[col] = subset[col].agg(how) + base_resampler = _resampler(df) + all_agg = base_resampler.agg(agg) + flag_col = base_resampler["flag"] + has_clean = flag_col.agg(lambda s: bool((s == 0).any())) + flag_or = flag_col.agg(lambda s: int(np.bitwise_or.reduce(s.to_numpy())) if len(s) else np.nan) + + clean_agg = clean_agg.reindex(all_agg.index) + has_clean = has_clean.reindex(all_agg.index) + flag_or = flag_or.reindex(all_agg.index) - values['flag'] = group_flag + out = clean_agg.where(has_clean, all_agg) + out["flag"] = np.where(has_clean, 0, flag_or) - return pd.Series(values) + return out.reset_index() def resample_dataframe( df: pd.DataFrame, @@ -208,19 +199,7 @@ def resample_dataframe( indexed = df.set_index(on) if flag_aware: - if keys: - out = ( - indexed.groupby(keys) - .resample(rule) - .apply(_aggregate_group, agg=agg) - .reset_index() - ) - else: - out = ( - indexed.resample(rule) - .apply(_aggregate_group, agg=agg) - .reset_index() - ) + out = _flag_aware_resample(indexed, rule, keys, agg) else: if keys: out = indexed.groupby(keys).resample(rule).agg(agg).reset_index() From f7d9b35681defdf1d2d2c6f22aea86e83c51bb7d Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:05:28 -0400 Subject: [PATCH 08/11] Re-add fix for naming collision --- quantaq_cli/toolkit/resample.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/quantaq_cli/toolkit/resample.py b/quantaq_cli/toolkit/resample.py index 6a93ed7..85cd581 100644 --- a/quantaq_cli/toolkit/resample.py +++ b/quantaq_cli/toolkit/resample.py @@ -192,11 +192,18 @@ def resample_dataframe( value_cols = [ c for c in df.columns if c != on and c != 'flag' and c not in keys and c not in derived ] + + # hack for naming collision issue with "first" and "last" agg methods + # (TypeError: NDFrame.first() missing 1 required positional argument: 'offset') agg = { - c: (numeric_how if is_numeric_dtype(df[c]) else nonnumeric_how) + c: ( + numeric_how if is_numeric_dtype(df[c]) + else (lambda s: s.iloc[0] if len(s) else np.nan) if nonnumeric_how == "first" + else (lambda s: s.iloc[-1] if len(s) else np.nan) if nonnumeric_how == "last" + else nonnumeric_how + ) for c in value_cols } - indexed = df.set_index(on) if flag_aware: out = _flag_aware_resample(indexed, rule, keys, agg) From d62b4e56ef9204d3cfdd4d5d013357f1a6c57277 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:41:51 -0400 Subject: [PATCH 09/11] Add a force_nonnumeric arg to resample_dataframe --- quantaq_cli/toolkit/resample.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/quantaq_cli/toolkit/resample.py b/quantaq_cli/toolkit/resample.py index 85cd581..3d6cdd7 100644 --- a/quantaq_cli/toolkit/resample.py +++ b/quantaq_cli/toolkit/resample.py @@ -108,6 +108,7 @@ def resample_dataframe( flag_aware: bool = False, numeric_how: str = "mean", nonnumeric_how: str = "first", + force_nonnumeric: str | list[str] | None = None, ) -> pd.DataFrame: """Resample a time-indexed frame, handling mixed dtypes safely. @@ -144,6 +145,9 @@ def resample_dataframe( output. numeric_how: Aggregation for numeric columns. nonnumeric_how: Aggregation for non-numeric columns. + force_nonnumeric: Column name(s) to always aggregate with + ``nonnumeric_how`` regardless of dtype, e.g. ``"fw"``, + which is numeric but shouldn't be averaged. Returns: A new frame with ``on`` (and any ``by`` keys) as columns. @@ -152,6 +156,11 @@ def resample_dataframe( df[on] = pd.to_datetime(df[on]) keys = [by] if isinstance(by, str) else list(by or []) + force_nonnumeric_set = ( + {force_nonnumeric} + if isinstance(force_nonnumeric, str) + else set(force_nonnumeric or []) + ) # When vector-averaging wind, the speed/direction columns must never be # scalar-aggregated -- drop them from the agg pass and derive them from the @@ -197,7 +206,7 @@ def resample_dataframe( # (TypeError: NDFrame.first() missing 1 required positional argument: 'offset') agg = { c: ( - numeric_how if is_numeric_dtype(df[c]) + numeric_how if is_numeric_dtype(df[c]) and c not in force_nonnumeric_set else (lambda s: s.iloc[0] if len(s) else np.nan) if nonnumeric_how == "first" else (lambda s: s.iloc[-1] if len(s) else np.nan) if nonnumeric_how == "last" else nonnumeric_how From c24bb4c310832fa8813808b990e6011b879fe4cb Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:01:12 -0400 Subject: [PATCH 10/11] Add optional data completeness requirement when resampling --- quantaq_cli/toolkit/resample.py | 98 +++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/quantaq_cli/toolkit/resample.py b/quantaq_cli/toolkit/resample.py index 3d6cdd7..3e0761d 100644 --- a/quantaq_cli/toolkit/resample.py +++ b/quantaq_cli/toolkit/resample.py @@ -51,6 +51,29 @@ def _components_from_polar( df[v_col] = df[ws_col] * np.cos(wd_rad) return df +def _min_required_count(timestamps, rule, threshold): + """The minimum number of non-null samples needed in one resampling bin to + meet the data completeness threshold. + + We infer the native sampling interval from the median tdiff (spacing between + consecutive timestamps in ``timestamps``), then compared to the resampling + bin width to estimate how many native-resolution rows should fall in + one bin. E.g. 1-min data resampled to "1hr" expects ~60 rows/bin; + at ``threshold=0.75`` a bin needs >= 45 non-null rows to survive. + + Args: + timestamps: The timestamp series (pre-resampling) + rule: The pandas offset alias passed to ``resample_dataframe``. + threshold: The required fraction of expected samples, e.g. ``0.75``. + + """ + bin_width = pd.Timedelta(pd.tseries.frequencies.to_offset(rule)) + median_tdiff = pd.to_datetime(timestamps).sort_values().diff().median() + if pd.isna(median_tdiff) or median_tdiff <= pd.Timedelta(0): + return 0 + expected_samples = bin_width / median_tdiff + return int(np.ceil(threshold * expected_samples)) + def _flag_aware_resample(df, rule, keys, agg): """Vectorized flag-aware resampling. @@ -109,6 +132,7 @@ def resample_dataframe( numeric_how: str = "mean", nonnumeric_how: str = "first", force_nonnumeric: str | list[str] | None = None, + completeness_threshold: float | None = None, ) -> pd.DataFrame: """Resample a time-indexed frame, handling mixed dtypes safely. @@ -148,6 +172,14 @@ def resample_dataframe( force_nonnumeric: Column name(s) to always aggregate with ``nonnumeric_how`` regardless of dtype, e.g. ``"fw"``, which is numeric but shouldn't be averaged. + completeness_threshold: If given (e.g. ``0.75``), bins that don't + meet the data-completeness requirement get every value column set + to NaN. The definition of "complete" depends on ``flag_aware``: + + - If ``flag_aware=True``: the bin needs this fraction of its expected + native-resolution sample count to be clean (flag==0) + - If ``flag_aware=False``: the bin needs this fraction of its expected + native-resolution sample count to be non-null Returns: A new frame with ``on`` (and any ``by`` keys) as columns. @@ -222,6 +254,72 @@ def resample_dataframe( else: out = indexed.resample(rule).agg(agg).reset_index() + if completeness_threshold is not None: + merge_on = keys + [on] + + if flag_aware: + # require >= completeness_threshold fraction of the expected + # sample count to be clean (flag == 0) + clean_indexed = indexed[indexed["flag"] == 0] + if keys: + clean_counts = ( + clean_indexed.groupby(keys).resample(rule)["flag"] + .count().rename("_clean_count").reset_index() + ) + min_count_by_group = ( + df.groupby(keys)[on] + .apply(lambda s: _min_required_count(s, rule, completeness_threshold)) + .rename("_min_count") + ) + clean_counts = clean_counts.merge(min_count_by_group, on=keys, how="left") + else: + clean_counts = ( + clean_indexed["flag"].resample(rule) + .count().rename("_clean_count").reset_index() + ) + clean_counts["_min_count"] = _min_required_count( + df[on], rule, completeness_threshold + ) + clean_counts["_clean_count"] = clean_counts["_clean_count"].fillna(0) + + merged = out[merge_on].merge(clean_counts, on=merge_on, how="left") + fail_mask = merged["_clean_count"] < merged["_min_count"] + + for c in value_cols: + if c not in out.columns: + continue + out.loc[fail_mask.to_numpy(), c] = np.nan + else: + # require >= completeness_threshold fraction of the expected + # sample count to be non-null, for each value column + if keys: + raw_counts = ( + indexed.groupby(keys).resample(rule)[value_cols].count().reset_index() + ) + min_count_by_group = ( + df.groupby(keys)[on] + .apply(lambda s: _min_required_count(s, rule, completeness_threshold)) + .rename("_min_count") + ) + raw_counts = raw_counts.merge(min_count_by_group, on=keys, how="left") + else: + raw_counts = indexed[value_cols].resample(rule).count().reset_index() + raw_counts["_min_count"] = _min_required_count( + df[on], rule, completeness_threshold + ) + + merged_counts = out[merge_on].merge(raw_counts, on=merge_on, how="left") + for c in value_cols: + if c not in out.columns: + continue + short = merged_counts[c] < merged_counts["_min_count"] + out.loc[short.to_numpy(), c] = np.nan + + logger.debug( + "Applied {:.0%} completeness threshold to {} value column(s)", + completeness_threshold, len(value_cols), + ) + if do_wind: out = _vector_wind(out, *wind) elif wind is not None: From 82a84e82436794f1944eedadc9a91f403c6a8a85 Mon Sep 17 00:00:00 2001 From: Emmie Le Roy <47359313+emmleroy@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:06:48 -0400 Subject: [PATCH 11/11] Only apply completeness_threshold to numeric cols --- quantaq_cli/toolkit/resample.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/quantaq_cli/toolkit/resample.py b/quantaq_cli/toolkit/resample.py index 3e0761d..75fddf3 100644 --- a/quantaq_cli/toolkit/resample.py +++ b/quantaq_cli/toolkit/resample.py @@ -173,13 +173,13 @@ def resample_dataframe( ``nonnumeric_how`` regardless of dtype, e.g. ``"fw"``, which is numeric but shouldn't be averaged. completeness_threshold: If given (e.g. ``0.75``), bins that don't - meet the data-completeness requirement get every value column set - to NaN. The definition of "complete" depends on ``flag_aware``: + meet the data-completeness requirement are set to NaN. + The definition of "complete" depends on ``flag_aware``: - If ``flag_aware=True``: the bin needs this fraction of its expected - native-resolution sample count to be clean (flag==0) + sample count to be clean (flag==0) - If ``flag_aware=False``: the bin needs this fraction of its expected - native-resolution sample count to be non-null + sample count to be non-null Returns: A new frame with ``on`` (and any ``by`` keys) as columns. @@ -256,6 +256,7 @@ def resample_dataframe( if completeness_threshold is not None: merge_on = keys + [on] + numeric_value_cols = [c for c in value_cols if is_numeric_dtype(df[c])] if flag_aware: # require >= completeness_threshold fraction of the expected @@ -285,16 +286,16 @@ def resample_dataframe( merged = out[merge_on].merge(clean_counts, on=merge_on, how="left") fail_mask = merged["_clean_count"] < merged["_min_count"] - for c in value_cols: + for c in numeric_value_cols: if c not in out.columns: continue out.loc[fail_mask.to_numpy(), c] = np.nan else: # require >= completeness_threshold fraction of the expected - # sample count to be non-null, for each value column + # sample count to be non-null, for each numeric value column if keys: raw_counts = ( - indexed.groupby(keys).resample(rule)[value_cols].count().reset_index() + indexed.groupby(keys).resample(rule)[numeric_value_cols].count().reset_index() ) min_count_by_group = ( df.groupby(keys)[on] @@ -303,21 +304,21 @@ def resample_dataframe( ) raw_counts = raw_counts.merge(min_count_by_group, on=keys, how="left") else: - raw_counts = indexed[value_cols].resample(rule).count().reset_index() + raw_counts = indexed[numeric_value_cols].resample(rule).count().reset_index() raw_counts["_min_count"] = _min_required_count( df[on], rule, completeness_threshold ) merged_counts = out[merge_on].merge(raw_counts, on=merge_on, how="left") - for c in value_cols: + for c in numeric_value_cols: if c not in out.columns: continue short = merged_counts[c] < merged_counts["_min_count"] out.loc[short.to_numpy(), c] = np.nan logger.debug( - "Applied {:.0%} completeness threshold to {} value column(s)", - completeness_threshold, len(value_cols), + "Applied {:.0%} completeness threshold to {} numeric value column(s)", + completeness_threshold, len(numeric_value_cols), ) if do_wind: