diff --git a/.github/workflows/release-legacy-assets.yml b/.github/workflows/release-legacy-assets.yml new file mode 100644 index 0000000..fa7a9c4 --- /dev/null +++ b/.github/workflows/release-legacy-assets.yml @@ -0,0 +1,59 @@ +name: Publish archived binary assets + +on: + workflow_dispatch: + inputs: + tag_name: + description: Release tag for the archived asset bundle + required: true + default: legacy-assets-v1.0.0 + type: string + +permissions: + contents: write + +jobs: + package-and-release: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + ASSET_SOURCE_SHA: 50dcd4dfd4aefab233a548faf400e46353291cb3 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Extract original binary files from repository history + run: | + set -euo pipefail + archive_dir="$(mktemp -d)" + git archive "$ASSET_SOURCE_SHA" \ + assets docs/images "docs/ClimateX.ai Features .pdf" \ + artifacts/reports models frontend/public/docs/images \ + | tar -xf - -C "$archive_dir" + find "$archive_dir" -type f -size 0 -delete + mkdir -p dist + tar -czf dist/climate-crew-legacy-assets.tar.gz \ + -C "$archive_dir" \ + assets docs/images "docs/ClimateX.ai Features .pdf" \ + artifacts/reports models frontend/public/docs/images + sha256sum dist/climate-crew-legacy-assets.tar.gz > dist/SHA256SUMS.txt + - name: Publish or replace the release asset + env: + GH_TOKEN: ${{ github.token }} + TAG_NAME: ${{ inputs.tag_name }} + run: | + set -euo pipefail + NOTES="dist/release-notes.md" + cat > "$NOTES" <<'NOTES' + Archived images, audit reports, and retail prototype model files retained for historical traceability. + These files are not loaded by the default Climate Crew application. The zero-byte audit PDF was excluded. + The archive was extracted from the repository snapshot recorded in .github/workflows/release-legacy-assets.yml. + See PROVENANCE.md and docs/release-assets.md. + NOTES + if gh release view "$TAG_NAME" >/dev/null 2>&1; then + gh release upload "$TAG_NAME" dist/climate-crew-legacy-assets.tar.gz dist/SHA256SUMS.txt --clobber + else + gh release create "$TAG_NAME" dist/climate-crew-legacy-assets.tar.gz dist/SHA256SUMS.txt \ + --title "Climate Crew archived assets" --notes-file "$NOTES" + fi diff --git a/.github/workflows/science-slice.yml b/.github/workflows/science-slice.yml new file mode 100644 index 0000000..508429f --- /dev/null +++ b/.github/workflows/science-slice.yml @@ -0,0 +1,33 @@ +name: Climate science slice + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + offline-science: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install minimal offline-science dependencies + run: python -m pip install -r requirements-science.txt + - name: Check generated benchmark dataset + run: | + python eval/science_quality/build_dataset.py + git diff --exit-code -- eval/science_quality/data/climate_evidence_benchmark.json + - name: Run offline tests with network blocked + run: python -m pytest -q tests/science + - name: Reproduce pinned GISTEMP output + run: python analysis/reproduce_gistemp.py --check + - name: Compile the focused Python modules + run: python -m compileall -q analysis src/science eval/science_quality tests/science diff --git a/.github/workflows/unpack-climatex.yml b/.github/workflows/unpack-climatex.yml deleted file mode 100644 index 2576ccf..0000000 --- a/.github/workflows/unpack-climatex.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Unpack ClimateX Source - -on: - push: - branches: [main] - paths: - - 'ClimateX.ai-main.zip' - - '.github/workflows/unpack-climatex.yml' - workflow_dispatch: - -permissions: - contents: write - -concurrency: - group: unpack-climatex-main - cancel-in-progress: false - -jobs: - unpack: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Validate archive - shell: bash - run: | - set -euo pipefail - test -f ClimateX.ai-main.zip - unzip -t ClimateX.ai-main.zip - - - name: Extract archive to staging directory - shell: bash - run: | - set -euo pipefail - rm -rf .unpack-climatex - mkdir .unpack-climatex - unzip -q ClimateX.ai-main.zip -d .unpack-climatex - echo 'Archive top-level contents:' - find .unpack-climatex -mindepth 1 -maxdepth 2 -type d -o -type f | head -100 - - - name: Normalize archive root into repository root - shell: bash - run: | - set -euo pipefail - shopt -s dotglob nullglob - - entries=(.unpack-climatex/*) - if [ ${#entries[@]} -eq 1 ] && [ -d "${entries[0]}" ]; then - SOURCE="${entries[0]}" - else - SOURCE='.unpack-climatex' - fi - - for item in "$SOURCE"/*; do - name="$(basename "$item")" - case "$name" in - .git|.github|ClimateX.ai-main.zip) continue ;; - esac - rm -rf -- "$name" - mv -- "$item" ./ - done - - rm -rf .unpack-climatex - - - name: Remove source ZIP after successful extraction - shell: bash - run: rm -f ClimateX.ai-main.zip - - - name: Show resulting repository tree - shell: bash - run: | - echo 'Extracted source:' - find . -maxdepth 3 -not -path './.git/*' | sort | head -500 - - - name: Commit extracted source - shell: bash - run: | - set -euo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add -A - if git diff --cached --quiet; then - echo 'No extraction changes to commit.' - exit 0 - fi - git commit -m 'Unpack ClimateX source into repository' - git push origin HEAD:main diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..67ecfd5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +MIT License + +Copyright (c) 2026 Jason Lee + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +This license applies to original project contributions. Third-party material, +if any, remains subject to its applicable license; see PROVENANCE.md. diff --git a/PROVENANCE.md b/PROVENANCE.md new file mode 100644 index 0000000..ab7d87e --- /dev/null +++ b/PROVENANCE.md @@ -0,0 +1,25 @@ +# Provenance and attribution + +## Climate Crew source lineage + +The public repository history contains commit `e2691313d9bd8dfdf3f07a3cc2cbaa0922f79b0a`, titled **“Unpack ClimateX source into repository”**, dated 2026-09-26. The prior workflow `.github/workflows/unpack-climatex.yml` extracted `ClimateX.ai-main.zip`, replaced matching repository paths, and pushed the result to `main`. That workflow is removed in this branch because it granted write access and could overwrite current files. + +The archive is not present in the current default-branch tree. The commit history does not record an upstream repository URL, original author, or source license. This document therefore records ClimateX as the codebase's predecessor but does not claim the contents came from a specific external project. The old audit agent, route identifiers, PDF footer and some comments still use ClimateX branding; those names are historical code identifiers, not an attribution claim. Historical binary assets are removed from the current source tree and can be reconstructed by the release workflow from the pre-cleanup snapshot `50dcd4dfd4aefab233a548faf400e46353291cb3`. The release itself still needs to be created from the workflow after merge. + +The former retail and climate-finance/ESG modules are moved under `examples/` and removed from the default API/frontend. They are archived prototypes, not supported app modules. `LICENSE` applies MIT terms to original project contributions by Jason Lee only; it does not establish licensing for historical code whose source and authorship are unknown or supersede any third-party license. Restore any discovered upstream notices alongside the affected files before redistribution. + +## NASA GISTEMP data + +The reproducible analysis uses NASA GISS GISTEMP v4 global land-ocean temperature anomalies: + +- Data page: +- CSV: +- Snapshot date: 2026-10-04 +- Snapshot SHA-256: `c9ce0750ca93a8241c42fe86cd5bf54d07b28b21ae650b0ae49a206907018cd9` +- Analysis: annual `J-D` column, 1951–1980 reference period, complete years 1880–2025 + +NASA periodically revises the source data. The checked-in file is a fixed reproducibility snapshot; later source revisions should be added as a new snapshot rather than silently replacing this one. + +## Test and evaluation provenance + +The CI tests use local data and mocked provider responses. The 60-case evaluation dataset is synthetic and generated by `eval/science_quality/build_dataset.py`; it is not observational climate evidence. The existing application and general evaluation modules have not been fully reproduced in CI by this workflow. diff --git a/README.md b/README.md index f709fcb..240b313 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ +# Climate Crew + +Climate Crew is a climate and environmental research prototype. The repository contains climate data connectors, specialist agents, a FastAPI backend, a React frontend, and a multi-agent deep-research pipeline. The broad application is not yet validated as an end-to-end scientific system. + +## Implemented, prototype, planned + +| Status | Capability | Evidence in this repository | +|---|---|---| +| Implemented and offline-tested | One reproducible climate result: source snapshot → trend statistics → endpoint sensitivity check → uncertainty → report and plot | `analysis/reproduce_gistemp.py`, `src/science/`, `data/science/`, `artifacts/science/` | +| Implemented as typed records | Evidence records with source, retrieval date, evidence type, independence group and derivation links; competing hypotheses; falsification test; uncertainty assessment | `src/science/models.py` | +| Prototype | Climate/environment data adapters, specialist agents, FastAPI routes and React workspace | `src/`, `main.py`, `frontend/` | +| Prototype | Deep-research orchestration with planning, search, debate and synthesis; provider credentials and external services are required | `src/agents/deep_research/` | +| Planned | Durable project-wide evidence graph, independent replication across datasets, and expert validation gates integrated into the application | Not implemented end to end |

Climate Crew research platform: specialist agents, data adapters, debate, and reviewable reports.

@@ -291,49 +304,68 @@ Climate Crew separates **producing a result** from **trusting a result**. Signif Assessment can consider source, dataset, methodological and model independence; measurement and statistical uncertainty; assumption sensitivity; unresolved contradictions; replication history; and missing evidence. -Ten articles derived from one dataset should not automatically count as ten independent confirmations. +The former retail and climate-finance/ESG experiments are preserved under [`examples/`](examples/README.md) and are no longer mounted by the default API or frontend. They are not part of the climate-science portfolio focus. ---- +## Reproduce the climate result -## 🔎 Discovery lifecycle +The checked-in input is a pinned snapshot of NASA GISTEMP v4 global land-ocean annual anomalies. The `J-D` column is measured in degrees Celsius relative to 1951–1980. The analysis excludes the incomplete 2026 row and uses complete years 1880–2025. -```text -Observation - ↓ -Finding - ↓ -Candidate Finding - ↓ -Replicated Finding - ↓ -Novel Finding - ↓ -Discovery Candidate - ↓ -Externally Validated Discovery +```bash +python -m pip install -r requirements-science.txt +python analysis/reproduce_gistemp.py +python analysis/reproduce_gistemp.py --check +python -m pytest -q tests/science ``` -A **Discovery Candidate** contains the evidence, provenance, methods, contradictions, uncertainty, replication history and novelty assessment required for external evaluation. Validation may require experts, independent datasets, field investigation, laboratory work or peer review. +The script has no network or model-key requirement. It writes a machine-readable report to [`artifacts/science/gistemp_trend_result.json`](artifacts/science/gistemp_trend_result.json) and a plot to [`artifacts/science/gistemp_global_trend.svg`](artifacts/science/gistemp_global_trend.svg). CI checks both outputs against the pinned data and runs the offline tests. ---- +**Result for this snapshot:** 146 annual observations; OLS trend **0.0834 °C/decade** (95% circular 5-year moving-block bootstrap interval **0.0676–0.0980**); Theil–Sen slope **0.0812 °C/decade**; two-sided Mann–Kendall tau-b **0.7347**, *p* = **3.44 × 10⁻³⁹**. The exploratory endpoint check excludes 2016–2025 and still finds a positive 1880–2015 trend (Sen slope 0.0711 °C/decade; Mann–Kendall *p* = 4.37 × 10⁻³³). That check does not overturn the observed trend in this series. -## 🌎 Distributed Climate Discovery Network +![NASA GISTEMP annual anomaly series and OLS trend](artifacts/science/gistemp_global_trend.svg) -Large investigations can be decomposed into bounded research tasks and distributed across agents, machines, institutions or community contributors. Returned work enters the same evidence and provenance system and remains subject to validation gates. +**Limits:** this is a descriptive result from one global dataset, not an independent replication or causal-attribution analysis. The Mann–Kendall p-value uses the normal approximation with tie correction and does not adjust its variance for serial autocorrelation. The block-bootstrap interval uses the stated 5-year residual blocks and does not cover every measurement or structural uncertainty. See the JSON report for methods and caveats. -Examples include reproducing calculations, analysing datasets, testing parameter regions, sensitivity analysis, comparing models, searching for contradictory evidence, investigating prior art and examining geographic anomalies. +Source: [NASA GISS GISTEMP v4 data downloads](https://data.giss.nasa.gov/gistemp/data_v4.html). Snapshot retrieved 2026-10-04; SHA-256 is recorded in `data/science/gistemp_manifest.json`. +The intended research-memory layer would preserve previous questions, hypotheses, rejected explanations, evidence, contradictions, datasets, methods, model outputs, replication attempts, uncertainty assessments and findings without confusing remembered information with verified evidence. ---- +## Evaluation -## 🧬 Research memory & integrity +`eval/science_quality/` contains 60 balanced, **synthetic controlled evidence scenarios** and a paired comparison harness for the Climate Crew adapter and a single-LLM adapter. It reports relationship accuracy with Wilson 95% intervals, evidence precision/recall/F1, a Brier score, and a paired bootstrap interval for the accuracy difference. -The intended research-memory layer would preserve previous questions, hypotheses, rejected explanations, evidence, contradictions, datasets, methods, model outputs, replication attempts, uncertainty assessments and findings without confusing remembered information with verified evidence. +CI uses mocked adapters to test the harness on all 60 cases without network access. Those mocked scores are plumbing checks, not model-performance evidence. To run a real provider comparison, follow [`eval/science_quality/README.md`](eval/science_quality/README.md); start with a small `--limit` because the multi-agent run makes multiple provider calls per case. No real comparative score is claimed until that run is made and its model IDs, date, commit, and cost are published. + +The older `eval/` runner remains a prototype: its system integration currently returns mock outputs. Do not treat its results as measured Climate Crew performance. + +## Tests and CI boundary + +The new workflow runs only the offline scientific slice: + +```bash +python -m pytest -q tests/science +python analysis/reproduce_gistemp.py --check +``` + +The older scripts under `test/` include provider-backed demos and integration checks. They are not all automated by this workflow and may require credentials, services, or network access. Passing the new suite validates the bounded result and benchmark harness, not the complete API, frontend, or all external integrations. + +The legacy three-query baseline can also be run with `python -m pytest test/test_eval.py -k offline_baseline`. It verifies the mock-runner plumbing only and is not a model benchmark. + +## Run the application prototype + +The complete backend and frontend have a larger pinned dependency set and external-service requirements: + +```bash +python -m venv .venv +source .venv/bin/activate # Windows: .venv\\Scripts\\activate +python -m pip install -r requirements.txt +uvicorn main:app --reload +``` -Research records can preserve source attribution, timestamps, dataset identity, transformations, model and tool involvement, assumptions, agent contributions, critiques, contradictions, replication results and human validation decisions. +The API docs are normally available at `http://localhost:8000/docs`. For the frontend, run `npm install` and `npm run dev` from `frontend/`. Set provider credentials through environment variables when using integrations that require them. -Synthetic, fallback, demonstration and simulated information must remain distinguishable from observational evidence throughout the pipeline. +## Provenance and license +The repository history records a migration from `ClimateX.ai-main.zip` on 2026-09-26. The archive is not in the current tree, and the history does not identify an upstream URL or license. [`PROVENANCE.md`](PROVENANCE.md) records this boundary and the NASA data source. Non-core retail and finance experiments are preserved under `examples/`; the historical binary asset bundle can be published with the workflow described in [`docs/release-assets.md`](docs/release-assets.md). --- ## 🌍 Climate Crew -**Investigate independently. Challenge aggressively. Preserve the evidence. Discover carefully.** +The repository includes an MIT license for original project contributions. Third-party material remains subject to its applicable terms; no unverified upstream attribution is invented. diff --git a/analysis/reproduce_gistemp.py b/analysis/reproduce_gistemp.py new file mode 100644 index 0000000..f379b34 --- /dev/null +++ b/analysis/reproduce_gistemp.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Reproduce a bounded, non-attribution trend analysis from a pinned NASA CSV.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from src.science.models import ( + EvidenceRecord, + FalsificationTest, + Hypothesis, + ScientificReport, + UncertaintyAssessment, +) +from src.science.statistics import AnnualSeries, load_gistemp_annual, ols_slope, trend_summary + + +DATA = ROOT / "data/science/GLB.Ts+dSST.csv" +MANIFEST = ROOT / "data/science/gistemp_manifest.json" +OUTPUT = ROOT / "artifacts/science" +SOURCE_PAGE = "https://data.giss.nasa.gov/gistemp/data_v4.html" +SOURCE_CSV = "https://data.giss.nasa.gov/gistemp/tabledata_v4/GLB.Ts+dSST.csv" + + +def make_report(series: AnnualSeries, summary: dict[str, object], sha256: str) -> ScientificReport: + ci = summary["ols_slope_95_ci_c_per_decade"] + primary = summary["mann_kendall"] + endpoint = summary["endpoint_falsification"] + rise_supported = primary["tau_b"] > 0 and primary["p_value"] < 0.05 + endpoint_survives = endpoint["outcome"] == "not_falsified" + return ScientificReport( + question=f"Did annual global GISTEMP land-ocean anomalies increase over {series.years[0]}–{series.years[-1]}?", + hypotheses=[ + Hypothesis( + hypothesis_id="H1", + statement="The annual global land-ocean temperature anomaly series has a positive monotonic trend over the stated period.", + status="supported" if rise_supported and endpoint_survives else "unresolved", + ), + Hypothesis( + hypothesis_id="H2", + statement="The apparent full-period rise is solely a recent-endpoint effect and disappears when the final ten years are omitted.", + status="challenged" if endpoint_survives else "unresolved", + ), + ], + evidence=[ + EvidenceRecord( + evidence_id="E1", + statement="NASA GISTEMP v4 global land-ocean annual mean temperature anomalies (J-D column), relative to the 1951–1980 baseline.", + source_title="NASA GISS Surface Temperature Analysis v4 data downloads", + source_url=SOURCE_PAGE, + evidence_kind="dataset", + retrieved_at="2026-10-04", + independence_group="NASA-GISTEMP-v4", + limitations=["A global aggregate does not represent local or regional trends.", "An anomaly series alone does not identify causal attribution."], + ), + EvidenceRecord( + evidence_id="E2", + statement=f"SHA-256 {sha256}; {summary['n']} complete annual values were analysed for {summary['period'][0]}–{summary['period'][1]}.", + source_title="Pinned GISTEMP CSV snapshot and deterministic analysis", + source_url=SOURCE_CSV, + evidence_kind="derived", + retrieved_at="2026-10-04", + independence_group="NASA-GISTEMP-v4-derived", + derived_from=["E1"], + limitations=["A reproducibility snapshot can differ from later NASA revisions."], + ), + EvidenceRecord( + evidence_id="E3", + statement=f"Mann–Kendall tau-b={primary['tau_b']:.4f}, p={primary['p_value']:.3g}; Sen slope={summary['sen_slope_c_per_decade']:.4f} °C/decade; the block-bootstrap OLS 95% interval is {ci[0]:.4f} to {ci[1]:.4f} °C/decade.", + source_title="Climate Crew reproducible calculation", + source_url=SOURCE_CSV, + evidence_kind="derived", + retrieved_at="2026-10-04", + independence_group="Climate-Crew-analysis-v1", + derived_from=["E2"], + limitations=["The bootstrap interval quantifies sampling sensitivity under the stated residual block scheme; it does not include all measurement or structural uncertainty."], + ), + ], + falsification_tests=[ + FalsificationTest( + test_id="F1", + hypothesis_id="H1", + question=endpoint["question"], + method="Exploratory endpoint sensitivity: re-run Mann–Kendall and Sen-slope estimates through the year before the final ten observations.", + outcome="not_falsified" if endpoint_survives else "inconclusive", + result_summary=f"For {endpoint['period'][0]}–{endpoint['period'][1]}, Sen slope={endpoint['sen_slope_c_per_decade']:.4f} °C/decade and Mann–Kendall p={endpoint['mann_kendall']['p_value']:.3g}.", + evidence_ids=["E2", "E3"], + ) + ], + uncertainty=UncertaintyAssessment( + estimate=summary["ols_slope_c_per_decade"], + unit="°C/decade", + interval_level=0.95, + lower_bound=ci[0], + upper_bound=ci[1], + method=summary["methods"]["interval"], + caveats=["This interval reflects the specified block-bootstrap procedure, not all sources of uncertainty.", "The analysis is descriptive and does not attribute causes."], + ), + conclusion=( + "The pinned global anomaly series shows a positive monotonic trend over the analysis period; " + "the exploratory endpoint sensitivity check does not overturn that pattern. This is a descriptive " + "time-series result, not a causal attribution analysis." + if rise_supported and endpoint_survives + else "The specified analysis does not establish a robust positive monotonic trend; treat the result as unresolved." + ), + limitations=[ + "One global annual series is not an independent replication across datasets.", + "A monotonic trend test does not explain causes or predict local impacts.", + "The pinned snapshot and the analysis choices should be revisited when reproducing later NASA revisions.", + ], + ) + + +def render_svg(series: AnnualSeries) -> str: + width, height = 1000, 560 + left, right, top, bottom = 86, 34, 70, 72 + values = series.anomalies_c + years = series.years + y_min = min(-0.6, min(values) - 0.08) + y_max = max(1.2, max(values) + 0.08) + plot_w, plot_h = width - left - right, height - top - bottom + + def px(year: int) -> float: + return left + (year - years[0]) / (years[-1] - years[0]) * plot_w + + def py(value: float) -> float: + return top + (y_max - value) / (y_max - y_min) * plot_h + + slope = ols_slope(years, values) + x_bar = sum(years) / len(years) + y_bar = sum(values) / len(values) + intercept = y_bar - slope * x_bar + points = " ".join(f"{px(year):.2f},{py(value):.2f}" for year, value in zip(years, values)) + trend_points = f"{px(years[0]):.2f},{py(intercept + slope * years[0]):.2f} {px(years[-1]):.2f},{py(intercept + slope * years[-1]):.2f}" + parts = [ + f'', + 'NASA GISTEMP global annual land-ocean temperature anomalies', + 'Annual anomalies for 1880 through 2025 and an ordinary least-squares trend line. Descriptive result; no causal attribution.', + '', + '', + f'Global land-ocean temperature anomalies (°C)', + ] + for step in range(7): + value = round(y_min + (y_max - y_min) * step / 6, 1) + y = py(value) + parts.append(f'') + parts.append(f'{value:.1f}') + for year in range((years[0] // 20) * 20, years[-1] + 1, 20): + if years[0] <= year <= years[-1]: + x = px(year) + parts.append(f'') + parts.append(f'{year}') + parts.extend([ + f'', + f'', + f'', + f'', + f'NASA GISTEMP v4 • 1951–1980 baseline • Source CSV snapshot checked in', + f'Annual anomaly', + f'OLS trend', + '', + ]) + return "\n".join(parts) + "\n" + + +def build_outputs(data_path: Path = DATA) -> tuple[bytes, bytes]: + series = load_gistemp_annual(data_path) + summary = trend_summary(series) + sha256 = hashlib.sha256(data_path.read_bytes()).hexdigest() + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + if data_path.resolve() == DATA.resolve() and sha256 != manifest["snapshot_sha256"]: + raise ValueError("Pinned GISTEMP snapshot hash does not match data/science/gistemp_manifest.json") + report = make_report(series, summary, sha256) + result = { + "dataset": manifest["dataset"], + "source_page": manifest["source_page"], + "source_csv": manifest["source_csv"], + "snapshot_sha256": sha256, + "baseline": manifest["baseline_period"], + "retrieved_at": manifest["retrieved_at"], + "summary": summary, + "report": report.model_dump(mode="json"), + } + json_bytes = (json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8") + svg_bytes = render_svg(series).encode("utf-8") + return json_bytes, svg_bytes + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data", type=Path, default=DATA, help="Pinned NASA GISTEMP CSV") + parser.add_argument("--output-dir", type=Path, default=OUTPUT) + parser.add_argument("--check", action="store_true", help="Fail if checked-in results differ") + args = parser.parse_args() + json_bytes, svg_bytes = build_outputs(args.data) + json_path = args.output_dir / "gistemp_trend_result.json" + svg_path = args.output_dir / "gistemp_global_trend.svg" + if args.check: + stale = [str(path) for path, data in ((json_path, json_bytes), (svg_path, svg_bytes)) if not path.exists() or path.read_bytes() != data] + if stale: + print("Reproducible output is stale: " + ", ".join(stale)) + return 1 + print(f"Verified {json_path.relative_to(ROOT)} and {svg_path.relative_to(ROOT)}") + return 0 + args.output_dir.mkdir(parents=True, exist_ok=True) + json_path.write_bytes(json_bytes) + svg_path.write_bytes(svg_bytes) + payload = json.loads(json_bytes) + summary = payload["summary"] + print(f"Years: {summary['period'][0]}–{summary['period'][1]} (n={summary['n']})") + print(f"OLS slope: {summary['ols_slope_c_per_decade']:.4f} °C/decade, 95% block-bootstrap CI {summary['ols_slope_95_ci_c_per_decade'][0]:.4f} to {summary['ols_slope_95_ci_c_per_decade'][1]:.4f}") + print(f"Mann–Kendall: tau-b={summary['mann_kendall']['tau_b']:.4f}, p={summary['mann_kendall']['p_value']:.3g}") + print(f"Endpoint check: {summary['endpoint_falsification']['outcome']} through {summary['endpoint_falsification']['period'][1]}") + print(f"Wrote {json_path} and {svg_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/artifacts/reports/ClimateX_Audit_1768716469.pdf b/artifacts/reports/ClimateX_Audit_1768716469.pdf deleted file mode 100644 index bb4e2bc..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1768716469.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1768716811.pdf b/artifacts/reports/ClimateX_Audit_1768716811.pdf deleted file mode 100644 index e69de29..0000000 diff --git a/artifacts/reports/ClimateX_Audit_1768717082.pdf b/artifacts/reports/ClimateX_Audit_1768717082.pdf deleted file mode 100644 index d1fa7ad..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1768717082.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1768765350.pdf b/artifacts/reports/ClimateX_Audit_1768765350.pdf deleted file mode 100644 index 37e5b13..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1768765350.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1768765579.pdf b/artifacts/reports/ClimateX_Audit_1768765579.pdf deleted file mode 100644 index fee1056..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1768765579.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1769094145.pdf b/artifacts/reports/ClimateX_Audit_1769094145.pdf deleted file mode 100644 index e16338b..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1769094145.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1769096920.pdf b/artifacts/reports/ClimateX_Audit_1769096920.pdf deleted file mode 100644 index 35e0245..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1769096920.pdf and /dev/null differ diff --git a/artifacts/reports/ClimateX_Audit_1770702527.pdf b/artifacts/reports/ClimateX_Audit_1770702527.pdf deleted file mode 100644 index 27740cc..0000000 Binary files a/artifacts/reports/ClimateX_Audit_1770702527.pdf and /dev/null differ diff --git a/artifacts/science/gistemp_global_trend.svg b/artifacts/science/gistemp_global_trend.svg new file mode 100644 index 0000000..6208d7b --- /dev/null +++ b/artifacts/science/gistemp_global_trend.svg @@ -0,0 +1,44 @@ + +NASA GISTEMP global annual land-ocean temperature anomalies +Annual anomalies for 1880 through 2025 and an ordinary least-squares trend line. Descriptive result; no causal attribution. + + +Global land-ocean temperature anomalies (°C) + +-0.6 + +-0.3 + +0.1 + +0.4 + +0.7 + +1.0 + +1.4 + +1880 + +1900 + +1920 + +1940 + +1960 + +1980 + +2000 + +2020 + + + + +NASA GISTEMP v4 • 1951–1980 baseline • Source CSV snapshot checked in +Annual anomaly +OLS trend + diff --git a/artifacts/science/gistemp_trend_result.json b/artifacts/science/gistemp_trend_result.json new file mode 100644 index 0000000..e80bbeb --- /dev/null +++ b/artifacts/science/gistemp_trend_result.json @@ -0,0 +1,157 @@ +{ + "baseline": "1951-1980", + "dataset": "NASA GISTEMP v4 global Land-Ocean Temperature Index", + "report": { + "conclusion": "The pinned global anomaly series shows a positive monotonic trend over the analysis period; the exploratory endpoint sensitivity check does not overturn that pattern. This is a descriptive time-series result, not a causal attribution analysis.", + "evidence": [ + { + "derived_from": [], + "evidence_id": "E1", + "evidence_kind": "dataset", + "independence_group": "NASA-GISTEMP-v4", + "limitations": [ + "A global aggregate does not represent local or regional trends.", + "An anomaly series alone does not identify causal attribution." + ], + "retrieved_at": "2026-10-04", + "source_title": "NASA GISS Surface Temperature Analysis v4 data downloads", + "source_url": "https://data.giss.nasa.gov/gistemp/data_v4.html", + "statement": "NASA GISTEMP v4 global land-ocean annual mean temperature anomalies (J-D column), relative to the 1951–1980 baseline." + }, + { + "derived_from": [ + "E1" + ], + "evidence_id": "E2", + "evidence_kind": "derived", + "independence_group": "NASA-GISTEMP-v4-derived", + "limitations": [ + "A reproducibility snapshot can differ from later NASA revisions." + ], + "retrieved_at": "2026-10-04", + "source_title": "Pinned GISTEMP CSV snapshot and deterministic analysis", + "source_url": "https://data.giss.nasa.gov/gistemp/tabledata_v4/GLB.Ts+dSST.csv", + "statement": "SHA-256 c9ce0750ca93a8241c42fe86cd5bf54d07b28b21ae650b0ae49a206907018cd9; 146 complete annual values were analysed for 1880–2025." + }, + { + "derived_from": [ + "E2" + ], + "evidence_id": "E3", + "evidence_kind": "derived", + "independence_group": "Climate-Crew-analysis-v1", + "limitations": [ + "The bootstrap interval quantifies sampling sensitivity under the stated residual block scheme; it does not include all measurement or structural uncertainty." + ], + "retrieved_at": "2026-10-04", + "source_title": "Climate Crew reproducible calculation", + "source_url": "https://data.giss.nasa.gov/gistemp/tabledata_v4/GLB.Ts+dSST.csv", + "statement": "Mann–Kendall tau-b=0.7347, p=3.44e-39; Sen slope=0.0812 °C/decade; the block-bootstrap OLS 95% interval is 0.0676 to 0.0980 °C/decade." + } + ], + "falsification_tests": [ + { + "evidence_ids": [ + "E2", + "E3" + ], + "hypothesis_id": "H1", + "method": "Exploratory endpoint sensitivity: re-run Mann–Kendall and Sen-slope estimates through the year before the final ten observations.", + "outcome": "not_falsified", + "predeclared": false, + "question": "Does the positive trend persist after excluding 2016-2025?", + "result_summary": "For 1880–2015, Sen slope=0.0711 °C/decade and Mann–Kendall p=4.37e-33.", + "test_id": "F1" + } + ], + "hypotheses": [ + { + "hypothesis_id": "H1", + "statement": "The annual global land-ocean temperature anomaly series has a positive monotonic trend over the stated period.", + "status": "supported" + }, + { + "hypothesis_id": "H2", + "statement": "The apparent full-period rise is solely a recent-endpoint effect and disappears when the final ten years are omitted.", + "status": "challenged" + } + ], + "limitations": [ + "One global annual series is not an independent replication across datasets.", + "A monotonic trend test does not explain causes or predict local impacts.", + "The pinned snapshot and the analysis choices should be revisited when reproducing later NASA revisions." + ], + "question": "Did annual global GISTEMP land-ocean anomalies increase over 1880–2025?", + "uncertainty": { + "caveats": [ + "This interval reflects the specified block-bootstrap procedure, not all sources of uncertainty.", + "The analysis is descriptive and does not attribute causes." + ], + "estimate": 0.08341029373487704, + "interval_level": 0.95, + "lower_bound": 0.06764098701131077, + "method": "95% circular moving-block bootstrap of detrended OLS residuals; 5-year blocks; seed 1042026; 2000 replicates", + "unit": "°C/decade", + "upper_bound": 0.0979996955747635 + } + }, + "retrieved_at": "2026-10-04", + "snapshot_sha256": "c9ce0750ca93a8241c42fe86cd5bf54d07b28b21ae650b0ae49a206907018cd9", + "source_csv": "https://data.giss.nasa.gov/gistemp/tabledata_v4/GLB.Ts%2BdSST.csv", + "source_page": "https://data.giss.nasa.gov/gistemp/data_v4.html", + "summary": { + "endpoint_falsification": { + "mann_kendall": { + "n": 136, + "p_value": 4.370251111832733e-33, + "s": 6369, + "tau_b": 0.6974856087004437, + "z": 11.982852127729286 + }, + "outcome": "not_falsified", + "period": [ + 1880, + 2015 + ], + "question": "Does the positive trend persist after excluding 2016-2025?", + "sen_slope_c_per_decade": 0.07111111111111111 + }, + "mann_kendall": { + "n": 146, + "p_value": 3.438382410479655e-39, + "s": 7740, + "tau_b": 0.7346671189353146, + "z": 13.096710851083717 + }, + "methods": { + "interval": "95% circular moving-block bootstrap of detrended OLS residuals; 5-year blocks; seed 1042026; 2000 replicates", + "robust_slope": "Theil-Sen median of all pairwise slopes", + "trend_test": "two-sided Mann-Kendall normal approximation with tie correction" + }, + "n": 146, + "ols_slope_95_ci_c_per_decade": [ + 0.06764098701131077, + 0.0979996955747635 + ], + "ols_slope_c_per_decade": 0.08341029373487704, + "period": [ + 1880, + 2025 + ], + "recent_window_check": { + "mann_kendall": { + "n": 56, + "p_value": 9.673426356180585e-20, + "s": 1287, + "tau_b": 0.839814298643509, + "z": 9.092559759612248 + }, + "period": [ + 1970, + 2025 + ], + "sen_slope_c_per_decade": 0.2 + }, + "sen_slope_c_per_decade": 0.08115942028985508 + } +} diff --git a/assets/Adversarial_Event_Generation.png b/assets/Adversarial_Event_Generation.png deleted file mode 100644 index 1551292..0000000 Binary files a/assets/Adversarial_Event_Generation.png and /dev/null differ diff --git a/assets/BaseAgent.png b/assets/BaseAgent.png deleted file mode 100644 index 135ab31..0000000 Binary files a/assets/BaseAgent.png and /dev/null differ diff --git a/assets/ClimateX_System_Architecture.png b/assets/ClimateX_System_Architecture.png deleted file mode 100644 index 9485a85..0000000 Binary files a/assets/ClimateX_System_Architecture.png and /dev/null differ diff --git a/assets/ClimateX_audit_agent.png b/assets/ClimateX_audit_agent.png deleted file mode 100644 index 4c4e0ef..0000000 Binary files a/assets/ClimateX_audit_agent.png and /dev/null differ diff --git a/assets/Climate_Disaster_Management.png b/assets/Climate_Disaster_Management.png deleted file mode 100644 index 227b1c8..0000000 Binary files a/assets/Climate_Disaster_Management.png and /dev/null differ diff --git a/assets/News_Intelligence.png b/assets/News_Intelligence.png deleted file mode 100644 index d1f1d37..0000000 Binary files a/assets/News_Intelligence.png and /dev/null differ diff --git a/assets/Retailer_agent.png b/assets/Retailer_agent.png deleted file mode 100644 index d19be81..0000000 Binary files a/assets/Retailer_agent.png and /dev/null differ diff --git a/assets/STRIDE.png b/assets/STRIDE.png deleted file mode 100644 index 7c82024..0000000 Binary files a/assets/STRIDE.png and /dev/null differ diff --git a/assets/climate_finance_refined.png b/assets/climate_finance_refined.png deleted file mode 100644 index 9ad2432..0000000 Binary files a/assets/climate_finance_refined.png and /dev/null differ diff --git a/assets/logo.png b/assets/logo.png deleted file mode 100644 index ac62597..0000000 Binary files a/assets/logo.png and /dev/null differ diff --git a/data/science/GLB.Ts+dSST.csv b/data/science/GLB.Ts+dSST.csv new file mode 100644 index 0000000..1bd2ca8 --- /dev/null +++ b/data/science/GLB.Ts+dSST.csv @@ -0,0 +1,149 @@ +Land-Ocean: Global Means +Year,Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec,J-D,D-N,DJF,MAM,JJA,SON +1880,-.19,-.26,-.10,-.17,-.11,-.22,-.19,-.11,-.15,-.24,-.23,-.18,-.18,***,***,-.13,-.17,-.21 +1881,-.20,-.15,.03,.05,.05,-.19,.00,-.04,-.16,-.22,-.19,-.08,-.09,-.10,-.18,.04,-.08,-.19 +1882,.16,.13,.04,-.16,-.14,-.23,-.17,-.08,-.15,-.24,-.17,-.37,-.12,-.09,.07,-.09,-.16,-.19 +1883,-.30,-.37,-.13,-.19,-.18,-.07,-.08,-.14,-.22,-.11,-.25,-.12,-.18,-.20,-.34,-.16,-.10,-.19 +1884,-.13,-.08,-.36,-.40,-.34,-.35,-.31,-.28,-.27,-.25,-.34,-.31,-.29,-.27,-.11,-.37,-.31,-.29 +1885,-.59,-.34,-.27,-.42,-.46,-.44,-.34,-.32,-.29,-.24,-.25,-.11,-.34,-.36,-.41,-.38,-.37,-.26 +1886,-.45,-.51,-.44,-.29,-.25,-.35,-.18,-.31,-.24,-.28,-.28,-.26,-.32,-.31,-.36,-.32,-.28,-.27 +1887,-.72,-.57,-.36,-.35,-.31,-.25,-.26,-.36,-.26,-.36,-.26,-.34,-.37,-.36,-.52,-.34,-.29,-.29 +1888,-.34,-.37,-.42,-.21,-.23,-.18,-.11,-.16,-.12,.01,.02,-.05,-.18,-.20,-.35,-.28,-.15,-.03 +1889,-.10,.16,.05,.08,-.02,-.11,-.08,-.21,-.24,-.26,-.34,-.29,-.11,-.09,.00,.04,-.13,-.28 +1890,-.43,-.46,-.41,-.30,-.40,-.25,-.29,-.40,-.37,-.26,-.45,-.32,-.36,-.36,-.39,-.37,-.31,-.36 +1891,-.34,-.47,-.18,-.28,-.17,-.21,-.18,-.18,-.16,-.22,-.32,-.04,-.23,-.25,-.38,-.21,-.19,-.23 +1892,-.30,-.12,-.40,-.34,-.23,-.22,-.32,-.27,-.17,-.14,-.43,-.40,-.28,-.25,-.15,-.33,-.27,-.25 +1893,-.81,-.57,-.23,-.28,-.34,-.26,-.14,-.25,-.22,-.18,-.19,-.32,-.32,-.32,-.59,-.28,-.22,-.20 +1894,-.53,-.30,-.24,-.45,-.32,-.41,-.24,-.24,-.28,-.23,-.26,-.21,-.31,-.32,-.38,-.34,-.30,-.26 +1895,-.41,-.43,-.33,-.22,-.27,-.21,-.17,-.17,-.12,-.11,-.17,-.15,-.23,-.24,-.35,-.28,-.18,-.13 +1896,-.23,-.13,-.27,-.31,-.16,-.12,-.03,-.06,-.08,.06,-.05,-.05,-.12,-.13,-.17,-.25,-.07,-.02 +1897,-.16,-.17,-.14,-.03,-.02,-.12,-.03,-.10,-.09,-.13,-.19,-.21,-.12,-.10,-.13,-.07,-.09,-.14 +1898,-.03,-.31,-.52,-.31,-.31,-.20,-.23,-.27,-.22,-.35,-.39,-.24,-.28,-.28,-.18,-.38,-.23,-.32 +1899,-.17,-.39,-.37,-.21,-.23,-.32,-.16,-.09,-.05,-.04,.12,-.27,-.18,-.18,-.27,-.27,-.19,.01 +1900,-.37,-.06,.00,-.10,-.09,-.10,-.13,-.09,-.06,.09,-.07,-.07,-.09,-.10,-.23,-.06,-.11,-.01 +1901,-.22,-.11,.06,-.03,-.15,-.12,-.15,-.20,-.23,-.29,-.17,-.28,-.16,-.14,-.14,-.04,-.16,-.23 +1902,-.17,-.09,-.28,-.28,-.33,-.32,-.28,-.30,-.29,-.30,-.37,-.43,-.29,-.27,-.18,-.30,-.30,-.32 +1903,-.25,-.07,-.24,-.42,-.40,-.43,-.35,-.45,-.49,-.47,-.42,-.52,-.38,-.37,-.25,-.35,-.41,-.46 +1904,-.64,-.59,-.52,-.51,-.52,-.49,-.52,-.50,-.56,-.40,-.19,-.34,-.48,-.50,-.58,-.52,-.50,-.38 +1905,-.36,-.60,-.22,-.34,-.30,-.29,-.27,-.20,-.19,-.25,-.08,-.14,-.27,-.29,-.43,-.29,-.25,-.17 +1906,-.29,-.30,-.19,-.06,-.26,-.20,-.23,-.20,-.28,-.20,-.38,-.15,-.23,-.23,-.24,-.17,-.21,-.29 +1907,-.43,-.51,-.27,-.37,-.46,-.42,-.35,-.35,-.34,-.23,-.47,-.47,-.39,-.36,-.37,-.37,-.37,-.35 +1908,-.45,-.33,-.56,-.45,-.38,-.39,-.36,-.47,-.36,-.44,-.52,-.50,-.43,-.43,-.42,-.46,-.41,-.44 +1909,-.74,-.47,-.55,-.60,-.55,-.52,-.46,-.34,-.39,-.40,-.33,-.57,-.49,-.49,-.57,-.57,-.44,-.37 +1910,-.43,-.42,-.51,-.43,-.35,-.40,-.35,-.37,-.39,-.41,-.58,-.68,-.44,-.44,-.47,-.43,-.37,-.46 +1911,-.63,-.59,-.61,-.55,-.52,-.51,-.43,-.44,-.41,-.27,-.22,-.23,-.45,-.49,-.63,-.56,-.46,-.30 +1912,-.28,-.17,-.40,-.19,-.22,-.25,-.43,-.55,-.58,-.59,-.40,-.44,-.37,-.36,-.22,-.27,-.41,-.52 +1913,-.42,-.47,-.44,-.41,-.45,-.46,-.37,-.34,-.35,-.33,-.21,-.04,-.36,-.39,-.44,-.43,-.39,-.30 +1914,.02,-.11,-.26,-.31,-.22,-.26,-.25,-.17,-.18,-.04,-.17,-.05,-.17,-.17,-.04,-.26,-.23,-.13 +1915,-.22,-.04,-.11,.06,-.06,-.23,-.13,-.23,-.21,-.25,-.14,-.22,-.15,-.13,-.10,-.04,-.20,-.20 +1916,-.14,-.15,-.29,-.31,-.36,-.50,-.37,-.28,-.36,-.33,-.47,-.82,-.37,-.32,-.17,-.32,-.39,-.39 +1917,-.57,-.63,-.63,-.55,-.56,-.44,-.26,-.23,-.23,-.45,-.34,-.68,-.46,-.48,-.68,-.58,-.31,-.34 +1918,-.48,-.35,-.26,-.45,-.43,-.37,-.32,-.32,-.17,-.06,-.12,-.30,-.30,-.34,-.51,-.38,-.34,-.12 +1919,-.20,-.24,-.21,-.13,-.28,-.36,-.29,-.33,-.25,-.20,-.42,-.43,-.28,-.27,-.25,-.21,-.33,-.29 +1920,-.26,-.27,-.12,-.25,-.27,-.36,-.31,-.26,-.22,-.27,-.28,-.47,-.28,-.27,-.32,-.21,-.31,-.25 +1921,-.05,-.18,-.23,-.30,-.30,-.27,-.15,-.27,-.19,-.03,-.15,-.18,-.19,-.22,-.23,-.28,-.23,-.12 +1922,-.34,-.45,-.15,-.23,-.33,-.31,-.27,-.33,-.35,-.31,-.14,-.20,-.29,-.28,-.32,-.24,-.30,-.27 +1923,-.27,-.39,-.34,-.41,-.34,-.29,-.31,-.34,-.31,-.13,-.02,-.04,-.27,-.28,-.29,-.36,-.32,-.16 +1924,-.22,-.23,-.09,-.30,-.18,-.26,-.29,-.36,-.32,-.35,-.20,-.43,-.27,-.24,-.16,-.19,-.30,-.29 +1925,-.38,-.39,-.27,-.26,-.29,-.33,-.27,-.21,-.20,-.17,.05,.06,-.22,-.26,-.40,-.27,-.27,-.11 +1926,.21,.02,.11,-.13,-.23,-.26,-.27,-.13,-.14,-.11,-.07,-.30,-.11,-.08,.10,-.08,-.22,-.11 +1927,-.27,-.18,-.39,-.31,-.26,-.28,-.19,-.24,-.12,-.01,-.06,-.33,-.22,-.22,-.25,-.32,-.24,-.06 +1928,-.02,-.09,-.25,-.29,-.30,-.39,-.20,-.22,-.21,-.19,-.10,-.16,-.20,-.22,-.15,-.28,-.27,-.17 +1929,-.45,-.58,-.33,-.42,-.38,-.43,-.37,-.33,-.25,-.14,-.12,-.55,-.36,-.33,-.40,-.37,-.38,-.17 +1930,-.31,-.27,-.11,-.25,-.24,-.22,-.21,-.15,-.15,-.12,.17,-.06,-.16,-.20,-.37,-.20,-.19,-.03 +1931,-.11,-.20,-.11,-.23,-.20,-.08,-.04,-.04,-.07,.06,-.06,-.05,-.09,-.09,-.12,-.18,-.05,-.02 +1932,.15,-.17,-.18,-.06,-.18,-.29,-.25,-.22,-.11,-.09,-.27,-.27,-.16,-.14,-.03,-.14,-.25,-.16 +1933,-.24,-.30,-.29,-.24,-.29,-.34,-.22,-.24,-.29,-.25,-.31,-.45,-.29,-.27,-.27,-.28,-.27,-.28 +1934,-.21,-.03,-.29,-.31,-.10,-.15,-.11,-.12,-.15,-.07,.03,-.03,-.13,-.16,-.23,-.23,-.13,-.06 +1935,-.34,.14,-.14,-.38,-.29,-.27,-.22,-.22,-.21,-.06,-.26,-.17,-.20,-.19,-.08,-.27,-.24,-.18 +1936,-.28,-.38,-.21,-.21,-.16,-.22,-.10,-.14,-.09,-.03,.01,-.01,-.15,-.17,-.28,-.19,-.15,-.04 +1937,-.08,.03,-.21,-.16,-.06,-.05,-.04,.01,.09,.08,.07,-.08,-.03,-.03,-.02,-.14,-.03,.08 +1938,.08,.03,.10,.06,-.10,-.17,-.10,-.06,.01,.14,.07,-.13,-.01,.00,.01,.02,-.11,.07 +1939,-.06,-.07,-.18,-.10,-.05,-.07,-.06,-.07,-.08,-.04,.07,.42,-.02,-.07,-.08,-.11,-.07,-.02 +1940,-.01,.08,.09,.17,.11,.11,.12,.06,.15,.11,.16,.31,.12,.13,.16,.12,.10,.14 +1941,.17,.30,.10,.16,.16,.12,.21,.14,.02,.35,.22,.21,.18,.19,.26,.14,.16,.19 +1942,.29,.02,.05,.09,.11,.04,.00,-.04,-.03,.01,.09,.11,.06,.07,.17,.08,.00,.02 +1943,-.01,.17,-.04,.11,.06,-.05,.08,.00,.05,.23,.20,.23,.09,.08,.09,.04,.01,.16 +1944,.36,.24,.26,.19,.18,.15,.17,.18,.28,.26,.11,.04,.20,.22,.28,.21,.17,.22 +1945,.10,.01,.06,.20,.06,.01,.04,.26,.21,.19,.07,-.07,.09,.10,.05,.11,.10,.15 +1946,.15,.03,.01,.06,-.07,-.21,-.12,-.20,-.07,-.08,-.06,-.31,-.07,-.05,.04,.00,-.18,-.07 +1947,-.07,-.08,.06,.06,-.02,-.02,-.04,-.07,-.12,.07,.02,-.13,-.03,-.04,-.15,.03,-.04,-.01 +1948,.06,-.15,-.24,-.12,-.01,-.05,-.11,-.12,-.14,-.05,-.13,-.24,-.11,-.10,-.07,-.12,-.09,-.11 +1949,.06,-.14,-.02,-.11,-.10,-.27,-.13,-.13,-.14,-.07,-.10,-.18,-.11,-.12,-.11,-.08,-.18,-.10 +1950,-.26,-.27,-.08,-.21,-.11,-.05,-.09,-.16,-.11,-.20,-.34,-.22,-.18,-.17,-.24,-.13,-.10,-.22 +1951,-.34,-.41,-.20,-.14,.00,-.07,-.01,.06,.05,.08,-.01,.15,-.07,-.10,-.33,-.12,-.01,.04 +1952,.11,.11,-.08,.03,-.03,-.03,.04,.05,.07,.00,-.13,-.02,.01,.03,.13,-.03,.02,-.02 +1953,.07,.15,.11,.19,.11,.12,.01,.05,.05,.08,-.03,.05,.08,.08,.07,.14,.06,.03 +1954,-.24,-.10,-.15,-.14,-.20,-.18,-.19,-.17,-.10,-.02,.08,-.18,-.13,-.11,-.10,-.16,-.18,-.01 +1955,.13,-.16,-.32,-.22,-.20,-.13,-.11,.02,-.11,-.05,-.25,-.28,-.14,-.13,-.07,-.25,-.07,-.14 +1956,-.13,-.24,-.21,-.28,-.29,-.15,-.09,-.25,-.19,-.23,-.15,-.06,-.19,-.21,-.22,-.26,-.17,-.19 +1957,-.09,-.03,-.05,.00,.09,.15,.02,.15,.09,.01,.08,.15,.05,.03,-.06,.01,.11,.06 +1958,.39,.22,.08,.01,.06,-.09,.05,-.05,-.03,.04,.02,.01,.06,.07,.25,.05,-.03,.01 +1959,.08,.07,.18,.16,.04,.03,.03,-.01,-.06,-.07,-.08,.00,.03,.03,.05,.12,.02,-.07 +1960,.00,.13,-.35,-.15,-.08,-.04,-.04,.02,.07,.06,-.11,.19,-.02,-.04,.04,-.19,-.02,.01 +1961,.07,.19,.09,.13,.12,.11,.01,.01,.08,.00,.03,-.16,.06,.09,.15,.12,.04,.04 +1962,.05,.15,.10,.05,-.06,.03,.02,-.01,.00,.01,.06,-.03,.03,.02,.01,.03,.01,.02 +1963,-.03,.18,-.14,-.07,-.06,.05,.06,.22,.17,.15,.15,-.03,.05,.05,.04,-.09,.11,.16 +1964,-.09,-.10,-.21,-.32,-.25,-.04,-.04,-.22,-.30,-.31,-.21,-.30,-.20,-.18,-.07,-.26,-.10,-.27 +1965,-.08,-.17,-.13,-.19,-.12,-.08,-.13,-.04,-.15,-.05,-.06,-.08,-.11,-.13,-.18,-.15,-.09,-.09 +1966,-.19,-.04,.03,-.13,-.12,.01,.08,-.09,-.03,-.17,-.01,-.03,-.06,-.06,-.10,-.07,.00,-.07 +1967,-.08,-.21,.05,-.05,.12,-.08,.02,.01,-.06,.09,-.05,-.05,-.02,-.02,-.10,.04,-.02,-.01 +1968,-.26,-.14,.20,-.06,-.14,-.09,-.13,-.08,-.19,.09,-.05,-.14,-.08,-.07,-.15,.00,-.10,-.05 +1969,-.11,-.18,.01,.17,.18,.03,-.04,.04,.08,.09,.12,.24,.05,.02,-.14,.12,.01,.10 +1970,.08,.22,.06,.05,-.03,-.02,.02,-.10,.12,.03,.02,-.12,.03,.06,.18,.03,-.04,.06 +1971,-.02,-.15,-.18,-.07,-.05,-.16,-.07,-.01,-.05,-.04,-.07,-.08,-.08,-.08,-.10,-.10,-.08,-.06 +1972,-.22,-.18,.03,.00,-.03,.04,.01,.16,.02,.08,.02,.18,.01,-.01,-.16,.00,.07,.04 +1973,.29,.32,.29,.27,.23,.19,.13,.05,.09,.10,.04,-.07,.16,.18,.26,.26,.12,.08 +1974,-.10,-.27,-.05,-.12,-.04,-.05,-.03,.10,-.08,-.06,-.08,-.08,-.07,-.07,-.14,-.07,.01,-.07 +1975,.10,.08,.12,.04,.16,-.01,-.01,-.17,-.02,-.11,-.17,-.17,-.01,-.01,.03,.11,-.06,-.10 +1976,-.03,-.06,-.22,-.07,-.20,-.12,-.10,-.12,-.06,-.24,-.06,.11,-.10,-.12,-.09,-.16,-.11,-.12 +1977,.19,.23,.24,.26,.33,.27,.20,.18,.02,.03,.16,.03,.18,.18,.17,.28,.22,.07 +1978,.06,.10,.19,.17,.09,-.01,.04,-.14,.06,.03,.13,.08,.07,.06,.06,.15,-.04,.07 +1979,.09,-.10,.19,.15,.03,.14,.04,.17,.25,.26,.28,.47,.16,.13,.02,.13,.11,.26 +1980,.30,.39,.29,.30,.35,.20,.22,.18,.20,.13,.30,.21,.26,.28,.39,.31,.20,.21 +1981,.53,.42,.48,.32,.25,.29,.33,.35,.15,.12,.23,.41,.32,.31,.39,.35,.32,.16 +1982,.05,.15,.03,.15,.18,.06,.14,.03,.14,.13,.18,.43,.14,.14,.20,.12,.08,.15 +1983,.53,.43,.42,.28,.34,.23,.18,.36,.37,.17,.30,.17,.31,.34,.46,.34,.26,.28 +1984,.31,.14,.26,.06,.33,.02,.19,.19,.21,.13,.06,-.04,.16,.17,.20,.22,.14,.14 +1985,.22,-.04,.17,.12,.15,.15,.04,.17,.13,.12,.05,.14,.12,.10,.05,.15,.12,.10 +1986,.26,.37,.30,.22,.21,.12,.11,.16,.03,.15,.10,.13,.18,.18,.26,.25,.13,.10 +1987,.32,.43,.18,.24,.26,.35,.40,.25,.35,.33,.29,.46,.32,.29,.29,.23,.33,.32 +1988,.57,.44,.52,.43,.44,.40,.33,.39,.37,.38,.12,.29,.39,.40,.49,.46,.38,.29 +1989,.12,.30,.36,.29,.18,.15,.35,.33,.35,.29,.20,.37,.27,.27,.24,.28,.28,.28 +1990,.42,.44,.80,.56,.46,.37,.46,.34,.23,.45,.46,.40,.45,.45,.41,.61,.39,.38 +1991,.43,.50,.36,.51,.34,.53,.48,.39,.44,.29,.30,.32,.41,.41,.44,.40,.47,.34 +1992,.48,.40,.48,.28,.31,.26,.09,.08,-.01,.07,.03,.22,.22,.23,.40,.36,.14,.03 +1993,.35,.37,.36,.28,.28,.23,.25,.11,.11,.24,.04,.18,.23,.24,.31,.31,.20,.13 +1994,.26,.02,.29,.41,.28,.43,.30,.22,.31,.43,.45,.39,.32,.30,.15,.33,.32,.39 +1995,.53,.79,.47,.47,.27,.43,.46,.45,.33,.46,.45,.26,.45,.46,.57,.40,.45,.41 +1996,.24,.46,.33,.33,.27,.26,.37,.48,.26,.22,.42,.37,.33,.32,.32,.31,.37,.30 +1997,.32,.41,.52,.34,.35,.55,.34,.41,.52,.61,.65,.59,.47,.45,.37,.40,.43,.59 +1998,.58,.88,.63,.64,.68,.77,.66,.65,.42,.42,.43,.55,.61,.61,.69,.65,.70,.42 +1999,.48,.64,.33,.32,.26,.36,.39,.31,.39,.34,.37,.40,.38,.39,.56,.30,.35,.37 +2000,.25,.56,.55,.56,.35,.40,.39,.42,.39,.27,.30,.28,.39,.40,.40,.49,.40,.32 +2001,.45,.44,.55,.50,.57,.53,.59,.48,.52,.50,.72,.55,.53,.51,.39,.54,.53,.58 +2002,.77,.78,.88,.58,.64,.53,.61,.53,.64,.54,.59,.44,.63,.64,.70,.70,.56,.59 +2003,.75,.58,.60,.55,.60,.48,.58,.64,.62,.72,.53,.75,.62,.59,.59,.58,.56,.62 +2004,.58,.72,.63,.61,.37,.44,.26,.45,.50,.61,.73,.51,.53,.55,.68,.53,.38,.61 +2005,.75,.60,.74,.67,.63,.65,.61,.60,.72,.76,.74,.68,.68,.66,.62,.68,.62,.74 +2006,.56,.74,.63,.47,.48,.67,.54,.71,.66,.71,.73,.79,.64,.63,.66,.53,.64,.70 +2007,1.01,.70,.71,.77,.69,.61,.60,.60,.60,.59,.59,.50,.66,.69,.83,.72,.60,.60 +2008,.30,.38,.75,.54,.50,.50,.60,.45,.60,.67,.69,.54,.54,.54,.39,.59,.52,.66 +2009,.65,.53,.53,.61,.65,.63,.74,.69,.72,.65,.80,.67,.66,.64,.57,.60,.69,.72 +2010,.75,.84,.92,.85,.75,.68,.63,.67,.63,.71,.81,.45,.72,.74,.75,.84,.66,.72 +2011,.52,.47,.65,.64,.53,.62,.71,.74,.56,.65,.60,.60,.61,.60,.48,.61,.69,.60 +2012,.47,.48,.57,.73,.78,.65,.59,.66,.72,.80,.78,.53,.65,.65,.52,.69,.63,.77 +2013,.71,.62,.67,.53,.60,.70,.60,.70,.77,.69,.83,.70,.68,.66,.62,.60,.67,.76 +2014,.76,.55,.78,.81,.87,.67,.58,.83,.88,.80,.67,.79,.75,.74,.67,.82,.70,.79 +2015,.86,.91,.96,.75,.80,.82,.73,.80,.84,1.08,1.06,1.17,.90,.87,.86,.84,.78,.99 +2016,1.18,1.37,1.35,1.09,.96,.81,.84,1.01,.90,.87,.91,.86,1.01,1.04,1.24,1.13,.89,.89 +2017,1.02,1.13,1.16,.94,.91,.71,.82,.87,.74,.88,.87,.93,.92,.91,1.01,1.00,.80,.83 +2018,.82,.85,.88,.88,.81,.77,.84,.78,.80,1.02,.83,.93,.85,.85,.87,.86,.80,.89 +2019,.93,.96,1.17,1.01,.86,.90,.94,.95,.93,1.00,.99,1.12,.98,.96,.94,1.01,.93,.97 +2020,1.18,1.24,1.18,1.12,.99,.91,.89,.86,.96,.87,1.09,.80,1.01,1.03,1.18,1.10,.89,.97 +2021,.82,.64,.89,.76,.79,.85,.92,.81,.92,.98,.93,.88,.85,.84,.75,.81,.86,.94 +2022,.91,.89,1.05,.83,.84,.93,.94,.94,.88,.97,.73,.80,.89,.90,.90,.91,.94,.86 +2023,.88,.97,1.23,.99,.94,1.09,1.19,1.19,1.48,1.35,1.40,1.36,1.17,1.12,.88,1.05,1.16,1.41 +2024,1.25,1.44,1.40,1.32,1.16,1.23,1.20,1.30,1.21,1.35,1.30,1.27,1.29,1.29,1.35,1.29,1.25,1.29 +2025,1.38,1.26,1.37,1.25,1.08,1.07,1.02,1.18,1.25,1.19,1.21,1.06,1.19,1.21,1.30,1.23,1.09,1.22 +2026,1.09,1.25,1.32,1.17,1.13,1.18,1.25,1.40,***,***,***,***,***,***,1.13,1.21,1.28,*** diff --git a/data/science/README.md b/data/science/README.md new file mode 100644 index 0000000..c5b63e7 --- /dev/null +++ b/data/science/README.md @@ -0,0 +1,5 @@ +# Pinned NASA GISTEMP input + +`GLB.Ts+dSST.csv` is a fixed snapshot of NASA GISS GISTEMP v4 global land-ocean anomaly data. Its retrieval date, SHA-256, baseline, source URLs, and analysis period are in `gistemp_manifest.json`. + +The analysis reads the annual `J-D` column in degrees Celsius anomaly relative to 1951–1980. It uses complete years 1880–2025 and excludes the incomplete 2026 row. NASA may revise the live source; do not overwrite this snapshot silently. diff --git a/data/science/gistemp_manifest.json b/data/science/gistemp_manifest.json new file mode 100644 index 0000000..83118a8 --- /dev/null +++ b/data/science/gistemp_manifest.json @@ -0,0 +1,16 @@ +{ + "dataset": "NASA GISTEMP v4 global Land-Ocean Temperature Index", + "source_page": "https://data.giss.nasa.gov/gistemp/data_v4.html", + "source_csv": "https://data.giss.nasa.gov/gistemp/tabledata_v4/GLB.Ts%2BdSST.csv", + "retrieved_at": "2026-10-04", + "snapshot_sha256": "c9ce0750ca93a8241c42fe86cd5bf54d07b28b21ae650b0ae49a206907018cd9", + "analysis_column": "J-D", + "unit": "degrees Celsius anomaly", + "baseline_period": "1951-1980", + "analysis_period": [1880, 2025], + "notes": [ + "The CSV is checked in so the analysis and CI run without network access.", + "The source is periodically revised; this snapshot is deliberately pinned.", + "The 2026 row is incomplete and excluded; 2025 is the latest complete year in this snapshot." + ] +} diff --git a/docs/ClimateX.ai Features .pdf b/docs/ClimateX.ai Features .pdf deleted file mode 100644 index 75d17e6..0000000 Binary files a/docs/ClimateX.ai Features .pdf and /dev/null differ diff --git a/docs/images/0ED9B80C-22EA-4698-BC40-08B375937F04.png b/docs/images/0ED9B80C-22EA-4698-BC40-08B375937F04.png deleted file mode 100644 index ae406d9..0000000 Binary files a/docs/images/0ED9B80C-22EA-4698-BC40-08B375937F04.png and /dev/null differ diff --git a/docs/images/1C6C37B3-7DA3-4F08-820E-3341C0CA952F.png b/docs/images/1C6C37B3-7DA3-4F08-820E-3341C0CA952F.png deleted file mode 100644 index f0edc4a..0000000 Binary files a/docs/images/1C6C37B3-7DA3-4F08-820E-3341C0CA952F.png and /dev/null differ diff --git a/docs/images/B97B6D0D-BF7B-4C95-B84A-B75670FB988E.png b/docs/images/B97B6D0D-BF7B-4C95-B84A-B75670FB988E.png deleted file mode 100644 index 6cb2a8e..0000000 Binary files a/docs/images/B97B6D0D-BF7B-4C95-B84A-B75670FB988E.png and /dev/null differ diff --git a/docs/images/D439C0D4-5355-45EC-83D9-699765D0EB5E.png b/docs/images/D439C0D4-5355-45EC-83D9-699765D0EB5E.png deleted file mode 100644 index 969fabf..0000000 Binary files a/docs/images/D439C0D4-5355-45EC-83D9-699765D0EB5E.png and /dev/null differ diff --git a/docs/images/climate-crew-architecture-dark.jpg b/docs/images/climate-crew-architecture-dark.jpg deleted file mode 100644 index 3c37130..0000000 Binary files a/docs/images/climate-crew-architecture-dark.jpg and /dev/null differ diff --git a/docs/images/climate-crew-architecture-light.jpg b/docs/images/climate-crew-architecture-light.jpg deleted file mode 100644 index e9fb186..0000000 Binary files a/docs/images/climate-crew-architecture-light.jpg and /dev/null differ diff --git a/docs/release-assets.md b/docs/release-assets.md new file mode 100644 index 0000000..5696b68 --- /dev/null +++ b/docs/release-assets.md @@ -0,0 +1,5 @@ +# Archived asset release + +Large images, generated audit reports, and the retail prototype model binaries were removed from the current source tree. They are not loaded by the default API or frontend. The manual workflow [Publish archived binary assets](../.github/workflows/release-legacy-assets.yml) extracts them from the pre-cleanup repository snapshot, creates a compressed GitHub Release bundle, and publishes a SHA-256 manifest. + +After merging, run the workflow on `main` with the default tag `legacy-assets-v1.0.0` and verify the release download and checksum. The original blobs remain in Git history for traceability; a shallow checkout of the cleaned tree no longer downloads them. The empty audit PDF is excluded. diff --git a/docs/scientific-slice.md b/docs/scientific-slice.md new file mode 100644 index 0000000..dd1f3ee --- /dev/null +++ b/docs/scientific-slice.md @@ -0,0 +1,25 @@ +# Reproducible GISTEMP trend slice + +## Question and hypotheses + +Question: did NASA GISTEMP v4 annual global land-ocean anomalies increase over 1880–2025? + +- H1: the annual series has a positive monotonic trend over the stated period. +- H2: the apparent rise is solely a recent-endpoint effect and disappears when 2016–2025 is omitted. + +## Evidence and methods + +The input is NASA's global GISTEMP v4 CSV snapshot. The analysis uses complete annual `J-D` anomalies, relative to 1951–1980. It reports the SHA-256 of the input and separates source data from derived statistics in a Pydantic-validated evidence graph. + +- Primary trend: two-sided Mann–Kendall normal approximation with tie correction. +- Robust slope: Theil–Sen median of pairwise slopes. +- Interval: 95% circular moving-block bootstrap of detrended OLS residuals, block length 5, 2,000 replicates, seed 1042026. +- Exploratory falsification attempt: rerun trend tests after omitting the final ten complete years. This sensitivity check was not preregistered, so it should not be read as confirmatory evidence. + +## Result for the pinned snapshot + +Across 146 years, OLS slope is 0.0834 °C/decade (95% block-bootstrap interval 0.0676–0.0980); Theil–Sen slope is 0.0812 °C/decade; Mann–Kendall tau-b is 0.7347 with p=3.44×10⁻³⁹. After excluding 2016–2025, the 1880–2015 Sen slope is 0.0711 °C/decade and the Mann–Kendall p-value is 4.37×10⁻³³. This endpoint check does not overturn H1 in this dataset; H2 is challenged by the pre-2016 result. + +## Interpretation limits + +This is one descriptive global time series. It is not an independent replication across datasets and it does not attribute causes. The Mann–Kendall p-value does not adjust its variance for serial autocorrelation. The bootstrap interval depends on the chosen 5-year block scheme and does not capture every measurement or structural uncertainty. These limits are included in the machine-readable report. diff --git a/eval/README.md b/eval/README.md index 8b23a1f..a9f3aa4 100644 --- a/eval/README.md +++ b/eval/README.md @@ -1,5 +1,7 @@ # Evaluation Framework +> **Status:** this general-purpose framework is a prototype. `eval/runner.py` currently produces mock system outputs and does not call the live Climate Crew orchestrator. Its results must not be presented as measured model performance. The bounded, provider-pluggable comparison harness is in [`eval/science_quality/`](science_quality/README.md); its CI adapters are mocks and do not establish a real crew-versus-baseline score. + Comprehensive evaluation framework for multi-agent systems with support for retrieval, synthesis, debate, memory, and thought tree components. ## Overview diff --git a/eval/__init__.py b/eval/__init__.py index e16c163..651ba33 100644 --- a/eval/__init__.py +++ b/eval/__init__.py @@ -1,5 +1,10 @@ -"""Evaluation framework for multi-agent system.""" -from .runner import run_evaluation +"""Evaluation framework for the multi-agent system. + +Keep the legacy runner lazy so lightweight subpackages such as +``eval.science_quality`` can run without importing the full app's optional +provider stack. +""" + from .types import ( EvaluationQuery, RunOutput, @@ -20,3 +25,11 @@ "AggregateMetrics", "Difficulty", ] + + +def __getattr__(name: str): + if name == "run_evaluation": + from .runner import run_evaluation + + return run_evaluation + raise AttributeError(name) diff --git a/eval/science_quality/README.md b/eval/science_quality/README.md new file mode 100644 index 0000000..b94f6d2 --- /dev/null +++ b/eval/science_quality/README.md @@ -0,0 +1,29 @@ +# Climate evidence reasoning benchmark + +This benchmark contains 60 **synthetic, controlled evidence packets**: 15 each labelled supports, contradicts, mixed, and insufficient. The scenarios test whether a system reads the supplied trend estimate, retains contradictions, cites relevant evidence IDs, and states a limitation. They are not climate observations and do not measure real-world climate knowledge. + +`runner.py` accepts two callables and compares their predictions on the same cases: Climate Crew and a single-LLM baseline. It reports relationship accuracy with Wilson 95% intervals, evidence-ID precision/recall/F1, nonempty caveat rate, Brier score for correctness confidence, and a paired bootstrap 95% interval for the accuracy difference. The JSON dataset keeps ground truth separate from each model's prompt payload. + +## Offline tests + +The CI tests run mocked adapters across all 60 cases. They verify dataset validation, ground-truth isolation, metric calculation, paired confidence intervals, and deterministic output without network or API keys. Their scores are harness checks only; they are not model-performance results. + +```bash +python -m pip install -r requirements-science.txt +pytest -q tests/science +``` + +## Optional live comparison + +The real adapters use the repository's `DeepResearchOrchestrator` (Google/Gemini, with optional search integrations configured by the app) and one OpenAI chat completion. This can make many provider requests: the crew orchestrator runs several research stages per case. Start with a small pilot and inspect usage before running all 60. + +```bash +export GOOGLE_API_KEY=... +export OPENAI_API_KEY=... +python -m eval.science_quality.runner \ + --crew-adapter eval.science_quality.adapters:deep_research_crew \ + --baseline-adapter eval.science_quality.adapters:single_llm_baseline \ + --limit 5 --output /tmp/climate-crew-eval.json +``` + +For the full benchmark, omit `--limit 5`. Save and publish the resulting JSON only after a real provider run, including the exact commit, model IDs, provider configuration, date, and cost. No comparative model-performance result is claimed by the repository's offline CI. diff --git a/eval/science_quality/__init__.py b/eval/science_quality/__init__.py new file mode 100644 index 0000000..e27addf --- /dev/null +++ b/eval/science_quality/__init__.py @@ -0,0 +1 @@ +"""Reproducible, provider-pluggable benchmark for supplied-evidence reasoning.""" diff --git a/eval/science_quality/adapters.py b/eval/science_quality/adapters.py new file mode 100644 index 0000000..d7e9544 --- /dev/null +++ b/eval/science_quality/adapters.py @@ -0,0 +1,84 @@ +"""Optional provider adapters; these functions are never called by offline CI.""" + +from __future__ import annotations + +import json +import os +import re +from functools import lru_cache +from typing import Any + + +def _prompt(payload: dict[str, Any]) -> str: + return ( + "Classify the claim using only the evidence packet below. Do not use outside facts. " + "Return exactly one JSON object with keys relationship (supports|contradicts|mixed|insufficient), " + "evidence_ids (array of supplied evidence IDs), confidence (number from 0 to 1), and caveat (string).\n\n" + + json.dumps(payload, ensure_ascii=False) + ) + + +def _prediction_from_text(text: str, valid_ids: set[str]) -> dict[str, Any]: + decoder = json.JSONDecoder() + for index, character in enumerate(text): + if character != "{": + continue + try: + candidate, _ = decoder.raw_decode(text[index:]) + except json.JSONDecodeError: + continue + if not isinstance(candidate, dict) or "relationship" not in candidate: + continue + candidate["evidence_ids"] = [ + value for value in candidate.get("evidence_ids", []) if value in valid_ids + ] + candidate.setdefault("confidence", 0.5) + candidate.setdefault("caveat", "") + return candidate + raise ValueError("provider response did not contain the requested JSON prediction") + + +@lru_cache(maxsize=1) +def _crew(): + from src.agents.deep_research.orchestrator import DeepResearchOrchestrator + + key = os.getenv("GOOGLE_API_KEY") + if not key: + raise RuntimeError("GOOGLE_API_KEY is required for the Climate Crew adapter") + return DeepResearchOrchestrator( + google_api_key=key, + enable_dynamic_routing=False, + enable_meta_learning=False, + enable_collaboration=False, + enable_memory=False, + ) + + +def deep_research_crew(payload: dict[str, Any]) -> dict[str, Any]: + """Run the repository's deep-research orchestrator on one evidence packet.""" + response = _crew().run_deep_research(_prompt(payload)) + report = response.get("report", "") if isinstance(response, dict) else str(response) + valid_ids = {item["evidence_id"] for item in payload["evidence"]} + return _prediction_from_text(report, valid_ids) + + +def single_llm_baseline(payload: dict[str, Any]) -> dict[str, Any]: + """Run one OpenAI chat completion as a declared single-model baseline.""" + from openai import OpenAI + + model = os.getenv("CLIMATE_CREW_BASELINE_MODEL", "gpt-4.1-mini") + response = OpenAI().chat.completions.create( + model=model, + temperature=0, + response_format={"type": "json_object"}, + messages=[ + { + "role": "system", + "content": "You are a single-model baseline for classifying supplied evidence. Follow the requested JSON schema and do not use outside facts.", + }, + {"role": "user", "content": _prompt(payload)}, + ], + ) + text = response.choices[0].message.content or "" + valid_ids = {item["evidence_id"] for item in payload["evidence"]} + return _prediction_from_text(text, valid_ids) diff --git a/eval/science_quality/build_dataset.py b/eval/science_quality/build_dataset.py new file mode 100644 index 0000000..c5410c6 --- /dev/null +++ b/eval/science_quality/build_dataset.py @@ -0,0 +1,113 @@ +"""Build the checked-in synthetic evidence-reasoning benchmark.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +CONTEXTS = [ + ("global annual surface-temperature anomaly", "°C/decade"), + ("upper-ocean heat content", "10^22 J/decade"), + ("global mean sea level", "mm/year"), + ("September Arctic sea-ice area", "million km²/decade"), + ("mountain glacier mass balance", "m water equivalent/year"), + ("atmospheric carbon-dioxide concentration", "ppm/decade"), + ("surface-ocean pH", "pH units/decade"), + ("annual heatwave days", "days/decade"), + ("Northern Hemisphere snow-cover duration", "days/decade"), + ("summer soil moisture in a dryland region", "percent/decade"), + ("coral-bleaching incidence in a monitored reef network", "percent/decade"), + ("spring leaf-out date (later dates are positive)", "days/decade"), + ("heavy-precipitation intensity", "mm/decade"), + ("area burned by wildfires in a defined region", "km²/decade"), + ("permafrost active-layer thickness", "cm/decade"), +] + + +def build() -> dict[str, object]: + cases: list[dict[str, object]] = [] + for relation in ("supports", "contradicts", "mixed", "insufficient"): + for index, (metric, unit) in enumerate(CONTEXTS, start=1): + case_id = f"{relation[:3]}-{index:02d}" + claim = f"The supplied record establishes a positive monotonic trend in {metric}." + question = ( + "Using only the supplied evidence packet, classify its relationship to the claim as " + "supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state " + "the main limitation. Do not infer causation." + ) + evidence: list[dict[str, str]] + supporting: list[str] = [] + contradicting: list[str] = [] + if relation == "supports": + amount = 0.03 + index * 0.007 + evidence = [{ + "evidence_id": f"{case_id}-A", + "summary": f"Synthetic controlled result for {metric}, 1981–2020: slope +{amount:.3f} {unit}; 95% interval [+{amount*0.55:.3f}, +{amount*1.45:.3f}]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": f"https://example.invalid/climate-crew/{case_id}/a", + }] + supporting = [f"{case_id}-A"] + elif relation == "contradicts": + amount = 0.03 + index * 0.007 + evidence = [{ + "evidence_id": f"{case_id}-A", + "summary": f"Synthetic controlled result for {metric}, 1981–2020: slope -{amount:.3f} {unit}; 95% interval [-{amount*1.45:.3f}, -{amount*0.55:.3f}]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": f"https://example.invalid/climate-crew/{case_id}/a", + }] + contradicting = [f"{case_id}-A"] + elif relation == "mixed": + amount = 0.04 + index * 0.005 + evidence = [ + { + "evidence_id": f"{case_id}-A", + "summary": f"Synthetic controlled result for {metric}, 1981–2020, method A: slope +{amount:.3f} {unit}; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": f"https://example.invalid/climate-crew/{case_id}/a", + }, + { + "evidence_id": f"{case_id}-B", + "summary": f"Synthetic controlled result for the same metric and period, independent method B: slope -{amount*0.8:.3f} {unit}; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": f"https://example.invalid/climate-crew/{case_id}/b", + }, + ] + supporting = [f"{case_id}-A"] + contradicting = [f"{case_id}-B"] + else: + evidence = [{ + "evidence_id": f"{case_id}-A", + "summary": f"Synthetic controlled record for {metric}: the first and last readings differ by +0.4 {unit.split('/')[0]}, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": f"https://example.invalid/climate-crew/{case_id}/a", + }] + cases.append({ + "case_id": case_id, + "question": question, + "claim": claim, + "evidence": evidence, + "ground_truth": { + "relationship": relation, + "supporting_evidence_ids": supporting, + "contradicting_evidence_ids": contradicting, + "relevant_evidence_ids": sorted(set(supporting + contradicting)) or [f"{case_id}-A"], + "required_caveat": ( + "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + ), + }, + }) + return { + "schema_version": 1, + "title": "Climate Crew controlled supplied-evidence reasoning benchmark", + "provenance": "Synthetic scenarios generated by eval/science_quality/build_dataset.py; not observational climate data.", + "design": "15 metrics x 4 balanced evidence relationships; tests evidence classification, contradiction handling, citation use, and calibrated confidence over supplied records.", + "cases": cases, + } + + +if __name__ == "__main__": + target = Path(__file__).with_name("data") / "climate_evidence_benchmark.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(build(), indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + print(f"Wrote {len(build()['cases'])} synthetic cases to {target}") diff --git a/eval/science_quality/data/climate_evidence_benchmark.json b/eval/science_quality/data/climate_evidence_benchmark.json new file mode 100644 index 0000000..85dd1ee --- /dev/null +++ b/eval/science_quality/data/climate_evidence_benchmark.json @@ -0,0 +1,1553 @@ +{ + "schema_version": 1, + "title": "Climate Crew controlled supplied-evidence reasoning benchmark", + "provenance": "Synthetic scenarios generated by eval/science_quality/build_dataset.py; not observational climate data.", + "design": "15 metrics x 4 balanced evidence relationships; tests evidence classification, contradiction handling, citation use, and calibrated confidence over supplied records.", + "cases": [ + { + "case_id": "sup-01", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global annual surface-temperature anomaly.", + "evidence": [ + { + "evidence_id": "sup-01-A", + "summary": "Synthetic controlled result for global annual surface-temperature anomaly, 1981–2020: slope +0.037 °C/decade; 95% interval [+0.020, +0.054]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-01/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-01-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-01-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-02", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in upper-ocean heat content.", + "evidence": [ + { + "evidence_id": "sup-02-A", + "summary": "Synthetic controlled result for upper-ocean heat content, 1981–2020: slope +0.044 10^22 J/decade; 95% interval [+0.024, +0.064]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-02/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-02-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-02-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-03", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global mean sea level.", + "evidence": [ + { + "evidence_id": "sup-03-A", + "summary": "Synthetic controlled result for global mean sea level, 1981–2020: slope +0.051 mm/year; 95% interval [+0.028, +0.074]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-03/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-03-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-03-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-04", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in September Arctic sea-ice area.", + "evidence": [ + { + "evidence_id": "sup-04-A", + "summary": "Synthetic controlled result for September Arctic sea-ice area, 1981–2020: slope +0.058 million km²/decade; 95% interval [+0.032, +0.084]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-04/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-04-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-04-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-05", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in mountain glacier mass balance.", + "evidence": [ + { + "evidence_id": "sup-05-A", + "summary": "Synthetic controlled result for mountain glacier mass balance, 1981–2020: slope +0.065 m water equivalent/year; 95% interval [+0.036, +0.094]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-05/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-05-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-05-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-06", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in atmospheric carbon-dioxide concentration.", + "evidence": [ + { + "evidence_id": "sup-06-A", + "summary": "Synthetic controlled result for atmospheric carbon-dioxide concentration, 1981–2020: slope +0.072 ppm/decade; 95% interval [+0.040, +0.104]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-06/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-06-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-06-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-07", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in surface-ocean pH.", + "evidence": [ + { + "evidence_id": "sup-07-A", + "summary": "Synthetic controlled result for surface-ocean pH, 1981–2020: slope +0.079 pH units/decade; 95% interval [+0.043, +0.115]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-07/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-07-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-07-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-08", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in annual heatwave days.", + "evidence": [ + { + "evidence_id": "sup-08-A", + "summary": "Synthetic controlled result for annual heatwave days, 1981–2020: slope +0.086 days/decade; 95% interval [+0.047, +0.125]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-08/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-08-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-08-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-09", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in Northern Hemisphere snow-cover duration.", + "evidence": [ + { + "evidence_id": "sup-09-A", + "summary": "Synthetic controlled result for Northern Hemisphere snow-cover duration, 1981–2020: slope +0.093 days/decade; 95% interval [+0.051, +0.135]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-09/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-09-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-09-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-10", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in summer soil moisture in a dryland region.", + "evidence": [ + { + "evidence_id": "sup-10-A", + "summary": "Synthetic controlled result for summer soil moisture in a dryland region, 1981–2020: slope +0.100 percent/decade; 95% interval [+0.055, +0.145]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-10/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-10-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-10-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-11", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in coral-bleaching incidence in a monitored reef network.", + "evidence": [ + { + "evidence_id": "sup-11-A", + "summary": "Synthetic controlled result for coral-bleaching incidence in a monitored reef network, 1981–2020: slope +0.107 percent/decade; 95% interval [+0.059, +0.155]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-11/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-11-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-11-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-12", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in spring leaf-out date (later dates are positive).", + "evidence": [ + { + "evidence_id": "sup-12-A", + "summary": "Synthetic controlled result for spring leaf-out date (later dates are positive), 1981–2020: slope +0.114 days/decade; 95% interval [+0.063, +0.165]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-12/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-12-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-12-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-13", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in heavy-precipitation intensity.", + "evidence": [ + { + "evidence_id": "sup-13-A", + "summary": "Synthetic controlled result for heavy-precipitation intensity, 1981–2020: slope +0.121 mm/decade; 95% interval [+0.067, +0.175]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-13/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-13-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-13-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-14", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in area burned by wildfires in a defined region.", + "evidence": [ + { + "evidence_id": "sup-14-A", + "summary": "Synthetic controlled result for area burned by wildfires in a defined region, 1981–2020: slope +0.128 km²/decade; 95% interval [+0.070, +0.186]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-14/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-14-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-14-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "sup-15", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in permafrost active-layer thickness.", + "evidence": [ + { + "evidence_id": "sup-15-A", + "summary": "Synthetic controlled result for permafrost active-layer thickness, 1981–2020: slope +0.135 cm/decade; 95% interval [+0.074, +0.196]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/sup-15/a" + } + ], + "ground_truth": { + "relationship": "supports", + "supporting_evidence_ids": [ + "sup-15-A" + ], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "sup-15-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-01", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global annual surface-temperature anomaly.", + "evidence": [ + { + "evidence_id": "con-01-A", + "summary": "Synthetic controlled result for global annual surface-temperature anomaly, 1981–2020: slope -0.037 °C/decade; 95% interval [-0.054, -0.020]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-01/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-01-A" + ], + "relevant_evidence_ids": [ + "con-01-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-02", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in upper-ocean heat content.", + "evidence": [ + { + "evidence_id": "con-02-A", + "summary": "Synthetic controlled result for upper-ocean heat content, 1981–2020: slope -0.044 10^22 J/decade; 95% interval [-0.064, -0.024]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-02/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-02-A" + ], + "relevant_evidence_ids": [ + "con-02-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-03", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global mean sea level.", + "evidence": [ + { + "evidence_id": "con-03-A", + "summary": "Synthetic controlled result for global mean sea level, 1981–2020: slope -0.051 mm/year; 95% interval [-0.074, -0.028]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-03/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-03-A" + ], + "relevant_evidence_ids": [ + "con-03-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-04", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in September Arctic sea-ice area.", + "evidence": [ + { + "evidence_id": "con-04-A", + "summary": "Synthetic controlled result for September Arctic sea-ice area, 1981–2020: slope -0.058 million km²/decade; 95% interval [-0.084, -0.032]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-04/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-04-A" + ], + "relevant_evidence_ids": [ + "con-04-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-05", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in mountain glacier mass balance.", + "evidence": [ + { + "evidence_id": "con-05-A", + "summary": "Synthetic controlled result for mountain glacier mass balance, 1981–2020: slope -0.065 m water equivalent/year; 95% interval [-0.094, -0.036]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-05/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-05-A" + ], + "relevant_evidence_ids": [ + "con-05-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-06", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in atmospheric carbon-dioxide concentration.", + "evidence": [ + { + "evidence_id": "con-06-A", + "summary": "Synthetic controlled result for atmospheric carbon-dioxide concentration, 1981–2020: slope -0.072 ppm/decade; 95% interval [-0.104, -0.040]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-06/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-06-A" + ], + "relevant_evidence_ids": [ + "con-06-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-07", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in surface-ocean pH.", + "evidence": [ + { + "evidence_id": "con-07-A", + "summary": "Synthetic controlled result for surface-ocean pH, 1981–2020: slope -0.079 pH units/decade; 95% interval [-0.115, -0.043]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-07/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-07-A" + ], + "relevant_evidence_ids": [ + "con-07-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-08", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in annual heatwave days.", + "evidence": [ + { + "evidence_id": "con-08-A", + "summary": "Synthetic controlled result for annual heatwave days, 1981–2020: slope -0.086 days/decade; 95% interval [-0.125, -0.047]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-08/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-08-A" + ], + "relevant_evidence_ids": [ + "con-08-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-09", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in Northern Hemisphere snow-cover duration.", + "evidence": [ + { + "evidence_id": "con-09-A", + "summary": "Synthetic controlled result for Northern Hemisphere snow-cover duration, 1981–2020: slope -0.093 days/decade; 95% interval [-0.135, -0.051]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-09/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-09-A" + ], + "relevant_evidence_ids": [ + "con-09-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-10", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in summer soil moisture in a dryland region.", + "evidence": [ + { + "evidence_id": "con-10-A", + "summary": "Synthetic controlled result for summer soil moisture in a dryland region, 1981–2020: slope -0.100 percent/decade; 95% interval [-0.145, -0.055]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-10/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-10-A" + ], + "relevant_evidence_ids": [ + "con-10-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-11", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in coral-bleaching incidence in a monitored reef network.", + "evidence": [ + { + "evidence_id": "con-11-A", + "summary": "Synthetic controlled result for coral-bleaching incidence in a monitored reef network, 1981–2020: slope -0.107 percent/decade; 95% interval [-0.155, -0.059]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-11/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-11-A" + ], + "relevant_evidence_ids": [ + "con-11-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-12", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in spring leaf-out date (later dates are positive).", + "evidence": [ + { + "evidence_id": "con-12-A", + "summary": "Synthetic controlled result for spring leaf-out date (later dates are positive), 1981–2020: slope -0.114 days/decade; 95% interval [-0.165, -0.063]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-12/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-12-A" + ], + "relevant_evidence_ids": [ + "con-12-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-13", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in heavy-precipitation intensity.", + "evidence": [ + { + "evidence_id": "con-13-A", + "summary": "Synthetic controlled result for heavy-precipitation intensity, 1981–2020: slope -0.121 mm/decade; 95% interval [-0.175, -0.067]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-13/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-13-A" + ], + "relevant_evidence_ids": [ + "con-13-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-14", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in area burned by wildfires in a defined region.", + "evidence": [ + { + "evidence_id": "con-14-A", + "summary": "Synthetic controlled result for area burned by wildfires in a defined region, 1981–2020: slope -0.128 km²/decade; 95% interval [-0.186, -0.070]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-14/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-14-A" + ], + "relevant_evidence_ids": [ + "con-14-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "con-15", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in permafrost active-layer thickness.", + "evidence": [ + { + "evidence_id": "con-15-A", + "summary": "Synthetic controlled result for permafrost active-layer thickness, 1981–2020: slope -0.135 cm/decade; 95% interval [-0.196, -0.074]; two-sided trend-test p=0.01.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/con-15/a" + } + ], + "ground_truth": { + "relationship": "contradicts", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [ + "con-15-A" + ], + "relevant_evidence_ids": [ + "con-15-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-01", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global annual surface-temperature anomaly.", + "evidence": [ + { + "evidence_id": "mix-01-A", + "summary": "Synthetic controlled result for global annual surface-temperature anomaly, 1981–2020, method A: slope +0.045 °C/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-01/a" + }, + { + "evidence_id": "mix-01-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.036 °C/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-01/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-01-A" + ], + "contradicting_evidence_ids": [ + "mix-01-B" + ], + "relevant_evidence_ids": [ + "mix-01-A", + "mix-01-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-02", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in upper-ocean heat content.", + "evidence": [ + { + "evidence_id": "mix-02-A", + "summary": "Synthetic controlled result for upper-ocean heat content, 1981–2020, method A: slope +0.050 10^22 J/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-02/a" + }, + { + "evidence_id": "mix-02-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.040 10^22 J/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-02/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-02-A" + ], + "contradicting_evidence_ids": [ + "mix-02-B" + ], + "relevant_evidence_ids": [ + "mix-02-A", + "mix-02-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-03", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global mean sea level.", + "evidence": [ + { + "evidence_id": "mix-03-A", + "summary": "Synthetic controlled result for global mean sea level, 1981–2020, method A: slope +0.055 mm/year; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-03/a" + }, + { + "evidence_id": "mix-03-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.044 mm/year; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-03/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-03-A" + ], + "contradicting_evidence_ids": [ + "mix-03-B" + ], + "relevant_evidence_ids": [ + "mix-03-A", + "mix-03-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-04", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in September Arctic sea-ice area.", + "evidence": [ + { + "evidence_id": "mix-04-A", + "summary": "Synthetic controlled result for September Arctic sea-ice area, 1981–2020, method A: slope +0.060 million km²/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-04/a" + }, + { + "evidence_id": "mix-04-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.048 million km²/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-04/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-04-A" + ], + "contradicting_evidence_ids": [ + "mix-04-B" + ], + "relevant_evidence_ids": [ + "mix-04-A", + "mix-04-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-05", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in mountain glacier mass balance.", + "evidence": [ + { + "evidence_id": "mix-05-A", + "summary": "Synthetic controlled result for mountain glacier mass balance, 1981–2020, method A: slope +0.065 m water equivalent/year; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-05/a" + }, + { + "evidence_id": "mix-05-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.052 m water equivalent/year; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-05/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-05-A" + ], + "contradicting_evidence_ids": [ + "mix-05-B" + ], + "relevant_evidence_ids": [ + "mix-05-A", + "mix-05-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-06", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in atmospheric carbon-dioxide concentration.", + "evidence": [ + { + "evidence_id": "mix-06-A", + "summary": "Synthetic controlled result for atmospheric carbon-dioxide concentration, 1981–2020, method A: slope +0.070 ppm/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-06/a" + }, + { + "evidence_id": "mix-06-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.056 ppm/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-06/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-06-A" + ], + "contradicting_evidence_ids": [ + "mix-06-B" + ], + "relevant_evidence_ids": [ + "mix-06-A", + "mix-06-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-07", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in surface-ocean pH.", + "evidence": [ + { + "evidence_id": "mix-07-A", + "summary": "Synthetic controlled result for surface-ocean pH, 1981–2020, method A: slope +0.075 pH units/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-07/a" + }, + { + "evidence_id": "mix-07-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.060 pH units/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-07/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-07-A" + ], + "contradicting_evidence_ids": [ + "mix-07-B" + ], + "relevant_evidence_ids": [ + "mix-07-A", + "mix-07-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-08", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in annual heatwave days.", + "evidence": [ + { + "evidence_id": "mix-08-A", + "summary": "Synthetic controlled result for annual heatwave days, 1981–2020, method A: slope +0.080 days/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-08/a" + }, + { + "evidence_id": "mix-08-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.064 days/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-08/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-08-A" + ], + "contradicting_evidence_ids": [ + "mix-08-B" + ], + "relevant_evidence_ids": [ + "mix-08-A", + "mix-08-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-09", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in Northern Hemisphere snow-cover duration.", + "evidence": [ + { + "evidence_id": "mix-09-A", + "summary": "Synthetic controlled result for Northern Hemisphere snow-cover duration, 1981–2020, method A: slope +0.085 days/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-09/a" + }, + { + "evidence_id": "mix-09-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.068 days/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-09/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-09-A" + ], + "contradicting_evidence_ids": [ + "mix-09-B" + ], + "relevant_evidence_ids": [ + "mix-09-A", + "mix-09-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-10", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in summer soil moisture in a dryland region.", + "evidence": [ + { + "evidence_id": "mix-10-A", + "summary": "Synthetic controlled result for summer soil moisture in a dryland region, 1981–2020, method A: slope +0.090 percent/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-10/a" + }, + { + "evidence_id": "mix-10-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.072 percent/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-10/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-10-A" + ], + "contradicting_evidence_ids": [ + "mix-10-B" + ], + "relevant_evidence_ids": [ + "mix-10-A", + "mix-10-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-11", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in coral-bleaching incidence in a monitored reef network.", + "evidence": [ + { + "evidence_id": "mix-11-A", + "summary": "Synthetic controlled result for coral-bleaching incidence in a monitored reef network, 1981–2020, method A: slope +0.095 percent/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-11/a" + }, + { + "evidence_id": "mix-11-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.076 percent/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-11/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-11-A" + ], + "contradicting_evidence_ids": [ + "mix-11-B" + ], + "relevant_evidence_ids": [ + "mix-11-A", + "mix-11-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-12", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in spring leaf-out date (later dates are positive).", + "evidence": [ + { + "evidence_id": "mix-12-A", + "summary": "Synthetic controlled result for spring leaf-out date (later dates are positive), 1981–2020, method A: slope +0.100 days/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-12/a" + }, + { + "evidence_id": "mix-12-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.080 days/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-12/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-12-A" + ], + "contradicting_evidence_ids": [ + "mix-12-B" + ], + "relevant_evidence_ids": [ + "mix-12-A", + "mix-12-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-13", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in heavy-precipitation intensity.", + "evidence": [ + { + "evidence_id": "mix-13-A", + "summary": "Synthetic controlled result for heavy-precipitation intensity, 1981–2020, method A: slope +0.105 mm/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-13/a" + }, + { + "evidence_id": "mix-13-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.084 mm/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-13/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-13-A" + ], + "contradicting_evidence_ids": [ + "mix-13-B" + ], + "relevant_evidence_ids": [ + "mix-13-A", + "mix-13-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-14", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in area burned by wildfires in a defined region.", + "evidence": [ + { + "evidence_id": "mix-14-A", + "summary": "Synthetic controlled result for area burned by wildfires in a defined region, 1981–2020, method A: slope +0.110 km²/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-14/a" + }, + { + "evidence_id": "mix-14-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.088 km²/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-14/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-14-A" + ], + "contradicting_evidence_ids": [ + "mix-14-B" + ], + "relevant_evidence_ids": [ + "mix-14-A", + "mix-14-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "mix-15", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in permafrost active-layer thickness.", + "evidence": [ + { + "evidence_id": "mix-15-A", + "summary": "Synthetic controlled result for permafrost active-layer thickness, 1981–2020, method A: slope +0.115 cm/decade; 95% interval excludes zero; p=0.02.", + "source_title": "Synthetic benchmark record A", + "source_url": "https://example.invalid/climate-crew/mix-15/a" + }, + { + "evidence_id": "mix-15-B", + "summary": "Synthetic controlled result for the same metric and period, independent method B: slope -0.092 cm/decade; 95% interval excludes zero; p=0.03.", + "source_title": "Synthetic benchmark record B", + "source_url": "https://example.invalid/climate-crew/mix-15/b" + } + ], + "ground_truth": { + "relationship": "mixed", + "supporting_evidence_ids": [ + "mix-15-A" + ], + "contradicting_evidence_ids": [ + "mix-15-B" + ], + "relevant_evidence_ids": [ + "mix-15-A", + "mix-15-B" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-01", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global annual surface-temperature anomaly.", + "evidence": [ + { + "evidence_id": "ins-01-A", + "summary": "Synthetic controlled record for global annual surface-temperature anomaly: the first and last readings differ by +0.4 °C, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-01/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-01-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-02", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in upper-ocean heat content.", + "evidence": [ + { + "evidence_id": "ins-02-A", + "summary": "Synthetic controlled record for upper-ocean heat content: the first and last readings differ by +0.4 10^22 J, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-02/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-02-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-03", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in global mean sea level.", + "evidence": [ + { + "evidence_id": "ins-03-A", + "summary": "Synthetic controlled record for global mean sea level: the first and last readings differ by +0.4 mm, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-03/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-03-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-04", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in September Arctic sea-ice area.", + "evidence": [ + { + "evidence_id": "ins-04-A", + "summary": "Synthetic controlled record for September Arctic sea-ice area: the first and last readings differ by +0.4 million km², but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-04/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-04-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-05", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in mountain glacier mass balance.", + "evidence": [ + { + "evidence_id": "ins-05-A", + "summary": "Synthetic controlled record for mountain glacier mass balance: the first and last readings differ by +0.4 m water equivalent, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-05/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-05-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-06", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in atmospheric carbon-dioxide concentration.", + "evidence": [ + { + "evidence_id": "ins-06-A", + "summary": "Synthetic controlled record for atmospheric carbon-dioxide concentration: the first and last readings differ by +0.4 ppm, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-06/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-06-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-07", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in surface-ocean pH.", + "evidence": [ + { + "evidence_id": "ins-07-A", + "summary": "Synthetic controlled record for surface-ocean pH: the first and last readings differ by +0.4 pH units, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-07/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-07-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-08", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in annual heatwave days.", + "evidence": [ + { + "evidence_id": "ins-08-A", + "summary": "Synthetic controlled record for annual heatwave days: the first and last readings differ by +0.4 days, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-08/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-08-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-09", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in Northern Hemisphere snow-cover duration.", + "evidence": [ + { + "evidence_id": "ins-09-A", + "summary": "Synthetic controlled record for Northern Hemisphere snow-cover duration: the first and last readings differ by +0.4 days, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-09/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-09-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-10", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in summer soil moisture in a dryland region.", + "evidence": [ + { + "evidence_id": "ins-10-A", + "summary": "Synthetic controlled record for summer soil moisture in a dryland region: the first and last readings differ by +0.4 percent, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-10/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-10-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-11", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in coral-bleaching incidence in a monitored reef network.", + "evidence": [ + { + "evidence_id": "ins-11-A", + "summary": "Synthetic controlled record for coral-bleaching incidence in a monitored reef network: the first and last readings differ by +0.4 percent, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-11/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-11-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-12", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in spring leaf-out date (later dates are positive).", + "evidence": [ + { + "evidence_id": "ins-12-A", + "summary": "Synthetic controlled record for spring leaf-out date (later dates are positive): the first and last readings differ by +0.4 days, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-12/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-12-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-13", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in heavy-precipitation intensity.", + "evidence": [ + { + "evidence_id": "ins-13-A", + "summary": "Synthetic controlled record for heavy-precipitation intensity: the first and last readings differ by +0.4 mm, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-13/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-13-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-14", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in area burned by wildfires in a defined region.", + "evidence": [ + { + "evidence_id": "ins-14-A", + "summary": "Synthetic controlled record for area burned by wildfires in a defined region: the first and last readings differ by +0.4 km², but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-14/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-14-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + }, + { + "case_id": "ins-15", + "question": "Using only the supplied evidence packet, classify its relationship to the claim as supports, contradicts, mixed, or insufficient. Cite the evidence IDs used and state the main limitation. Do not infer causation.", + "claim": "The supplied record establishes a positive monotonic trend in permafrost active-layer thickness.", + "evidence": [ + { + "evidence_id": "ins-15-A", + "summary": "Synthetic controlled record for permafrost active-layer thickness: the first and last readings differ by +0.4 cm, but intermediate observations, a trend estimate, measurement uncertainty, and method details are not supplied.", + "source_title": "Synthetic incomplete benchmark record", + "source_url": "https://example.invalid/climate-crew/ins-15/a" + } + ], + "ground_truth": { + "relationship": "insufficient", + "supporting_evidence_ids": [], + "contradicting_evidence_ids": [], + "relevant_evidence_ids": [ + "ins-15-A" + ], + "required_caveat": "This classification concerns only the supplied synthetic evidence and does not establish causation or a real-world climate fact." + } + } + ] +} diff --git a/eval/science_quality/runner.py b/eval/science_quality/runner.py new file mode 100644 index 0000000..9fdae08 --- /dev/null +++ b/eval/science_quality/runner.py @@ -0,0 +1,250 @@ +"""Run a paired evidence-reasoning comparison with pluggable model adapters.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import math +import random +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +ROOT = Path(__file__).resolve().parents[2] +DATASET = Path(__file__).with_name("data") / "climate_evidence_benchmark.json" +RELATIONSHIPS = ("supports", "contradicts", "mixed", "insufficient") +Adapter = Callable[[dict[str, Any]], Any] + + +class Evidence(BaseModel): + model_config = ConfigDict(extra="forbid") + evidence_id: str = Field(min_length=1) + summary: str = Field(min_length=1) + source_title: str = Field(min_length=1) + source_url: str + + +class GroundTruth(BaseModel): + model_config = ConfigDict(extra="forbid") + relationship: Literal["supports", "contradicts", "mixed", "insufficient"] + supporting_evidence_ids: list[str] + contradicting_evidence_ids: list[str] + relevant_evidence_ids: list[str] + required_caveat: str + + +class BenchmarkCase(BaseModel): + model_config = ConfigDict(extra="forbid") + case_id: str + question: str + claim: str + evidence: list[Evidence] = Field(min_length=1) + ground_truth: GroundTruth + + @model_validator(mode="after") + def references_resolve(self) -> "BenchmarkCase": + ids = {item.evidence_id for item in self.evidence} + unknown = set(self.ground_truth.relevant_evidence_ids) - ids + if unknown: + raise ValueError(f"unknown relevant evidence IDs: {sorted(unknown)}") + return self + + def prompt_payload(self) -> dict[str, Any]: + """Return model input without leaking ground truth.""" + return { + "case_id": self.case_id, + "question": self.question, + "claim": self.claim, + "evidence": [item.model_dump(mode="json") for item in self.evidence], + } + + +class Prediction(BaseModel): + model_config = ConfigDict(extra="forbid") + relationship: Literal["supports", "contradicts", "mixed", "insufficient"] + evidence_ids: list[str] = Field(default_factory=list) + confidence: float = Field(ge=0.0, le=1.0) + caveat: str = "" + + +def load_cases(path: Path = DATASET) -> list[BenchmarkCase]: + raw = path.read_bytes() + payload = json.loads(raw) + cases = [BenchmarkCase.model_validate(item) for item in payload["cases"]] + ids = [case.case_id for case in cases] + if len(ids) != len(set(ids)): + raise ValueError("benchmark case IDs must be unique") + if len(cases) < 50: + raise ValueError(f"benchmark needs at least 50 cases; found {len(cases)}") + counts = {label: sum(case.ground_truth.relationship == label for case in cases) for label in RELATIONSHIPS} + if any(count == 0 for count in counts.values()): + raise ValueError(f"benchmark must contain every relationship; got {counts}") + return cases + + +def wilson_interval(successes: int, total: int, z: float = 1.959963984540054) -> list[float]: + if total == 0: + return [0.0, 1.0] + proportion = successes / total + denominator = 1 + z * z / total + centre = (proportion + z * z / (2 * total)) / denominator + margin = z * math.sqrt(proportion * (1 - proportion) / total + z * z / (4 * total * total)) / denominator + return [max(0.0, centre - margin), min(1.0, centre + margin)] + + +def _system_metrics( + cases: list[BenchmarkCase], + predictions: dict[str, Prediction], +) -> dict[str, Any]: + correct = 0 + relevant_hit = relevant_total = cited_hit = cited_total = caveats_present = 0 + brier_total = 0.0 + per_case: dict[str, bool] = {} + for case in cases: + prediction = predictions[case.case_id] + is_correct = prediction.relationship == case.ground_truth.relationship + per_case[case.case_id] = is_correct + correct += int(is_correct) + brier_total += (prediction.confidence - float(is_correct)) ** 2 + expected = set(case.ground_truth.relevant_evidence_ids) + cited = set(prediction.evidence_ids) + relevant_hit += len(expected & cited) + relevant_total += len(expected) + cited_hit += len(expected & cited) + cited_total += len(cited) + caveats_present += int(bool(prediction.caveat.strip())) + evidence_recall = relevant_hit / relevant_total if relevant_total else 0.0 + evidence_precision = cited_hit / cited_total if cited_total else 0.0 + evidence_f1 = 2 * evidence_precision * evidence_recall / (evidence_precision + evidence_recall) if evidence_precision + evidence_recall else 0.0 + return { + "n": len(cases), + "accuracy": correct / len(cases) if cases else 0.0, + "accuracy_95_wilson_ci": wilson_interval(correct, len(cases)), + "evidence_precision": evidence_precision, + "evidence_recall": evidence_recall, + "evidence_f1": evidence_f1, + "nonempty_caveat_rate": caveats_present / len(cases) if cases else 0.0, + "brier_score_correctness": brier_total / len(cases) if cases else 0.0, + "per_case_correct": per_case, + } + + +def paired_bootstrap_ci( + crew_correct: dict[str, bool], + baseline_correct: dict[str, bool], + *, + replicates: int = 10000, + seed: int = 1042026, +) -> list[float]: + ids = sorted(set(crew_correct) & set(baseline_correct)) + if not ids: + return [0.0, 0.0] + differences = [int(crew_correct[key]) - int(baseline_correct[key]) for key in ids] + rng = random.Random(seed) + means = [] + for _ in range(replicates): + sample = [differences[rng.randrange(len(differences))] for _ in differences] + means.append(sum(sample) / len(sample)) + means.sort() + return [means[int(0.025 * (len(means) - 1))], means[int(0.975 * (len(means) - 1))]] + + +def compare_predictions( + cases: list[BenchmarkCase], + crew_predictions: dict[str, Prediction], + single_predictions: dict[str, Prediction], + *, + bootstrap_replicates: int = 10000, +) -> dict[str, Any]: + expected_ids = {case.case_id for case in cases} + if set(crew_predictions) != expected_ids or set(single_predictions) != expected_ids: + raise ValueError("each system must return exactly one prediction per benchmark case") + crew = _system_metrics(cases, crew_predictions) + single = _system_metrics(cases, single_predictions) + delta = crew["accuracy"] - single["accuracy"] + return { + "case_count": len(cases), + "crew": {key: value for key, value in crew.items() if key != "per_case_correct"}, + "single_llm_baseline": {key: value for key, value in single.items() if key != "per_case_correct"}, + "paired_accuracy_difference_crew_minus_baseline": { + "estimate": delta, + "95_ci": paired_bootstrap_ci( + crew["per_case_correct"], single["per_case_correct"], replicates=bootstrap_replicates + ), + "method": "paired bootstrap over cases", + "seed": 1042026, + "replicates": bootstrap_replicates, + }, + } + + +def run_comparison( + cases: list[BenchmarkCase], + crew_adapter: Adapter, + baseline_adapter: Adapter, + *, + bootstrap_replicates: int = 10000, +) -> tuple[dict[str, Any], dict[str, Any]]: + crew_predictions: dict[str, Prediction] = {} + baseline_predictions: dict[str, Prediction] = {} + for case in cases: + payload = case.prompt_payload() + crew_predictions[case.case_id] = Prediction.model_validate(crew_adapter(payload)) + baseline_predictions[case.case_id] = Prediction.model_validate(baseline_adapter(payload)) + result = compare_predictions( + cases, crew_predictions, baseline_predictions, bootstrap_replicates=bootstrap_replicates + ) + result["dataset_sha256"] = hashlib.sha256(DATASET.read_bytes()).hexdigest() + result["run_kind"] = "provider-run; inspect adapter/model names before interpreting" + return result, {"crew": crew_predictions, "single_llm_baseline": baseline_predictions} + + +def _load_callable(spec: str) -> Adapter: + module_name, separator, attribute = spec.partition(":") + if not separator: + raise ValueError("adapter must use module:function syntax") + function = getattr(importlib.import_module(module_name), attribute) + if not callable(function): + raise TypeError(f"adapter {spec} is not callable") + return function + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--crew-adapter", required=True, help="module:function for the multi-agent crew") + parser.add_argument("--baseline-adapter", required=True, help="module:function for one LLM call") + parser.add_argument("--limit", type=int, default=None, help="run the first N cases for a cost-limited pilot") + parser.add_argument("--output", type=Path, default=None, help="optional JSON report path") + parser.add_argument("--bootstrap-replicates", type=int, default=10000) + args = parser.parse_args() + cases = load_cases() + if args.limit is not None: + if args.limit < 1: + parser.error("--limit must be >= 1") + cases = cases[:args.limit] + result, predictions = run_comparison( + cases, _load_callable(args.crew_adapter), _load_callable(args.baseline_adapter), + bootstrap_replicates=args.bootstrap_replicates, + ) + result["run_kind"] = "provider-run" + result["adapters"] = {"crew": args.crew_adapter, "single_llm_baseline": args.baseline_adapter} + result["created_at_utc"] = datetime.now(timezone.utc).isoformat() + result["predictions"] = { + name: {case_id: prediction.model_dump(mode="json") for case_id, prediction in rows.items()} + for name, rows in predictions.items() + } + output = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(output, encoding="utf-8") + print(output, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..20194fd --- /dev/null +++ b/examples/README.md @@ -0,0 +1,9 @@ +# Archived experiments + +These directories preserve non-core experiments removed from the default Climate Crew API and frontend. + +- `retail_analytics/` — former retail audit, causal-analysis, dataset and UI experiment; model binaries will be in the archived asset release. +- `climate_finance/` — former market-impact, ESG, climate-finance API and UI experiment. +- `legacy_assets/` — README for the historical images, generated audit reports and feature document, which the release workflow packages. + +The source experiments remain in `examples/` for traceability and are not mounted by `main.py` or the current frontend. They may depend on provider credentials and the former application package layout; they are not maintained as standalone packages. Their presence is not evidence that those features are part of the current Climate Crew science workflow. diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/examples/climate_finance/README.md b/examples/climate_finance/README.md new file mode 100644 index 0000000..a238331 --- /dev/null +++ b/examples/climate_finance/README.md @@ -0,0 +1,3 @@ +# Climate finance and ESG archive + +This is a preserved prototype, not a supported Climate Crew feature. Its API route, agent exports and frontend page were removed from the default application. The directory keeps the former implementation and manual checks for provenance. The code may require optional dependencies, provider credentials and package-path adjustments before it can run independently. diff --git a/examples/climate_finance/__init__.py b/examples/climate_finance/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/frontend/src/pages/ClimateFinancePage.tsx b/examples/climate_finance/frontend/ClimateFinancePage.tsx similarity index 100% rename from frontend/src/pages/ClimateFinancePage.tsx rename to examples/climate_finance/frontend/ClimateFinancePage.tsx diff --git a/frontend/src/services/climateFinanceApi.ts b/examples/climate_finance/frontend/climateFinanceApi.ts similarity index 100% rename from frontend/src/services/climateFinanceApi.ts rename to examples/climate_finance/frontend/climateFinanceApi.ts diff --git a/test/test_climate_finance.py b/examples/climate_finance/manual/climate_finance.py similarity index 97% rename from test/test_climate_finance.py rename to examples/climate_finance/manual/climate_finance.py index ff8960e..60ab041 100644 --- a/test/test_climate_finance.py +++ b/examples/climate_finance/manual/climate_finance.py @@ -18,7 +18,7 @@ def test_yahoo_finance(): print("🧪 Testing Yahoo Finance Client") print("="*60) - from src.data_sources.yahoo_finance import YahooFinanceClient + from examples.climate_finance.src.data_sources.yahoo_finance import YahooFinanceClient client = YahooFinanceClient() @@ -101,7 +101,7 @@ def test_finnhub(): print("🧪 Testing Finnhub Client (ESG Data)") print("="*60) - from src.data_sources.esg_data import FinnhubClient + from examples.climate_finance.src.data_sources.esg_data import FinnhubClient api_key = os.getenv("FINNHUB_API_KEY") if not api_key: @@ -159,7 +159,7 @@ def test_alpha_vantage(): print("🧪 Testing Alpha Vantage Client (News Sentiment)") print("="*60) - from src.data_sources.esg_data import AlphaVantageClient + from examples.climate_finance.src.data_sources.esg_data import AlphaVantageClient api_key = os.getenv("ALPHA_VANTAGE_API_KEY") if not api_key: diff --git a/test/test_agents.py b/examples/climate_finance/manual/finance_agents.py similarity index 100% rename from test/test_agents.py rename to examples/climate_finance/manual/finance_agents.py diff --git a/examples/climate_finance/src/__init__.py b/examples/climate_finance/src/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/src/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/examples/climate_finance/src/agents/__init__.py b/examples/climate_finance/src/agents/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/src/agents/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/src/agents/climate_finance_orchestrator.py b/examples/climate_finance/src/agents/climate_finance_orchestrator.py similarity index 96% rename from src/agents/climate_finance_orchestrator.py rename to examples/climate_finance/src/agents/climate_finance_orchestrator.py index 2ffca73..05898d2 100644 --- a/src/agents/climate_finance_orchestrator.py +++ b/examples/climate_finance/src/agents/climate_finance_orchestrator.py @@ -1,9 +1,9 @@ from typing import Any, Dict, List, Optional, TYPE_CHECKING from datetime import datetime -from .base import BaseAgent +from src.agents.base import BaseAgent from .market_impact_agent import MarketImpactAgent -from .regulatory_risk_agent import RegulatoryRiskAgent +from src.agents.regulatory_risk_agent import RegulatoryRiskAgent from .esg_agent import ESGAgent if TYPE_CHECKING: diff --git a/src/agents/esg_agent.py b/examples/climate_finance/src/agents/esg_agent.py similarity index 98% rename from src/agents/esg_agent.py rename to examples/climate_finance/src/agents/esg_agent.py index e7494be..a752c07 100644 --- a/src/agents/esg_agent.py +++ b/examples/climate_finance/src/agents/esg_agent.py @@ -4,9 +4,9 @@ from langchain_core.tools import tool -from .base import BaseAgent +from src.agents.base import BaseAgent from ..data_sources.esg_data import FinnhubClient, AlphaVantageClient -from ..data_sources.web_search import WebSearchClient +from src.data_sources.web_search import WebSearchClient if TYPE_CHECKING: from ..memory import MemoryManager diff --git a/src/agents/market_impact_agent.py b/examples/climate_finance/src/agents/market_impact_agent.py similarity index 98% rename from src/agents/market_impact_agent.py rename to examples/climate_finance/src/agents/market_impact_agent.py index 640bcbc..f7679ad 100644 --- a/src/agents/market_impact_agent.py +++ b/examples/climate_finance/src/agents/market_impact_agent.py @@ -3,10 +3,10 @@ from langchain_core.tools import tool -from .base import BaseAgent +from src.agents.base import BaseAgent from ..data_sources.yahoo_finance import YahooFinanceClient -from ..data_sources.web_search import WebSearchClient -from ..rag.rag_manager import RAGManager +from src.data_sources.web_search import WebSearchClient +from src.rag.rag_manager import RAGManager if TYPE_CHECKING: from ..memory import MemoryManager diff --git a/examples/climate_finance/src/api/__init__.py b/examples/climate_finance/src/api/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/src/api/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/src/api/climate_finance.py b/examples/climate_finance/src/api/climate_finance.py similarity index 99% rename from src/api/climate_finance.py rename to examples/climate_finance/src/api/climate_finance.py index 6390980..33892ef 100644 --- a/src/api/climate_finance.py +++ b/examples/climate_finance/src/api/climate_finance.py @@ -5,7 +5,7 @@ from ..agents.climate_finance_orchestrator import ClimateFinanceOrchestrator from ..agents.market_impact_agent import MarketImpactAgent -from ..agents.regulatory_risk_agent import RegulatoryRiskAgent +from src.agents.regulatory_risk_agent import RegulatoryRiskAgent from ..agents.esg_agent import ESGAgent router = APIRouter(prefix="/climate-finance", tags=["Climate Finance"]) diff --git a/examples/climate_finance/src/data_sources/__init__.py b/examples/climate_finance/src/data_sources/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/src/data_sources/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/src/data_sources/esg_data.py b/examples/climate_finance/src/data_sources/esg_data.py similarity index 99% rename from src/data_sources/esg_data.py rename to examples/climate_finance/src/data_sources/esg_data.py index 6508a3f..11236c7 100644 --- a/src/data_sources/esg_data.py +++ b/examples/climate_finance/src/data_sources/esg_data.py @@ -4,7 +4,7 @@ import requests import os -from .base import BaseDataSource +from src.data_sources.base import BaseDataSource logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/data_sources/yahoo_finance.py b/examples/climate_finance/src/data_sources/yahoo_finance.py similarity index 99% rename from src/data_sources/yahoo_finance.py rename to examples/climate_finance/src/data_sources/yahoo_finance.py index 1f1fc2d..91b834e 100644 --- a/src/data_sources/yahoo_finance.py +++ b/examples/climate_finance/src/data_sources/yahoo_finance.py @@ -10,7 +10,7 @@ YFINANCE_AVAILABLE = False yf = None -from .base import BaseDataSource +from src.data_sources.base import BaseDataSource logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/examples/climate_finance/src/models/__init__.py b/examples/climate_finance/src/models/__init__.py new file mode 100644 index 0000000..61cfbbe --- /dev/null +++ b/examples/climate_finance/src/models/__init__.py @@ -0,0 +1 @@ +from .climate_risk_model import ClimateRiskModel diff --git a/src/models/climate_risk_model.py b/examples/climate_finance/src/models/climate_risk_model.py similarity index 100% rename from src/models/climate_risk_model.py rename to examples/climate_finance/src/models/climate_risk_model.py diff --git a/examples/climate_finance/src/tools/__init__.py b/examples/climate_finance/src/tools/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/climate_finance/src/tools/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/src/tools/climate_finance_tools.py b/examples/climate_finance/src/tools/climate_finance_tools.py similarity index 99% rename from src/tools/climate_finance_tools.py rename to examples/climate_finance/src/tools/climate_finance_tools.py index 4adefb8..9ed2380 100644 --- a/src/tools/climate_finance_tools.py +++ b/examples/climate_finance/src/tools/climate_finance_tools.py @@ -16,7 +16,7 @@ from ..data_sources.yahoo_finance import YahooFinanceClient from ..data_sources.esg_data import FinnhubClient, AlphaVantageClient, ESGDataClient -from ..data_sources.regulatory_data import RegulatoryDataClient, ClimateWatchClient +from src.data_sources.regulatory_data import RegulatoryDataClient, ClimateWatchClient logger = logging.getLogger(__name__) diff --git a/examples/legacy_assets/README.md b/examples/legacy_assets/README.md new file mode 100644 index 0000000..8a620fc --- /dev/null +++ b/examples/legacy_assets/README.md @@ -0,0 +1,3 @@ +# Legacy assets + +Historical images, audit reports, and the retail model binaries were removed from the current source tree. The `legacy-assets-v1.0.0` release bundle recovers them from the pre-cleanup repository snapshot after the release workflow runs. Their upstream origin and license are not documented in repository history. These files are retained for traceability only and are not used by the default Climate Crew application. See [`PROVENANCE.md`](../../PROVENANCE.md) and [`docs/release-assets.md`](../../docs/release-assets.md). diff --git a/examples/retail_analytics/README.md b/examples/retail_analytics/README.md new file mode 100644 index 0000000..5634352 --- /dev/null +++ b/examples/retail_analytics/README.md @@ -0,0 +1,3 @@ +# Retail analytics archive + +This is a preserved prototype, not a supported Climate Crew feature. Its API route and frontend page were removed from the default application. The directory keeps the former implementation, manual checks and dataset together for provenance. Model binaries were removed from the source tree and are packaged in the archived asset release. The code may require optional dependencies and provider credentials before it can run independently. diff --git a/examples/retail_analytics/__init__.py b/examples/retail_analytics/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/retail_analytics/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/artifacts/retail_audit_log.txt b/examples/retail_analytics/artifacts/retail_audit_log.txt similarity index 100% rename from artifacts/retail_audit_log.txt rename to examples/retail_analytics/artifacts/retail_audit_log.txt diff --git a/data/ingest_hf_data.py b/examples/retail_analytics/data/ingest_hf_data.py similarity index 100% rename from data/ingest_hf_data.py rename to examples/retail_analytics/data/ingest_hf_data.py diff --git a/data/inspect_dataset.py b/examples/retail_analytics/data/inspect_dataset.py similarity index 100% rename from data/inspect_dataset.py rename to examples/retail_analytics/data/inspect_dataset.py diff --git a/data/real_causal_data.csv b/examples/retail_analytics/data/real_causal_data.csv similarity index 100% rename from data/real_causal_data.csv rename to examples/retail_analytics/data/real_causal_data.csv diff --git a/frontend/src/pages/RetailPage.tsx b/examples/retail_analytics/frontend/RetailPage.tsx similarity index 100% rename from frontend/src/pages/RetailPage.tsx rename to examples/retail_analytics/frontend/RetailPage.tsx diff --git a/test/test_live_retail.py b/examples/retail_analytics/manual/live_retail.py similarity index 100% rename from test/test_live_retail.py rename to examples/retail_analytics/manual/live_retail.py diff --git a/test/test_retail_analytics.py b/examples/retail_analytics/manual/retail_analytics.py similarity index 98% rename from test/test_retail_analytics.py rename to examples/retail_analytics/manual/retail_analytics.py index 2c077a9..7d73550 100644 --- a/test/test_retail_analytics.py +++ b/examples/retail_analytics/manual/retail_analytics.py @@ -5,7 +5,7 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from src.agents.retail_analytics import RetailSwarm +from examples.retail_analytics.src.agents.retail_analytics import RetailSwarm from src.agents.air_quality_agent import AirQualityAgent from src.agents.wildfire_agent import WildfireAgent from src.agents.flood_agent import FloodAgent diff --git a/test/test_retail_endpoint.py b/examples/retail_analytics/manual/retail_endpoint.py similarity index 95% rename from test/test_retail_endpoint.py rename to examples/retail_analytics/manual/retail_endpoint.py index 63b875e..fe42762 100644 --- a/test/test_retail_endpoint.py +++ b/examples/retail_analytics/manual/retail_endpoint.py @@ -12,7 +12,7 @@ # Import the router directly to test it # We need to create a dummy app to mount the router from fastapi import FastAPI -from src.agents.retail_analytics.router import router +from examples.retail_analytics.src.agents.retail_analytics.router import router app = FastAPI() app.include_router(router) diff --git a/test/verify_retail_ml.py b/examples/retail_analytics/manual/verify_retail_ml.py similarity index 95% rename from test/verify_retail_ml.py rename to examples/retail_analytics/manual/verify_retail_ml.py index 98e513c..779b653 100644 --- a/test/verify_retail_ml.py +++ b/examples/retail_analytics/manual/verify_retail_ml.py @@ -8,7 +8,7 @@ # Add properties relative to location sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) -from src.agents.retail_analytics.orchestrator import RetailSwarm +from examples.retail_analytics.src.agents.retail_analytics.orchestrator import RetailSwarm from src.agents.flood_agent import FloodAgent from src.agents.wildfire_agent import WildfireAgent diff --git a/examples/retail_analytics/src/__init__.py b/examples/retail_analytics/src/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/retail_analytics/src/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/examples/retail_analytics/src/agents/__init__.py b/examples/retail_analytics/src/agents/__init__.py new file mode 100644 index 0000000..f421f70 --- /dev/null +++ b/examples/retail_analytics/src/agents/__init__.py @@ -0,0 +1 @@ +"""Archived prototype modules retained for historical context.""" diff --git a/src/agents/retail_analytics/__init__.py b/examples/retail_analytics/src/agents/retail_analytics/__init__.py similarity index 100% rename from src/agents/retail_analytics/__init__.py rename to examples/retail_analytics/src/agents/retail_analytics/__init__.py diff --git a/src/agents/retail_analytics/causal_agent.py b/examples/retail_analytics/src/agents/retail_analytics/causal_agent.py similarity index 99% rename from src/agents/retail_analytics/causal_agent.py rename to examples/retail_analytics/src/agents/retail_analytics/causal_agent.py index 6cf1114..3915592 100644 --- a/src/agents/retail_analytics/causal_agent.py +++ b/examples/retail_analytics/src/agents/retail_analytics/causal_agent.py @@ -1,7 +1,7 @@ from typing import List, Dict, Any from langchain_core.tools import tool from langchain_google_genai import ChatGoogleGenerativeAI -from ..base import BaseAgent +from src.agents.base import BaseAgent from .models import CausalNode class CausalAnalystAgent(BaseAgent): diff --git a/src/agents/retail_analytics/causal_learning.py b/examples/retail_analytics/src/agents/retail_analytics/causal_learning.py similarity index 91% rename from src/agents/retail_analytics/causal_learning.py rename to examples/retail_analytics/src/agents/retail_analytics/causal_learning.py index 2932644..940fd94 100644 --- a/src/agents/retail_analytics/causal_learning.py +++ b/examples/retail_analytics/src/agents/retail_analytics/causal_learning.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import pandas as pd import numpy as np import joblib @@ -16,9 +17,12 @@ class CausalLearner: Predicts 'Impact Score' and 'Probability of Disruption' based on weather features. """ - def __init__(self, data_path: str = "data/real_causal_data.csv", model_dir: str = "models"): - self.data_path = os.path.join(os.getcwd(), data_path) - self.model_dir = os.path.join(os.getcwd(), model_dir) + def __init__(self, data_path: Optional[str] = None, model_dir: Optional[str] = None): + example_root = Path(__file__).resolve().parents[3] + data_path = Path(data_path or "data/real_causal_data.csv") + model_dir = Path(model_dir or "models") + self.data_path = str(data_path if data_path.is_absolute() else example_root / data_path) + self.model_dir = str(model_dir if model_dir.is_absolute() else example_root / model_dir) self.impact_model_path = os.path.join(self.model_dir, "causal_impact_regressor.pkl") self.prob_model_path = os.path.join(self.model_dir, "causal_prob_classifier.pkl") diff --git a/src/agents/retail_analytics/communications_agent.py b/examples/retail_analytics/src/agents/retail_analytics/communications_agent.py similarity index 97% rename from src/agents/retail_analytics/communications_agent.py rename to examples/retail_analytics/src/agents/retail_analytics/communications_agent.py index 363cde3..3001d64 100644 --- a/src/agents/retail_analytics/communications_agent.py +++ b/examples/retail_analytics/src/agents/retail_analytics/communications_agent.py @@ -1,4 +1,4 @@ -from ..base import BaseAgent +from src.agents.base import BaseAgent from .models import CausalNode, ActionPlan, CommunicationPackage class CorporateCommsAgent(BaseAgent): diff --git a/src/agents/retail_analytics/dashboard_agent.py b/examples/retail_analytics/src/agents/retail_analytics/dashboard_agent.py similarity index 98% rename from src/agents/retail_analytics/dashboard_agent.py rename to examples/retail_analytics/src/agents/retail_analytics/dashboard_agent.py index f9c2e55..c11e370 100644 --- a/src/agents/retail_analytics/dashboard_agent.py +++ b/examples/retail_analytics/src/agents/retail_analytics/dashboard_agent.py @@ -1,5 +1,5 @@ from typing import Any, Dict -from ..base import BaseAgent +from src.agents.base import BaseAgent from .models import DashboardPayload, CausalNode, ActionPlan, KPIGauge from langchain_core.tools import tool diff --git a/src/agents/retail_analytics/fetch_real_data.py b/examples/retail_analytics/src/agents/retail_analytics/fetch_real_data.py similarity index 96% rename from src/agents/retail_analytics/fetch_real_data.py rename to examples/retail_analytics/src/agents/retail_analytics/fetch_real_data.py index ac2a574..4f9096e 100644 --- a/src/agents/retail_analytics/fetch_real_data.py +++ b/examples/retail_analytics/src/agents/retail_analytics/fetch_real_data.py @@ -1,6 +1,7 @@ import csv import os import sys +from pathlib import Path import random import time import numpy as np @@ -8,10 +9,11 @@ from sklearn.ensemble import IsolationForest from sklearn.preprocessing import MinMaxScaler -# Add src to path to import OpenMeteoClient -sys.path.append(os.path.join(os.path.dirname(__file__), "../../../src")) +# Add the repository root so this archived module can reuse the climate connector. +REPO_ROOT = Path(__file__).resolve().parents[5] +sys.path.append(str(REPO_ROOT)) -from data_sources.open_meteo import OpenMeteoClient +from src.data_sources.open_meteo import OpenMeteoClient def fetch_and_process_data(): client = OpenMeteoClient() diff --git a/src/agents/retail_analytics/models.py b/examples/retail_analytics/src/agents/retail_analytics/models.py similarity index 100% rename from src/agents/retail_analytics/models.py rename to examples/retail_analytics/src/agents/retail_analytics/models.py diff --git a/src/agents/retail_analytics/orchestrator.py b/examples/retail_analytics/src/agents/retail_analytics/orchestrator.py similarity index 100% rename from src/agents/retail_analytics/orchestrator.py rename to examples/retail_analytics/src/agents/retail_analytics/orchestrator.py diff --git a/src/agents/retail_analytics/router.py b/examples/retail_analytics/src/agents/retail_analytics/router.py similarity index 86% rename from src/agents/retail_analytics/router.py rename to examples/retail_analytics/src/agents/retail_analytics/router.py index 2f15573..278a083 100644 --- a/src/agents/retail_analytics/router.py +++ b/examples/retail_analytics/src/agents/retail_analytics/router.py @@ -2,12 +2,12 @@ from pydantic import BaseModel import os from .orchestrator import RetailSwarm -from ..air_quality_agent import AirQualityAgent -from ..wildfire_agent import WildfireAgent -from ..flood_agent import FloodAgent -from ..biodiversity_agent import BiodiversityAgent -from ..deforestation_agent import DeforestationAgent -from ..climate_anomaly_agent import ClimateAnomalyAgent +from src.agents.air_quality_agent import AirQualityAgent +from src.agents.wildfire_agent import WildfireAgent +from src.agents.flood_agent import FloodAgent +from src.agents.biodiversity_agent import BiodiversityAgent +from src.agents.deforestation_agent import DeforestationAgent +from src.agents.climate_anomaly_agent import ClimateAnomalyAgent from .models import CausalNode, ActionPlan, CommunicationPackage, DashboardPayload diff --git a/src/agents/retail_analytics/supply_chain_agent.py b/examples/retail_analytics/src/agents/retail_analytics/supply_chain_agent.py similarity index 97% rename from src/agents/retail_analytics/supply_chain_agent.py rename to examples/retail_analytics/src/agents/retail_analytics/supply_chain_agent.py index 386533f..816d87b 100644 --- a/src/agents/retail_analytics/supply_chain_agent.py +++ b/examples/retail_analytics/src/agents/retail_analytics/supply_chain_agent.py @@ -1,4 +1,4 @@ -from ..base import BaseAgent +from src.agents.base import BaseAgent from .models import CausalNode, ActionPlan class SupplyChainOptimizerAgent(BaseAgent): diff --git a/src/agents/retail_analytics/train_pipeline.py b/examples/retail_analytics/src/agents/retail_analytics/train_pipeline.py similarity index 67% rename from src/agents/retail_analytics/train_pipeline.py rename to examples/retail_analytics/src/agents/retail_analytics/train_pipeline.py index b2cdbc8..5052964 100644 --- a/src/agents/retail_analytics/train_pipeline.py +++ b/examples/retail_analytics/src/agents/retail_analytics/train_pipeline.py @@ -1,11 +1,13 @@ import sys import os +from pathlib import Path import logging -# Add project root to path -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) +# Add the repository root to support the archived package import. +REPO_ROOT = Path(__file__).resolve().parents[5] +sys.path.append(str(REPO_ROOT)) -from src.agents.retail_analytics.causal_learning import CausalLearner +from examples.retail_analytics.src.agents.retail_analytics.causal_learning import CausalLearner # Configure logging logging.basicConfig(level=logging.INFO) diff --git a/frontend/public/docs/images/climate-crew-architecture-dark.jpg b/frontend/public/docs/images/climate-crew-architecture-dark.jpg deleted file mode 100644 index 3c37130..0000000 Binary files a/frontend/public/docs/images/climate-crew-architecture-dark.jpg and /dev/null differ diff --git a/frontend/public/docs/images/climate-crew-architecture-light.jpg b/frontend/public/docs/images/climate-crew-architecture-light.jpg deleted file mode 100644 index e9fb186..0000000 Binary files a/frontend/public/docs/images/climate-crew-architecture-light.jpg and /dev/null differ diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7fea5b1..3cbe6d1 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,12 +3,10 @@ import { CacheProvider } from './contexts/CacheContext'; import DashboardLayout from './layouts/DashboardLayout'; import OverviewPage from './pages/OverviewPage'; import DeepResearchPage from './pages/DeepResearchPage'; -import RetailPage from './pages/RetailPage'; import AuditPage from './pages/AuditPage'; import TimeMachinePage from './pages/TimeMachinePage'; import AgentUplinkPage from './pages/AgentUplinkPage'; import LandingPage from './pages/LandingPage'; -import ClimateFinancePage from './pages/ClimateFinancePage'; import DisasterPage from './pages/DisasterPage'; import IntelSwarmPage from './pages/IntelSwarmPage'; @@ -27,8 +25,6 @@ function App() { } /> } /> } /> - } /> - } /> } /> } /> {/* Redirect /app/* to /app */} diff --git a/frontend/src/layouts/DashboardLayout.tsx b/frontend/src/layouts/DashboardLayout.tsx index 0695d3f..2dac00c 100644 --- a/frontend/src/layouts/DashboardLayout.tsx +++ b/frontend/src/layouts/DashboardLayout.tsx @@ -5,8 +5,6 @@ import { Globe, Microscope, FileCheck, - ShoppingBag, - DollarSign, MessageSquare, Menu, X, @@ -22,8 +20,6 @@ const DashboardLayout: React.FC = () => { { path: '/app/globe', label: 'Earth Observatory', icon: Globe }, { path: '/app/research', label: 'Research', icon: Microscope }, { path: '/app/audit', label: 'Environmental Analysis', icon: FileCheck }, - { path: '/app/retail', label: 'Applied Research', icon: ShoppingBag }, - { path: '/app/finance', label: 'Climate Finance', icon: DollarSign }, { path: '/app/disaster', label: 'Hazards & Resilience', icon: Siren }, { path: '/app/chat', label: 'Research Crew', icon: MessageSquare }, ]; diff --git a/frontend/src/pages/LandingPage.tsx b/frontend/src/pages/LandingPage.tsx index 65c38a2..a298459 100644 --- a/frontend/src/pages/LandingPage.tsx +++ b/frontend/src/pages/LandingPage.tsx @@ -1,73 +1,84 @@ import { useNavigate } from 'react-router-dom'; -import { ArrowRight, Database, FlaskConical, Globe2, Microscope, Network, SearchCheck, ShieldCheck } from 'lucide-react'; - -const ASSET_ROOT = 'https://raw.githubusercontent.com/Masterleeaus/Climate-crew/main/docs/images'; -const HERO_IMAGE = `${ASSET_ROOT}/B97B6D0D-BF7B-4C95-B84A-B75670FB988E.png`; -const ARCHITECTURE_LIGHT = `${ASSET_ROOT}/1C6C37B3-7DA3-4F08-820E-3341C0CA952F.png`; -const ARCHITECTURE_DARK = `${ASSET_ROOT}/0ED9B80C-22EA-4698-BC40-08B375937F04.png`; -const LOGO_IMAGE = `${ASSET_ROOT}/D439C0D4-5355-45EC-83D9-699765D0EB5E.png`; +import { ArrowRight, BarChart3, Database, FlaskConical, Network, ShieldCheck, Telescope } from 'lucide-react'; const capabilities = [ - [Microscope, 'Specialist Research Crew', 'Coordinate climate, Earth-system and research-method agents around shared scientific questions.'], - [Network, 'Evidence Graph', 'Connect claims to sources, datasets, observations, methods, assumptions, contradictions and replication history.'], - [SearchCheck, 'Adversarial Validation', 'Challenge findings through competing explanations, falsification, replication, uncertainty and novelty review.'], - [Globe2, 'Earth Observatory', 'Bring live and historical Earth-observation data into the same workspace as literature, models and simulations.'], - [Database, 'Research Memory', 'Preserve hypotheses, evidence, rejected explanations and unresolved questions across long-running projects.'], - [ShieldCheck, 'Traceable Discovery', 'Advance findings only with visible provenance, uncertainty, replication and validation history.'], -] as const; + { + icon: BarChart3, + status: 'Implemented and tested', + title: 'Reproducible climate result', + description: 'A pinned NASA GISTEMP v4 snapshot, Mann–Kendall trend test, Sen slope, block-bootstrap interval, endpoint sensitivity check, JSON report, and SVG plot.', + }, + { + icon: Database, + status: 'Prototype', + title: 'Climate data connectors', + description: 'The repository contains climate and environmental data-source adapters. Provider availability and data quality still depend on each integration.', + }, + { + icon: Telescope, + status: 'Prototype', + title: 'Multi-agent research', + description: 'A deep-research orchestrator coordinates planning, search, debate, and synthesis. It depends on external model and search credentials.', + }, + { + icon: Network, + status: 'Bounded slice', + title: 'Evidence and hypotheses', + description: 'Typed records connect two hypotheses, source-attributed evidence, one falsification test, uncertainty, and a report for the GISTEMP case.', + }, + { + icon: ShieldCheck, + status: 'Planned', + title: 'Project-wide validation', + description: 'A persistent evidence graph, independent dataset replication, and expert approval gates are not yet integrated across the application.', + }, +]; const LandingPage = () => { const navigate = useNavigate(); - return
-
+ return
+
- +
-
- Climate Crew evidence-driven multi-agent climate research and discovery -
-
- - -
-
- -
-
-
Evidence-driven multi-agent research
-

From questions and observations to validated climate discovery.

-

Climate Crew coordinates specialised research agents, Earth observations, literature, datasets, models and simulations through independent investigation, transparent evidence, open challenge and reproducible validation.

-

Agreement is not evidence. Scientific claims advance because the supporting evidence survives scrutiny.

+
+
+
Active climate research prototype
+

Evidence first. Claims kept in scope.

+

Climate Crew is a climate and environmental research codebase with specialist agents, data connectors, and a multi-agent research prototype. Its complete application is not yet validated as a scientific workflow.

+
+ + View code and results +
-
-
-

System architecture

The whole research system at a glance.

Research lifecycle, specialist crew, Evidence Graph, Earth Observatory, Discovery Scouts and validation gates operate as one scientific workflow.

- Climate Crew platform architecture overview +
+
+

Capability status

What the repository supports today

Each label describes the current code and test boundary, not a product-readiness claim.

+
+ {capabilities.map(({ icon: Icon, status, title, description }) =>
+
{status}
+

{title}

{description}

+
)} +
-
-

Research architecture

A research system, not a chatbot.

{capabilities.map(([Icon,title,description]) =>

{title}

{description}

)}
-
- -
-
- Climate Crew research engine, agent team, Earth Observatory, discovery scouts and validation architecture -
+
+

Reproducible path

One result can be rerun without API keys.

The GISTEMP analysis uses a checked-in NASA data snapshot and reports its source, hash, trend test, uncertainty interval, sensitivity check, and limits. CI checks the output and runs offline tests; it does not score model performance.

Open the analysis code
-

From investigation to discovery

Discover carefully.

Preserve disagreement, expose uncertainty and keep the path from observation to externally validated discovery auditable.

+

Research principle

Agreement is not evidence.

A useful research system must expose its sources, methods, uncertainty, and remaining gaps so another person can challenge the result.

-
Climate Crew — evidence-driven multi-agent climate research and discovery.Investigate independently · Challenge aggressively · Preserve the evidence · Discover carefully
+
; }; diff --git a/main.py b/main.py index 76713d4..35334e2 100644 --- a/main.py +++ b/main.py @@ -6,14 +6,12 @@ from fastapi.middleware.cors import CORSMiddleware from dotenv import load_dotenv from src.agents.deep_research.router import router as deep_research_router -from src.agents.retail_analytics.router import router as retail_router from src.api.agents import router as agents_router from src.api.data import router as data_router from src.api.media import router as media_router from src.agents.climate_time_machine.router import router as ctm_router from src.api.global_pulse import router as global_pulse_router from src.agents.audit_router import router as audit_router -from src.api.climate_finance import router as climate_finance_router from src.api.news import router as news_router, get_qdrant_manager, get_news_fetcher, get_article_processor from src.api.disaster_management import router as disaster_router @@ -23,7 +21,7 @@ app = FastAPI( title="Climate Crew API", - description="Evidence-driven multi-agent climate research and discovery system", + description="Climate and environmental research prototype with specialist agents and data-source integrations", version="1.0.0" ) @@ -38,14 +36,12 @@ # Include Routers app.include_router(deep_research_router) -app.include_router(retail_router) app.include_router(agents_router) app.include_router(data_router) app.include_router(media_router) app.include_router(ctm_router) app.include_router(global_pulse_router) app.include_router(audit_router) -app.include_router(climate_finance_router) app.include_router(news_router) app.include_router(disaster_router) @@ -64,4 +60,4 @@ async def health_check(): if __name__ == "__main__": port = int(os.getenv("PORT", 8000)) - uvicorn.run("main:app", host="0.0.0.0", port=port, reload=True) \ No newline at end of file + uvicorn.run("main:app", host="0.0.0.0", port=port, reload=True) diff --git a/models/causal_impact_regressor.pkl b/models/causal_impact_regressor.pkl deleted file mode 100644 index 06c805d..0000000 Binary files a/models/causal_impact_regressor.pkl and /dev/null differ diff --git a/models/causal_prob_classifier.pkl b/models/causal_prob_classifier.pkl deleted file mode 100644 index c714957..0000000 Binary files a/models/causal_prob_classifier.pkl and /dev/null differ diff --git a/requirements-science.txt b/requirements-science.txt new file mode 100644 index 0000000..f5228db --- /dev/null +++ b/requirements-science.txt @@ -0,0 +1,3 @@ +# Minimal environment for the offline scientific slice, reproducibility script, and CI tests. +pydantic==2.12.5 +pytest==9.0.2 diff --git a/src/agents/__init__.py b/src/agents/__init__.py index 99b5c32..3ffffcd 100644 --- a/src/agents/__init__.py +++ b/src/agents/__init__.py @@ -9,10 +9,7 @@ from .climate_anomaly_agent import ClimateAnomalyAgent from .carbon_emissions_agent import CarbonEmissionsAgent from .orchestrator import OrchestratorAgent -from .market_impact_agent import MarketImpactAgent from .regulatory_risk_agent import RegulatoryRiskAgent -from .esg_agent import ESGAgent -from .climate_finance_orchestrator import ClimateFinanceOrchestrator from .disaster_management_agent import DisasterManagementAgent from .meta_learning import MetaLearner, RoutingSuggestion from .dynamic_router import DynamicRouter, QueryComplexity, RoutingDecision, AGENT_CAPABILITIES @@ -30,10 +27,7 @@ "ClimateAnomalyAgent", "CarbonEmissionsAgent", "OrchestratorAgent", - "MarketImpactAgent", "RegulatoryRiskAgent", - "ESGAgent", - "ClimateFinanceOrchestrator", "DisasterManagementAgent", # Meta-learning & Dynamic Routing "MetaLearner", diff --git a/src/data_sources/__init__.py b/src/data_sources/__init__.py index a8bb7df..650609d 100644 --- a/src/data_sources/__init__.py +++ b/src/data_sources/__init__.py @@ -9,9 +9,7 @@ from .gbif import GBIFClient from .climate_trace import ClimateTraceClient -# Climate-Finance Data Sources (REAL APIs) -from .yahoo_finance import YahooFinanceClient -from .esg_data import FinnhubClient, AlphaVantageClient, ESGDataClient +# Climate policy and regulatory data sources from .regulatory_data import CarbonPricingClient, RegulatoryDataClient # Disaster Management Data Sources @@ -33,11 +31,7 @@ "OpenMeteoMarineClient", "GBIFClient", "ClimateTraceClient", - # Climate-Finance - "YahooFinanceClient", - "FinnhubClient", - "AlphaVantageClient", - "ESGDataClient", + # Climate policy and regulatory data "CarbonPricingClient", "RegulatoryDataClient", # Disaster Management diff --git a/src/models/__init__.py b/src/models/__init__.py index 61cfbbe..f285c41 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -1 +1 @@ -from .climate_risk_model import ClimateRiskModel +"""Shared mathematical models used by the default climate application.""" diff --git a/src/science/__init__.py b/src/science/__init__.py new file mode 100644 index 0000000..69c446b --- /dev/null +++ b/src/science/__init__.py @@ -0,0 +1,11 @@ +"""Small, auditable scientific review components for Climate Crew.""" + +from .models import EvidenceRecord, FalsificationTest, Hypothesis, ScientificReport, UncertaintyAssessment + +__all__ = [ + "EvidenceRecord", + "FalsificationTest", + "Hypothesis", + "ScientificReport", + "UncertaintyAssessment", +] diff --git a/src/science/models.py b/src/science/models.py new file mode 100644 index 0000000..4361ad4 --- /dev/null +++ b/src/science/models.py @@ -0,0 +1,100 @@ +"""Typed records for an evidence-to-falsification research review.""" + +from __future__ import annotations + +from datetime import date +from typing import Literal + +from pydantic import AnyHttpUrl, BaseModel, ConfigDict, Field, model_validator + + +EvidenceKind = Literal["observation", "dataset", "publication", "derived", "modelled"] +HypothesisStatus = Literal["supported", "challenged", "unresolved"] +TestOutcome = Literal["falsified", "not_falsified", "inconclusive"] + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + +class EvidenceRecord(StrictModel): + """An attributable input or derived result in a small evidence graph.""" + + evidence_id: str = Field(min_length=1) + statement: str = Field(min_length=1) + source_title: str = Field(min_length=1) + source_url: AnyHttpUrl + evidence_kind: EvidenceKind + retrieved_at: date + independence_group: str = Field(min_length=1) + derived_from: list[str] = Field(default_factory=list) + limitations: list[str] = Field(default_factory=list) + + +class Hypothesis(StrictModel): + hypothesis_id: str = Field(min_length=1) + statement: str = Field(min_length=1) + status: HypothesisStatus + + +class FalsificationTest(StrictModel): + test_id: str = Field(min_length=1) + hypothesis_id: str = Field(min_length=1) + question: str = Field(min_length=1) + method: str = Field(min_length=1) + outcome: TestOutcome + result_summary: str = Field(min_length=1) + evidence_ids: list[str] = Field(min_length=1) + predeclared: bool = False + + +class UncertaintyAssessment(StrictModel): + estimate: float + unit: str = Field(min_length=1) + interval_level: float = Field(gt=0.0, lt=1.0) + lower_bound: float + upper_bound: float + method: str = Field(min_length=1) + caveats: list[str] = Field(min_length=1) + + @model_validator(mode="after") + def bounds_are_ordered(self) -> "UncertaintyAssessment": + if self.lower_bound > self.upper_bound: + raise ValueError("lower_bound must not exceed upper_bound") + if not self.lower_bound <= self.estimate <= self.upper_bound: + raise ValueError("estimate must lie inside the uncertainty interval") + return self + + +class ScientificReport(StrictModel): + """A reviewable result with explicit evidence, alternatives and limits.""" + + question: str = Field(min_length=1) + hypotheses: list[Hypothesis] = Field(min_length=2) + evidence: list[EvidenceRecord] = Field(min_length=1) + falsification_tests: list[FalsificationTest] = Field(min_length=1) + uncertainty: UncertaintyAssessment + conclusion: str = Field(min_length=1) + limitations: list[str] = Field(min_length=1) + + @model_validator(mode="after") + def graph_references_resolve(self) -> "ScientificReport": + evidence_ids = {record.evidence_id for record in self.evidence} + hypothesis_ids = {hypothesis.hypothesis_id for hypothesis in self.hypotheses} + if len(evidence_ids) != len(self.evidence): + raise ValueError("evidence_id values must be unique") + if len(hypothesis_ids) != len(self.hypotheses): + raise ValueError("hypothesis_id values must be unique") + for record in self.evidence: + unknown = set(record.derived_from) - evidence_ids + if unknown: + raise ValueError(f"{record.evidence_id} refers to unknown evidence: {sorted(unknown)}") + if record.evidence_id in record.derived_from: + raise ValueError("an evidence record cannot derive from itself") + for test in self.falsification_tests: + if test.hypothesis_id not in hypothesis_ids: + raise ValueError(f"{test.test_id} refers to an unknown hypothesis") + unknown = set(test.evidence_ids) - evidence_ids + if unknown: + raise ValueError(f"{test.test_id} refers to unknown evidence: {sorted(unknown)}") + return self diff --git a/src/science/statistics.py b/src/science/statistics.py new file mode 100644 index 0000000..e9806b3 --- /dev/null +++ b/src/science/statistics.py @@ -0,0 +1,199 @@ +"""Dependency-light trend tests used by the reproducible GISTEMP example.""" + +from __future__ import annotations + +import csv +import math +import random +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + + +@dataclass(frozen=True) +class AnnualSeries: + years: tuple[int, ...] + anomalies_c: tuple[float, ...] + + +def load_gistemp_annual(path: str | Path, start: int = 1880, end: int = 2025) -> AnnualSeries: + """Read complete-year J-D anomalies from NASA's published GISTEMP CSV.""" + with Path(path).open("r", encoding="utf-8-sig", newline="") as stream: + reader = csv.reader(stream) + next(reader) # descriptive title row + header = next(reader) + year_index = header.index("Year") + annual_index = header.index("J-D") + values: list[tuple[int, float]] = [] + for row in reader: + if len(row) <= annual_index or not row[year_index].strip().isdigit(): + continue + year = int(row[year_index]) + raw = row[annual_index].strip() + if start <= year <= end and raw not in {"", "***"}: + values.append((year, float(raw))) + if len(values) < 8: + raise ValueError(f"Need at least 8 annual observations, found {len(values)}") + years = tuple(year for year, _ in values) + if years != tuple(range(years[0], years[-1] + 1)): + raise ValueError("Annual series has missing or duplicated years") + return AnnualSeries(years, tuple(value for _, value in values)) + + +def ols_slope(years: Iterable[int], values: Iterable[float]) -> float: + """Return the ordinary least-squares slope in units per calendar year.""" + x = tuple(float(value) for value in years) + y = tuple(float(value) for value in values) + if len(x) != len(y) or len(x) < 2: + raise ValueError("years and values must have equal length >= 2") + x_bar = sum(x) / len(x) + y_bar = sum(y) / len(y) + denominator = sum((value - x_bar) ** 2 for value in x) + if denominator == 0: + raise ValueError("years must contain at least two distinct values") + return sum((xv - x_bar) * (yv - y_bar) for xv, yv in zip(x, y)) / denominator + + +def sen_slope(years: Iterable[int], values: Iterable[float]) -> float: + """Return the median of all pairwise slopes (Theil-Sen estimator).""" + x = tuple(years) + y = tuple(values) + slopes = [ + (y[j] - y[i]) / (x[j] - x[i]) + for i in range(len(x) - 1) + for j in range(i + 1, len(x)) + ] + if not slopes: + raise ValueError("at least two observations are required") + slopes.sort() + middle = len(slopes) // 2 + if len(slopes) % 2: + return slopes[middle] + return (slopes[middle - 1] + slopes[middle]) / 2 + + +def mann_kendall(years: Iterable[int], values: Iterable[float]) -> dict[str, float | int]: + """Two-sided normal-approximation Mann-Kendall monotonic trend test.""" + x = tuple(years) + y = tuple(values) + if len(x) != len(y) or len(y) < 8: + raise ValueError("years and values must have equal length >= 8") + n = len(y) + score = sum( + 1 if y[j] > y[i] else -1 if y[j] < y[i] else 0 + for i in range(n - 1) + for j in range(i + 1, n) + ) + tied = 0 + counts: dict[float, int] = {} + for value in y: + counts[value] = counts.get(value, 0) + 1 + for count in counts.values(): + tied += count * (count - 1) * (2 * count + 5) + variance = (n * (n - 1) * (2 * n + 5) - tied) / 18 + if variance <= 0: + z = 0.0 + p_value = 1.0 + else: + z = (score - 1) / math.sqrt(variance) if score > 0 else ( + (score + 1) / math.sqrt(variance) if score < 0 else 0.0 + ) + p_value = math.erfc(abs(z) / math.sqrt(2.0)) + pair_count = n * (n - 1) / 2 + tied_pairs = sum(count * (count - 1) / 2 for count in counts.values()) + tau_b = score / math.sqrt(pair_count * (pair_count - tied_pairs)) if pair_count > tied_pairs else 0.0 + return {"n": n, "s": score, "tau_b": tau_b, "z": z, "p_value": p_value} + + +def _percentile(sorted_values: list[float], probability: float) -> float: + position = (len(sorted_values) - 1) * probability + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return sorted_values[lower] + weight = position - lower + return sorted_values[lower] * (1 - weight) + sorted_values[upper] * weight + + +def moving_block_bootstrap_ci( + years: Iterable[int], + values: Iterable[float], + *, + block_length: int = 5, + replicates: int = 2000, + seed: int = 1042026, + alpha: float = 0.05, +) -> tuple[float, float]: + """Bootstrap OLS slope CI by resampling circular blocks of detrended residuals.""" + x = tuple(float(value) for value in years) + y = tuple(float(value) for value in values) + if len(x) != len(y) or len(y) < 12: + raise ValueError("years and values must have equal length >= 12") + if not 1 <= block_length <= len(y): + raise ValueError("block_length must be between 1 and the series length") + slope = ols_slope(x, y) + x_bar = sum(x) / len(x) + y_bar = sum(y) / len(y) + intercept = y_bar - slope * x_bar + residuals = [value - (intercept + slope * year) for year, value in zip(x, y)] + rng = random.Random(seed) + estimates: list[float] = [] + n = len(y) + for _ in range(replicates): + sample: list[float] = [] + while len(sample) < n: + first = rng.randrange(n) + sample.extend(residuals[(first + offset) % n] for offset in range(block_length)) + boot_y = [intercept + slope * year + residual for year, residual in zip(x, sample[:n])] + estimates.append(ols_slope(x, boot_y)) + estimates.sort() + return _percentile(estimates, alpha / 2), _percentile(estimates, 1 - alpha / 2) + + +def trend_summary(series: AnnualSeries, *, bootstrap_replicates: int = 2000) -> dict[str, object]: + """Compute the primary trend and endpoint sensitivity checks.""" + years, values = series.years, series.anomalies_c + if len(years) < 20: + raise ValueError("at least 20 consecutive annual observations are required for sensitivity checks") + slope = ols_slope(years, values) + lower, upper = moving_block_bootstrap_ci( + years, values, block_length=5, replicates=bootstrap_replicates, seed=1042026 + ) + mk = mann_kendall(years, values) + cut_year = years[-1] - 10 + older = AnnualSeries( + tuple(year for year in years if year <= cut_year), + tuple(value for year, value in zip(years, values) if year <= cut_year), + ) + recent_start = max(years[0], 1970) + recent = AnnualSeries( + tuple(year for year in years if year >= recent_start), + tuple(value for year, value in zip(years, values) if year >= recent_start), + ) + older_test = mann_kendall(older.years, older.anomalies_c) + recent_test = mann_kendall(recent.years, recent.anomalies_c) + return { + "period": [years[0], years[-1]], + "n": len(years), + "ols_slope_c_per_decade": slope * 10, + "ols_slope_95_ci_c_per_decade": [lower * 10, upper * 10], + "sen_slope_c_per_decade": sen_slope(years, values) * 10, + "mann_kendall": mk, + "endpoint_falsification": { + "question": f"Does the positive trend persist after excluding {cut_year + 1}-{years[-1]}?", + "period": [years[0], cut_year], + "sen_slope_c_per_decade": sen_slope(older.years, older.anomalies_c) * 10, + "mann_kendall": older_test, + "outcome": "not_falsified" if older_test["tau_b"] > 0 and older_test["p_value"] < 0.05 else "inconclusive_or_falsified", + }, + "recent_window_check": { + "period": [recent_start, years[-1]], + "sen_slope_c_per_decade": sen_slope(recent.years, recent.anomalies_c) * 10, + "mann_kendall": recent_test, + }, + "methods": { + "trend_test": "two-sided Mann-Kendall normal approximation with tie correction", + "robust_slope": "Theil-Sen median of all pairwise slopes", + "interval": "95% circular moving-block bootstrap of detrended OLS residuals; 5-year blocks; seed 1042026; 2000 replicates", + }, + } diff --git a/test/test_api_endpoints.py b/test/test_api_endpoints.py index 000534e..ef16f12 100644 --- a/test/test_api_endpoints.py +++ b/test/test_api_endpoints.py @@ -15,22 +15,6 @@ def test_root(): assert response.status_code == 200 assert "message" in response.json() -# --- Retail Analytics --- -def test_retail_audit(): - print("\nTesting /retail/audit ...") - # Use a location that should work (e.g., California redwoods) - payload = {"latitude": 41.3, "longitude": -124.0} - response = client.post("/retail/audit", json=payload) - - if response.status_code == 200: - data = response.json() - assert "causal_graph" in data - assert "action_plan" in data - assert "dashboard_payload" in data - print(" [PASS] Retail Audit") - else: - print(f" [FAIL] Retail Audit: {response.status_code} - {response.text}") - # --- Domain Agents --- def test_all_agents(): print("\nTesting /agents endpoints ...") diff --git a/tests/science/conftest.py b/tests/science/conftest.py new file mode 100644 index 0000000..2b0274c --- /dev/null +++ b/tests/science/conftest.py @@ -0,0 +1,20 @@ +"""Fail fast if an offline scientific test unexpectedly opens a network socket.""" + +import socket +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +@pytest.fixture(autouse=True) +def forbid_network(monkeypatch): + def denied(*args, **kwargs): + raise AssertionError("network access is disabled in the science test suite") + + monkeypatch.setattr(socket.socket, "connect", denied) + monkeypatch.setattr(socket, "create_connection", denied) diff --git a/tests/science/test_benchmark.py b/tests/science/test_benchmark.py new file mode 100644 index 0000000..786ce74 --- /dev/null +++ b/tests/science/test_benchmark.py @@ -0,0 +1,67 @@ +from eval.science_quality.runner import ( + Prediction, + compare_predictions, + load_cases, + run_comparison, + wilson_interval, +) + + +def test_benchmark_is_balanced_and_model_prompt_hides_labels(): + cases = load_cases() + assert len(cases) == 60 + assert {case.ground_truth.relationship for case in cases} == { + "supports", "contradicts", "mixed", "insufficient" + } + assert "ground_truth" not in cases[0].prompt_payload() + + +def test_mocked_crew_and_single_llm_compare_all_cases_offline(): + cases = load_cases() + labels = {case.case_id: case.ground_truth.relationship for case in cases} + evidence = {case.case_id: case.ground_truth.relevant_evidence_ids for case in cases} + + # These are test doubles for the comparison plumbing, not benchmark results. + def crew_mock(payload): + return { + "relationship": labels[payload["case_id"]], + "evidence_ids": evidence[payload["case_id"]], + "confidence": 0.9, + "caveat": "synthetic test response", + } + + def single_mock(payload): + return { + "relationship": "insufficient", + "evidence_ids": [], + "confidence": 0.6, + "caveat": "synthetic test response", + } + + result, _ = run_comparison(cases, crew_mock, single_mock, bootstrap_replicates=500) + assert result["case_count"] == 60 + assert result["crew"]["accuracy"] == 1.0 + assert result["single_llm_baseline"]["accuracy"] == 0.25 + interval = result["paired_accuracy_difference_crew_minus_baseline"]["95_ci"] + assert interval[0] > 0 + assert interval[0] <= interval[1] + + +def test_prediction_validation_and_wilson_bounds(): + prediction = Prediction.model_validate({ + "relationship": "mixed", "evidence_ids": ["a", "b"], "confidence": 0.72 + }) + assert prediction.confidence == 0.72 + interval = wilson_interval(7, 10) + assert 0 <= interval[0] < 0.7 < interval[1] <= 1 + + +def test_comparison_requires_one_prediction_per_case(): + cases = load_cases()[:2] + empty = {} + try: + compare_predictions(cases, empty, empty) + except ValueError as error: + assert "exactly one prediction" in str(error) + else: + raise AssertionError("incomplete model outputs must fail closed") diff --git a/tests/science/test_scientific_slice.py b/tests/science/test_scientific_slice.py new file mode 100644 index 0000000..87d1fb7 --- /dev/null +++ b/tests/science/test_scientific_slice.py @@ -0,0 +1,72 @@ +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from analysis.reproduce_gistemp import DATA, OUTPUT, build_outputs, make_report +from src.science.models import ScientificReport +from src.science.statistics import ( + load_gistemp_annual, + mann_kendall, + moving_block_bootstrap_ci, + sen_slope, + trend_summary, +) + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_trend_statistics_match_hand_calculable_series(): + years = [2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007] + values = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + assert sen_slope(years, values) == 1.0 + positive = mann_kendall(years, values) + negative = mann_kendall(years, list(reversed(values))) + tied = mann_kendall(years, [2.0] * len(years)) + assert positive["tau_b"] == 1.0 + assert positive["p_value"] < 0.01 + assert negative["tau_b"] == -1.0 + assert negative["p_value"] < 0.01 + assert tied["p_value"] == 1.0 + + +def test_moving_block_bootstrap_is_seeded_and_contains_slope(): + years = list(range(1980, 2000)) + values = [0.04 * (year - 1980) + ((year % 3) - 1) * 0.05 for year in years] + first = moving_block_bootstrap_ci(years, values, replicates=100, seed=7) + second = moving_block_bootstrap_ci(years, values, replicates=100, seed=7) + assert first == second + assert first[0] < first[1] + + +def test_report_requires_valid_evidence_and_hypothesis_references(): + # Build a minimal valid graph through the production report builder's schema. + report, _ = build_outputs(DATA) + parsed = ScientificReport.model_validate(json.loads(report)["report"]) + assert len(parsed.hypotheses) >= 2 + assert all(test.evidence_ids for test in parsed.falsification_tests) + assert all(not test.predeclared for test in parsed.falsification_tests) + invalid = parsed.model_dump(mode="json") + invalid["falsification_tests"][0]["evidence_ids"] = ["missing"] + with pytest.raises(ValidationError, match="unknown evidence"): + ScientificReport.model_validate(invalid) + + +def test_pinned_gistemp_result_and_endpoint_sensitivity(): + series = load_gistemp_annual(DATA) + summary = trend_summary(series, bootstrap_replicates=200) + assert (series.years[0], series.years[-1], len(series.years)) == (1880, 2025, 146) + assert summary["ols_slope_c_per_decade"] > 0 + assert summary["ols_slope_95_ci_c_per_decade"][0] > 0 + assert summary["mann_kendall"]["p_value"] < 0.001 + assert summary["endpoint_falsification"]["period"] == [1880, 2015] + assert summary["endpoint_falsification"]["outcome"] == "not_falsified" + assert summary["endpoint_falsification"]["mann_kendall"]["p_value"] < 0.05 + + +def test_checked_in_json_and_svg_reproduce_exactly(): + json_bytes, svg_bytes = build_outputs() + assert (OUTPUT / "gistemp_trend_result.json").read_bytes() == json_bytes + assert (OUTPUT / "gistemp_global_trend.svg").read_bytes() == svg_bytes