Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/adr/005-support-ensure.md
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,11 @@ clean verrebbe considerato mancante). `flatten_support_template_ctx` espone:

| Placeholder | Risolve a |
|---|---|
| `{support.NAME.clean}` | parquet clean del support (per anno) |
| `{support.NAME.clean}` | **dataset**: parquet clean (glob locale). **external multi-anno**: lista URL risolti (array SQL DuckDB). **external single-anno**: URL singolo. |
| `{support.NAME.mart}` | prima tabella mart (backward compat) |
| `{support.NAME.mart.TABLE}` | tabella mart specifica (il caso `mart_codici_catastali`) |
| `{support.NAME.outputs}` | lista completa (invariato) |
| `{support.NAME.path}` | file materializzato (codelist/file) o URI GCS (external) |
| `{support.NAME.path}` | file materializzato (codelist/file) o URI GCS (external, template `{year}`) |

`check_support_path_drift` esteso ai nuovi placeholder
(`clean`, `mart.TABLE`, `path`).
Expand Down
2 changes: 1 addition & 1 deletion docs/config-schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ blocca il candidate. L'esecuzione di un `command` file richiede
|---|---|
| `{support.NAME.mart}` | prima tabella mart del support (compat) |
| `{support.NAME.mart.TABLE}` | tabella mart specifica |
| `{support.NAME.clean}` | parquet clean del support |
| `{support.NAME.clean}` | **dataset**: parquet clean (glob locale). **external multi-anno**: lista URL (array SQL DuckDB). **external single-anno**: URL singolo. |
| `{support.NAME.path}` | file materializzato (codelist/file) o URI GCS (external) |
| `{support.NAME.outputs}` | lista completa degli output |

Expand Down
3 changes: 2 additions & 1 deletion docs/conventions.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ Versioni schema stabili:
referenziano nel SQL **solo** con i placeholder `{support.NAME.*}` — mai con path
hardcoded (il drift è segnalato come warning in dry-run da `check_support_path_drift`).
- Placeholder disponibili: `{support.NAME.mart}` (prima tabella), `{support.NAME.mart.TABLE}`,
`{support.NAME.clean}`, `{support.NAME.path}` (codelist/file), `{support.NAME.outputs}`.
`{support.NAME.clean}` (dataset: glob locale; external multi-anno: lista URL come array SQL),
`{support.NAME.path}` (codelist/file/external URI), `{support.NAME.outputs}`.
- Orchestrazione: i support sono eseguiti prima del candidate e riusati se gli output
(clean+mart per dataset; parquet canonico per codelist; path per file) sono già presenti.
Rigenerazione forzata: `toolkit run --refresh-support`.
Expand Down
8 changes: 6 additions & 2 deletions tests/test_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ def test_single_support_multiple_years(self, tmp_path: Path):
assert yr0["year"] == 2023
yr1 = payload["years_resolved"][1]
assert yr1["year"] == 2024
# type: dataset multi-anno → clean resta stringa (glob locale), non lista
assert isinstance(payload["clean"], str)

def test_multiple_support_datasets(self, tmp_path: Path):
config_a = _make_support_dataset(tmp_path, name="support_a", create_mart_outputs=True)
Expand Down Expand Up @@ -635,8 +637,10 @@ def test_multi_year_uri(self):
assert "2026" in p["outputs"][2]
# path remains the template URI
assert "{year}" in p["path"]
# clean is a glob
assert "*" in p["clean"]
# clean = lista URL risolti (DuckDB read_parquet accetta liste, non glob su HTTP)
assert isinstance(p["clean"], list)
assert len(p["clean"]) == 3
assert "2024" in p["clean"][0]

def test_multi_year_bucket_pattern(self):
entry = {
Expand Down
37 changes: 37 additions & 0 deletions tests/test_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,43 @@ def test_unresolved_in_code_still_raises():
render_template(sql, {"year": 2024})


def test_render_template_list_becomes_sql_array():
"""Lista nel ctx → array SQL DuckDB senza virgolette esterne."""
sql = "SELECT * FROM read_parquet({support.ext.clean}, union_by_name=true)"
ctx = {"support.ext.clean": ["https://a/2024.parquet", "https://a/2025.parquet"]}
result = render_template(sql, ctx)
expected = (
"read_parquet(['https://a/2024.parquet', 'https://a/2025.parquet'], union_by_name=true)"
)
assert expected in result
# nessuna virgolette esterne attorno all'array
assert "read_parquet('[" not in result


def test_render_template_empty_list():
"""Lista vuota → array SQL vuoto."""
sql = "SELECT * FROM read_parquet({support.ext.clean})"
result = render_template(sql, {"support.ext.clean": []})
assert "read_parquet([])" in result


def test_render_template_string_unchanged():
"""Stringa nel ctx → invariata (backward compat)."""
sql = "SELECT * FROM read_parquet('{support.ext.path}')"
ctx = {"support.ext.path": "https://a/2024.parquet"}
result = render_template(sql, ctx)
assert "read_parquet('https://a/2024.parquet')" in result


def test_render_template_dataset_multi_year_clean_stays_string():
"""type: dataset multi-anno → clean resta stringa (glob locale), non lista."""
sql = "SELECT * FROM read_parquet('{support.ds.clean}')"
# dataset type produce clean come stringa glob locale
ctx = {"support.ds.clean": "/out/data/clean/demo/*/demo_*_clean.parquet"}
result = render_template(sql, ctx)
assert "read_parquet('/out/data/clean/demo/*/demo_*_clean.parquet')" in result


def test_strip_sql_comments_empty():
assert _strip_sql_comments("") == ""

Expand Down
20 changes: 7 additions & 13 deletions toolkit/core/support.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,11 +241,12 @@ def _resolve_external_entry(
- ``bucket``: "clean" or "mart" + ``pattern``: pattern key + ``slug``
- ``table``: optional, for mart patterns
- ``years``: optional list of years. When provided with a ``{year}``
URI, resolves one path per year and builds a clean glob.
URI, resolves one path per year.

Resolution logic:
- ``uri`` with ``{year}`` + ``years`` → resolved per year, ``path``
is the URI template for the first year, ``clean`` is a glob.
is the URI template, ``clean`` is a list of resolved URLs
(DuckDB read_parquet accepts lists; globs don't work over HTTP).
- ``uri`` without ``{year}`` → single file, ``years`` is ignored.
- ``bucket``+``pattern``+``slug`` → uses ``gs_url()``, year from
``years[0]`` or omitted.
Expand Down Expand Up @@ -283,15 +284,8 @@ def _resolve_external_entry(
# Multi-year: resolve per year when uri contains {year}
if has_year_placeholder and years:
resolved_uris = [uri.replace("{year}", str(y)) for y in years]
# clean = glob pattern covering all years
# Use string ops (not Path) to preserve https:// double slash
first = resolved_uris[0]
last_slash = first.rfind("/")
second_last_slash = first.rfind("/", 0, last_slash)
base = first[:second_last_slash] if second_last_slash > 0 else first
filename = first[last_slash + 1 :]
slug = filename.rsplit("_", 2)[0]
clean_glob = f"{base}/*/{slug}_*_clean.parquet"
# clean = lista URL risolti (DuckDB read_parquet accetta liste; i glob
# non funzionano su HTTP → lista è l'unico formato sicuro per external)
return {
"name": name,
"type": "external",
Expand All @@ -304,8 +298,8 @@ def _resolve_external_entry(
"all_outputs_exist": True,
"mart": None,
"mart_by_table": {},
"clean": clean_glob,
"path": uri, # template URI for {support.X.path} in SQL
"clean": resolved_uris, # lista URL per anno (read_parquet accetta liste)
"path": uri, # template URI per {support.X.path} (single-anno)
}

# Single file (no {year} in uri, or no years configured)
Expand Down
8 changes: 7 additions & 1 deletion toolkit/core/template.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,13 @@ def render_template(text: str, ctx: dict[str, Any]) -> str:
"""
out = text
for k, v in sorted(ctx.items(), key=lambda item: len(item[0]), reverse=True):
out = out.replace("{" + k + "}", str(v))
if isinstance(v, list):
# Lista support multi-anno external → array SQL DuckDB
# es. ['url1', 'url2'] → read_parquet(['url1', 'url2'])
formatted = "[" + ", ".join(f"'{str(item)}'" for item in v) + "]"
out = out.replace("{" + k + "}", formatted)
else:
out = out.replace("{" + k + "}", str(v))
# Strip comments before checking for unresolved placeholders, so
# that {n} or other patterns in DuckDB error messages don't break.
code_only = _strip_sql_comments(out)
Expand Down
Loading