diff --git a/lib/arcada/register.ex b/lib/arcada/register.ex index 2f752a4..16d9e92 100644 --- a/lib/arcada/register.ex +++ b/lib/arcada/register.ex @@ -361,12 +361,15 @@ defmodule Arcada.Register do newest first, what `Arcada.Summarizer.SummarySweeper` drains each tick. Backlog acts and any act whose summary never landed (a failed daily job) both surface here, so the sweeper keeps retrying until all are summarized. + + Acts without `full_text` are left out of both: the summarizer refuses them + (it would only see the title), so they wait until a re-ingest enriches them. """ def acts_without_summary(%Date{} = date) do from(a in Act, join: e in assoc(a, :edition), left_join: s in assoc(a, :summaries), - where: e.date == ^date and is_nil(s.id) + where: e.date == ^date and is_nil(s.id) and not is_nil(a.full_text) ) |> Repo.all() end @@ -374,7 +377,7 @@ defmodule Arcada.Register do def acts_without_summary(limit) when is_integer(limit) do from(a in Act, left_join: s in assoc(a, :summaries), - where: is_nil(s.id), + where: is_nil(s.id) and not is_nil(a.full_text), order_by: [desc: a.id], limit: ^limit ) diff --git a/lib/arcada/scraper/api_version_resolver.ex b/lib/arcada/scraper/api_version_resolver.ex index de7fbaf..f4eebab 100644 --- a/lib/arcada/scraper/api_version_resolver.ex +++ b/lib/arcada/scraper/api_version_resolver.ex @@ -11,8 +11,11 @@ defmodule Arcada.Scraper.ApiVersionResolver do 1. `GET /dr/moduleservices/moduleinfo` → `manifest.urlVersions`, a map of every asset path (incl. each screen's `*.mvc.js`) to its `?` suffix. 2. `GET /dr/scripts/..mvc.js?` → the screen bundle. - 3. Extract the 3rd `callDataAction("", "", "", …)` - argument for the action we care about. + 3. Extract the `apiVersion` argument of `callDataAction(...)` for the + action we care about: the string right after the `screenservices/...` + path. Its position isn't fixed — DRE's late-September 2026 deploy added + a GUID before the path (`callDataAction("", "", + "", "", …)`), so anchor on the path, not the index. Used by `Arcada.Scraper.Client` to self-heal: when a data-action reports `versionInfo.hasApiVersionChanged: true`, we re-resolve, swap in the fresh @@ -67,8 +70,8 @@ defmodule Arcada.Scraper.ApiVersionResolver do end end - # callDataAction("", "", "", …) + # callDataAction("", [other string args,] "screenservices/…", "", …) defp version_regex(action) do - ~r/callDataAction\(\s*"#{Regex.escape(action)}"\s*,\s*"[^"]*"\s*,\s*"([^"]+)"/ + ~r/callDataAction\(\s*"#{Regex.escape(action)}"\s*,(?:\s*"[^"]*"\s*,)*?\s*"screenservices\/[^"]*"\s*,\s*"([^"]+)"/ end end diff --git a/lib/arcada/summarizer.ex b/lib/arcada/summarizer.ex index a584ce7..28f8c8d 100644 --- a/lib/arcada/summarizer.ex +++ b/lib/arcada/summarizer.ex @@ -148,9 +148,22 @@ defmodule Arcada.Summarizer do only talks to its backend — the cap/ranking decision lives in one place. """ def summarize(%Act{} = act, %Provider{} = provider, model, opts \\ []) do + case PlainText.from_html(act.full_text) do + clean when is_binary(clean) and clean != "" -> + summarize_text(act, provider, model, clean, opts) + + _ -> + # Without the act's text the model only sees its title ("Decreto-Lei + # n.º 200/2026") and invents a plausible summary. Refuse instead; the act + # stays unsummarized until a re-ingest fills `full_text`. + Logger.warning("summarizer: act #{act.id} has no full_text, not summarizing") + {:error, :no_full_text} + end + end + + defp summarize_text(act, provider, model, clean, opts) do model = model || List.first(provider.models) requested = opts |> Keyword.get(:text_strategy, :auto) |> normalize_strategy() - clean = PlainText.from_html(act.full_text) || act.title if requested == :auto and oversized?(clean, model) and extractor_configured?() do # Omnibus act + an extractor configured: the strong model lists the concrete diff --git a/lib/arcada/summarizer/summarize_worker.ex b/lib/arcada/summarizer/summarize_worker.ex index 92312cc..530f34c 100644 --- a/lib/arcada/summarizer/summarize_worker.ex +++ b/lib/arcada/summarizer/summarize_worker.ex @@ -61,5 +61,7 @@ defmodule Arcada.Summarizer.SummarizeWorker do defp handle({:ok, _summary}), do: :ok defp handle({:async, _ref}), do: :ok + # Retrying can't help until a re-ingest fills the act's text. + defp handle({:error, :no_full_text}), do: {:cancel, :no_full_text} defp handle({:error, reason}), do: {:error, reason} end diff --git a/test/arcada/register_test.exs b/test/arcada/register_test.exs index c2c60ae..853779c 100644 --- a/test/arcada/register_test.exs +++ b/test/arcada/register_test.exs @@ -150,7 +150,7 @@ defmodule Arcada.RegisterTest do end defp bare_act(edition, n) do - insert_act(edition, %{dre_id: "sweep-#{n}", title: "Act #{n}"}) + insert_act(edition, %{dre_id: "sweep-#{n}", title: "Act #{n}", full_text: "Texto."}) end defp summarized_act(edition, n) do @@ -174,6 +174,13 @@ defmodule Arcada.RegisterTest do assert Enum.map(Arcada.Register.acts_without_summary(2), & &1.id) == [c.id, b.id] end + test "skips acts with no full_text", %{edition: edition} do + with_text = bare_act(edition, 1) + _no_text = insert_act(edition, %{dre_id: "sweep-2", title: "Act 2"}) + + assert Enum.map(Arcada.Register.acts_without_summary(100), & &1.id) == [with_text.id] + end + test "empty when every act is summarized", %{edition: edition} do summarized_act(edition, 1) assert Arcada.Register.acts_without_summary(100) == [] diff --git a/test/arcada/scraper/api_version_resolver_test.exs b/test/arcada/scraper/api_version_resolver_test.exs index b78a4a6..ad61d63 100644 --- a/test/arcada/scraper/api_version_resolver_test.exs +++ b/test/arcada/scraper/api_version_resolver_test.exs @@ -52,6 +52,21 @@ defmodule Arcada.Scraper.ApiVersionResolverTest do assert {:ok, %{detail: "DET-42"}} = ApiVersionResolver.resolve(client(), [:detail]) end + test "reads the apiVersion after the path when a GUID precedes it" do + # DRE's late-September 2026 bundles: callDataAction(name, guid, path, apiVersion, cb). + js = + ~s|callDataAction("DataActionGetAllConteudoDetalheData", "0f36ef54-97a2-4b72-a7b7-7d5b0a5401cb", "screenservices/dr/Legislacao_Conteudos/Conteudo_Detalhe/DataActionGetAllConteudoDetalheData", "B3l5BYZxaLmWUxGR3tmKGw", function (b) {})| + + Req.Test.stub(__MODULE__, fn conn -> + if String.contains?(conn.request_path, "moduleservices/moduleinfo"), + do: Req.Test.json(conn, manifest()), + else: Plug.Conn.send_resp(conn, 200, js) + end) + + assert {:ok, %{detail: "B3l5BYZxaLmWUxGR3tmKGw"}} = + ApiVersionResolver.resolve(client(), [:detail]) + end + test "returns nil for an action whose script is missing from the manifest" do Req.Test.stub(__MODULE__, fn conn -> # Manifest present but with no urlVersions entries → script path unresolved. diff --git a/test/arcada/scraper/ingest_worker_test.exs b/test/arcada/scraper/ingest_worker_test.exs index 8376c3a..241b8f7 100644 --- a/test/arcada/scraper/ingest_worker_test.exs +++ b/test/arcada/scraper/ingest_worker_test.exs @@ -36,7 +36,7 @@ defmodule Arcada.Scraper.IngestWorkerTest do Req.Test.stub(Arcada.IngestStub, fn conn -> if String.contains?(conn.request_path, "WB_Serie1_List"), do: Req.Test.json(conn, fixture), - else: Req.Test.json(conn, %{}) + else: Req.Test.json(conn, %{"data" => %{"Detalhe" => %{"Texto" => "Texto do ato."}}}) end) client = %{ @@ -47,8 +47,8 @@ defmodule Arcada.Scraper.IngestWorkerTest do } prev_client = Application.get_env(:arcada, :ingest_client) - # enrich: false — the stub only serves the list call; skip per-act detail. - Application.put_env(:arcada, :ingest_client, %{client | detail_api_version: nil}) + # The stub's detail answer gives every act text, so each one can be summarized. + Application.put_env(:arcada, :ingest_client, client) on_exit(fn -> if prev_client, diff --git a/test/arcada/summarizer/summary_sweeper_test.exs b/test/arcada/summarizer/summary_sweeper_test.exs index 799d2fc..d3acc44 100644 --- a/test/arcada/summarizer/summary_sweeper_test.exs +++ b/test/arcada/summarizer/summary_sweeper_test.exs @@ -31,7 +31,12 @@ defmodule Arcada.Summarizer.SummarySweeperTest do defp bare_act(edition, n) do %Act{} - |> Act.changeset(%{edition_id: edition.id, dre_id: "sw-#{n}", title: "Act #{n}"}) + |> Act.changeset(%{ + edition_id: edition.id, + dre_id: "sw-#{n}", + title: "Act #{n}", + full_text: "Texto." + }) |> Repo.insert!() end diff --git a/test/arcada/summarizer_test.exs b/test/arcada/summarizer_test.exs index c00ab12..f47ce79 100644 --- a/test/arcada/summarizer_test.exs +++ b/test/arcada/summarizer_test.exs @@ -25,7 +25,8 @@ defmodule Arcada.SummarizerTest do |> Act.changeset(%{ edition_id: edition.id, dre_id: "1138160247", - title: "Decreto-Lei n.º 1/2026" + title: "Decreto-Lei n.º 1/2026", + full_text: "O presente decreto-lei altera o Código do IRS." }) |> Repo.insert!() end @@ -49,6 +50,15 @@ defmodule Arcada.SummarizerTest do end describe "summarize/3 (explicit provider+model)" do + test "refuses an act with no full_text instead of summarizing its title" do + stub_ssh_runner(fn _ -> flunk("the model must not be called") end) + + act = act_fixture() |> Act.changeset(%{full_text: nil}) |> Repo.update!() + + assert {:error, :no_full_text} = Summarizer.summarize(act, ssh_provider(), "claude-cli") + assert Repo.aggregate(Summary, :count) == 0 + end + test "persists the result linked to the provider" do stub_ssh_runner(fn _ -> {:ok,