Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions lib/arcada/register.ex
Original file line number Diff line number Diff line change
Expand Up @@ -361,20 +361,23 @@ defmodule Arcada.Register do
newest first, what `Arcada.Summarizer.SummarySweeper` drains each tick.
Backlog acts and any act whose summary never landed (a failed daily job)
both surface here, so the sweeper keeps retrying until all are summarized.

Acts without `full_text` are left out of both: the summarizer refuses them
(it would only see the title), so they wait until a re-ingest enriches them.
"""
def acts_without_summary(%Date{} = date) do
from(a in Act,
join: e in assoc(a, :edition),
left_join: s in assoc(a, :summaries),
where: e.date == ^date and is_nil(s.id)
where: e.date == ^date and is_nil(s.id) and not is_nil(a.full_text)
)
|> Repo.all()
end

def acts_without_summary(limit) when is_integer(limit) do
from(a in Act,
left_join: s in assoc(a, :summaries),
where: is_nil(s.id),
where: is_nil(s.id) and not is_nil(a.full_text),
order_by: [desc: a.id],
limit: ^limit
)
Expand Down
11 changes: 7 additions & 4 deletions lib/arcada/scraper/api_version_resolver.ex
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@ defmodule Arcada.Scraper.ApiVersionResolver do
1. `GET /dr/moduleservices/moduleinfo` → `manifest.urlVersions`, a map of
every asset path (incl. each screen's `*.mvc.js`) to its `?<hash>` suffix.
2. `GET /dr/scripts/<Module>.<Screen>.mvc.js?<hash>` → the screen bundle.
3. Extract the 3rd `callDataAction("<Action>", "<path>", "<apiVersion>", …)`
argument for the action we care about.
3. Extract the `apiVersion` argument of `callDataAction(...)` for the
action we care about: the string right after the `screenservices/...`
path. Its position isn't fixed — DRE's late-September 2026 deploy added
a GUID before the path (`callDataAction("<Action>", "<guid>",
"<path>", "<apiVersion>", …)`), so anchor on the path, not the index.

Used by `Arcada.Scraper.Client` to self-heal: when a data-action reports
`versionInfo.hasApiVersionChanged: true`, we re-resolve, swap in the fresh
Expand Down Expand Up @@ -67,8 +70,8 @@ defmodule Arcada.Scraper.ApiVersionResolver do
end
end

# callDataAction("<Action>", "<screenservices path>", "<apiVersion>", …)
# callDataAction("<Action>", [other string args,] "screenservices/…", "<apiVersion>", …)
defp version_regex(action) do
~r/callDataAction\(\s*"#{Regex.escape(action)}"\s*,\s*"[^"]*"\s*,\s*"([^"]+)"/
~r/callDataAction\(\s*"#{Regex.escape(action)}"\s*,(?:\s*"[^"]*"\s*,)*?\s*"screenservices\/[^"]*"\s*,\s*"([^"]+)"/
end
end
15 changes: 14 additions & 1 deletion lib/arcada/summarizer.ex
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,22 @@ defmodule Arcada.Summarizer do
only talks to its backend — the cap/ranking decision lives in one place.
"""
def summarize(%Act{} = act, %Provider{} = provider, model, opts \\ []) do
case PlainText.from_html(act.full_text) do
clean when is_binary(clean) and clean != "" ->
summarize_text(act, provider, model, clean, opts)

_ ->
# Without the act's text the model only sees its title ("Decreto-Lei
# n.º 200/2026") and invents a plausible summary. Refuse instead; the act
# stays unsummarized until a re-ingest fills `full_text`.
Logger.warning("summarizer: act #{act.id} has no full_text, not summarizing")
{:error, :no_full_text}
end
end

defp summarize_text(act, provider, model, clean, opts) do
model = model || List.first(provider.models)
requested = opts |> Keyword.get(:text_strategy, :auto) |> normalize_strategy()
clean = PlainText.from_html(act.full_text) || act.title

if requested == :auto and oversized?(clean, model) and extractor_configured?() do
# Omnibus act + an extractor configured: the strong model lists the concrete
Expand Down
2 changes: 2 additions & 0 deletions lib/arcada/summarizer/summarize_worker.ex
Original file line number Diff line number Diff line change
Expand Up @@ -61,5 +61,7 @@ defmodule Arcada.Summarizer.SummarizeWorker do

defp handle({:ok, _summary}), do: :ok
defp handle({:async, _ref}), do: :ok
# Retrying can't help until a re-ingest fills the act's text.
defp handle({:error, :no_full_text}), do: {:cancel, :no_full_text}
defp handle({:error, reason}), do: {:error, reason}
end
9 changes: 8 additions & 1 deletion test/arcada/register_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ defmodule Arcada.RegisterTest do
end

defp bare_act(edition, n) do
insert_act(edition, %{dre_id: "sweep-#{n}", title: "Act #{n}"})
insert_act(edition, %{dre_id: "sweep-#{n}", title: "Act #{n}", full_text: "Texto."})
end

defp summarized_act(edition, n) do
Expand All @@ -174,6 +174,13 @@ defmodule Arcada.RegisterTest do
assert Enum.map(Arcada.Register.acts_without_summary(2), & &1.id) == [c.id, b.id]
end

test "skips acts with no full_text", %{edition: edition} do
with_text = bare_act(edition, 1)
_no_text = insert_act(edition, %{dre_id: "sweep-2", title: "Act 2"})

assert Enum.map(Arcada.Register.acts_without_summary(100), & &1.id) == [with_text.id]
end

test "empty when every act is summarized", %{edition: edition} do
summarized_act(edition, 1)
assert Arcada.Register.acts_without_summary(100) == []
Expand Down
15 changes: 15 additions & 0 deletions test/arcada/scraper/api_version_resolver_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,21 @@ defmodule Arcada.Scraper.ApiVersionResolverTest do
assert {:ok, %{detail: "DET-42"}} = ApiVersionResolver.resolve(client(), [:detail])
end

test "reads the apiVersion after the path when a GUID precedes it" do
# DRE's late-September 2026 bundles: callDataAction(name, guid, path, apiVersion, cb).
js =
~s|callDataAction("DataActionGetAllConteudoDetalheData", "0f36ef54-97a2-4b72-a7b7-7d5b0a5401cb", "screenservices/dr/Legislacao_Conteudos/Conteudo_Detalhe/DataActionGetAllConteudoDetalheData", "B3l5BYZxaLmWUxGR3tmKGw", function (b) {})|

Req.Test.stub(__MODULE__, fn conn ->
if String.contains?(conn.request_path, "moduleservices/moduleinfo"),
do: Req.Test.json(conn, manifest()),
else: Plug.Conn.send_resp(conn, 200, js)
end)

assert {:ok, %{detail: "B3l5BYZxaLmWUxGR3tmKGw"}} =
ApiVersionResolver.resolve(client(), [:detail])
end

test "returns nil for an action whose script is missing from the manifest" do
Req.Test.stub(__MODULE__, fn conn ->
# Manifest present but with no urlVersions entries → script path unresolved.
Expand Down
6 changes: 3 additions & 3 deletions test/arcada/scraper/ingest_worker_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ defmodule Arcada.Scraper.IngestWorkerTest do
Req.Test.stub(Arcada.IngestStub, fn conn ->
if String.contains?(conn.request_path, "WB_Serie1_List"),
do: Req.Test.json(conn, fixture),
else: Req.Test.json(conn, %{})
else: Req.Test.json(conn, %{"data" => %{"Detalhe" => %{"Texto" => "Texto do ato."}}})
end)

client = %{
Expand All @@ -47,8 +47,8 @@ defmodule Arcada.Scraper.IngestWorkerTest do
}

prev_client = Application.get_env(:arcada, :ingest_client)
# enrich: false — the stub only serves the list call; skip per-act detail.
Application.put_env(:arcada, :ingest_client, %{client | detail_api_version: nil})
# The stub's detail answer gives every act text, so each one can be summarized.
Application.put_env(:arcada, :ingest_client, client)

on_exit(fn ->
if prev_client,
Expand Down
7 changes: 6 additions & 1 deletion test/arcada/summarizer/summary_sweeper_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ defmodule Arcada.Summarizer.SummarySweeperTest do

defp bare_act(edition, n) do
%Act{}
|> Act.changeset(%{edition_id: edition.id, dre_id: "sw-#{n}", title: "Act #{n}"})
|> Act.changeset(%{
edition_id: edition.id,
dre_id: "sw-#{n}",
title: "Act #{n}",
full_text: "Texto."
})
|> Repo.insert!()
end

Expand Down
12 changes: 11 additions & 1 deletion test/arcada/summarizer_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ defmodule Arcada.SummarizerTest do
|> Act.changeset(%{
edition_id: edition.id,
dre_id: "1138160247",
title: "Decreto-Lei n.º 1/2026"
title: "Decreto-Lei n.º 1/2026",
full_text: "O presente decreto-lei altera o Código do IRS."
})
|> Repo.insert!()
end
Expand All @@ -49,6 +50,15 @@ defmodule Arcada.SummarizerTest do
end

describe "summarize/3 (explicit provider+model)" do
test "refuses an act with no full_text instead of summarizing its title" do
stub_ssh_runner(fn _ -> flunk("the model must not be called") end)

act = act_fixture() |> Act.changeset(%{full_text: nil}) |> Repo.update!()

assert {:error, :no_full_text} = Summarizer.summarize(act, ssh_provider(), "claude-cli")
assert Repo.aggregate(Summary, :count) == 0
end

test "persists the result linked to the provider" do
stub_ssh_runner(fn _ ->
{:ok,
Expand Down
Loading