Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion backend/cortex_backend/execution/artifact_boundary.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,15 @@ def _publish_bytes(
mime_type=mime_type,
retention_seconds=retention_seconds,
)
except ExecutionRepositoryError:
except (ExecutionRepositoryError, OSError):
# OSError as well, because publish_artifact re-raises it: its own
# handler removes the partial files and then `raise`s the original,
# so a full disk, a permission error or an antivirus lock arrives
# here unwrapped. Without this the exception escaped the boundary
# entirely -- the caller's `except ArtifactBoundaryError` missed
# it, so the staging job was never failed and stayed non-terminal
# for good, and the route answered 500 instead of a stable code.
# publish_outputs below already treats both the same way.
raise ArtifactBoundaryError("artifact_publish_failed") from None

def stage_bytes(
Expand Down
32 changes: 32 additions & 0 deletions tests/test_attachment_staging.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,3 +128,35 @@ def test_stage_bytes_duplicate_terminal_result_is_revalidated(tmp_path: Path):
with pytest.raises(AttachmentStagingError) as error:
service.stage(owner=OWNER, request_id="attach-integrity", content=_image_bytes())
assert error.value.code in {"attachment_artifact_unavailable", "attachment_artifact_invalid"}


def test_a_disk_failure_fails_the_job_instead_of_escaping(tmp_path: Path, monkeypatch):
"""A full disk must produce a stable code and a terminal job.

`publish_artifact` removes its partial files and then re-raises the
original exception, so an OSError -- a full disk, a permission error, an
antivirus lock -- reached `_publish_bytes` unwrapped. That method caught
only `ExecutionRepositoryError`, so the exception escaped the boundary
entirely: the caller's `except ArtifactBoundaryError` missed it, `_fail`
never ran, the job stayed non-terminal for the rest of the installation's
life, and the route answered HTTP 500 rather than a stable code.

`publish_outputs` in the same class already treats both the same way.
"""
repository, service = _service(tmp_path)

def full_disk(*args, **kwargs):
raise OSError(28, "No space left on device")

monkeypatch.setattr(ExecutionRepository, "publish_artifact", full_disk)

with pytest.raises(AttachmentStagingError):
service.stage(owner=OWNER, request_id="attach-disk-full", content=_image_bytes())

with repository.connect() as connection:
statuses = [
str(row["status"])
for row in connection.execute("SELECT status FROM execution_jobs").fetchall()
]

assert statuses == ["failed"], "the staging job was left non-terminal"