ENG-8819 fix(network-ops): collect cracked passwords on time-boxed/partial hashcat and john runs

capabilities/network-ops/capability.yaml

@@ -1,6 +1,6 @@
 schema: 1
 name: network-ops
-version: "2.1.8"
+version: "2.1.9"
 description: >
   Network operations and Active Directory exploitation. Multi-agent
   pipeline for autonomous red teaming with Nmap scanning, Netexec

capabilities/network-ops/tests/test_tool_fixes.py

@@ -599,14 +599,10 @@ async def test_backend_error_suggests_john_with_known_mode(self, tmp_path):
         cracker = cracking_mod.Cracking()
 
         async def fake_execute(cmd, **kwargs):
-            raise RuntimeError(
-                "Command failed (255): clGetPlatformIDs(): CL_PLATFORM_NOT_FOUND_KHR"
-            )
+            raise RuntimeError("Command failed (255): clGetPlatformIDs(): CL_PLATFORM_NOT_FOUND_KHR")
 
         with patch.object(cracking_mod, "execute", side_effect=fake_execute):
-            with pytest.raises(
-                RuntimeError, match=r"Use john_the_ripper.*hash_format='nt'"
-            ):
+            with pytest.raises(RuntimeError, match=r"Use john_the_ripper.*hash_format='nt'"):
                 await cracker.hashcat(
                     hashcat_mode=1000,
                     hash_file=str(hash_file),
@@ -622,9 +618,7 @@ async def test_backend_error_chains_original_exception(self, tmp_path):
         wordlist.write_text("password\n")
 
         cracker = cracking_mod.Cracking()
-        original = RuntimeError(
-            "Command failed (255): clGetPlatformIDs(): CL_PLATFORM_NOT_FOUND_KHR"
-        )
+        original = RuntimeError("Command failed (255): clGetPlatformIDs(): CL_PLATFORM_NOT_FOUND_KHR")
 
         async def fake_execute(cmd, **kwargs):
             raise original
@@ -658,3 +652,197 @@ async def fake_execute(cmd, **kwargs):
                     hash_file=str(hash_file),
                     wordlist_path=str(wordlist),
                 )
+
+
+# ===================================================================
+# Cracking: results survive hashcat's non-zero / time-boxed exits
+# ===================================================================
+
+
+def _split_execute(crack_error: BaseException, show_output: str):
+    """Build an execute() side effect: the --show call returns `show_output`,
+    every other (crack) call raises `crack_error`. Records the commands seen."""
+    calls: list[list[str]] = []
+
+    async def fake_execute(cmd, **kwargs):
+        calls.append(cmd)
+        if "--show" in cmd:
+            return show_output
+        raise crack_error
+
+    return fake_execute, calls
+
+
+class TestHashcatResultsSurviveNonZeroExit:
+    """A time-boxed crack that exits non-zero must still return potfile results."""
+
+    def _write_inputs(self, tmp_path):
+        hash_file = tmp_path / "hashes.txt"
+        hash_file.write_text("aad3b435b51404eeaad3b435b51404ee\n")
+        wordlist = tmp_path / "wordlist.txt"
+        wordlist.write_text("Password1\n")
+        return hash_file, wordlist
+
+    @pytest.mark.asyncio
+    async def test_exhausted_exit_1_returns_recovered(self, tmp_path):
+        """Exit 1 (exhausted keyspace) with a partial recovery returns the creds."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, calls = _split_execute(
+            RuntimeError("Command failed (1):\nStatus: Exhausted\nRecovered: 1/2"),
+            "svc_sql:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            result = await cracking_mod.Cracking().hashcat(
+                hashcat_mode=1000,
+                hash_file=str(hash_file),
+                wordlist_path=str(wordlist),
+            )
+        assert result == "svc_sql:Password1"
+        assert any("--show" in cmd for cmd in calls)
+
+    @pytest.mark.asyncio
+    async def test_runtime_exit_4_returns_recovered(self, tmp_path):
+        """Exit 4 (halted by --runtime, the real code on current hashcat) returns creds."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, _ = _split_execute(
+            RuntimeError("Command failed (4):\nStatus: Aborted (Runtime)"),
+            "svc_sql:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            result = await cracking_mod.Cracking().hashcat(
+                hashcat_mode=1000,
+                hash_file=str(hash_file),
+                wordlist_path=str(wordlist),
+            )
+        assert result == "svc_sql:Password1"
+
+    @pytest.mark.asyncio
+    async def test_aborted_exit_2_returns_recovered(self, tmp_path):
+        """Exit 2 (aborted) still returns the creds."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, _ = _split_execute(
+            RuntimeError("Command failed (2):\nStatus: Aborted"),
+            "svc_sql:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            result = await cracking_mod.Cracking().hashcat(
+                hashcat_mode=1000,
+                hash_file=str(hash_file),
+                wordlist_path=str(wordlist),
+            )
+        assert result == "svc_sql:Password1"
+
+    @pytest.mark.asyncio
+    async def test_genuine_failure_surfaces_instead_of_empty(self, tmp_path):
+        """A non-backend fatal exit (255) must surface as an error, not an empty
+        '--show' result that reads as 'nothing cracked'."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        # Crack fails for a real reason; --show would exit 0 with nothing to report.
+        fake_execute, calls = _split_execute(
+            RuntimeError("Command failed (255):\nCUDA API call failed (cuInit)"),
+            "",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            with pytest.raises(RuntimeError, match=r"cuInit"):
+                await cracking_mod.Cracking().hashcat(
+                    hashcat_mode=1000,
+                    hash_file=str(hash_file),
+                    wordlist_path=str(wordlist),
+                )
+        assert not any("--show" in cmd for cmd in calls)
+
+    @pytest.mark.asyncio
+    async def test_error_without_parseable_code_surfaces(self, tmp_path):
+        """An error whose text carries no exit code is treated as genuine and surfaced."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, calls = _split_execute(
+            RuntimeError("hashcat process died unexpectedly"),
+            "",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            with pytest.raises(RuntimeError, match=r"died unexpectedly"):
+                await cracking_mod.Cracking().hashcat(
+                    hashcat_mode=1000,
+                    hash_file=str(hash_file),
+                    wordlist_path=str(wordlist),
+                )
+        assert not any("--show" in cmd for cmd in calls)
+
+    @pytest.mark.asyncio
+    async def test_timeout_kill_returns_recovered(self, tmp_path):
+        """A timeout that kills hashcat still collects potfile results via --show."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, _ = _split_execute(
+            TimeoutError("Command 'hashcat ...' timed out after 150 seconds."),
+            "svc_sql:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            result = await cracking_mod.Cracking().hashcat(
+                hashcat_mode=1000,
+                hash_file=str(hash_file),
+                wordlist_path=str(wordlist),
+            )
+        assert result == "svc_sql:Password1"
+
+    @pytest.mark.asyncio
+    async def test_no_backend_short_circuits_before_show(self, tmp_path):
+        """A missing GPU backend raises the john hint and never reaches --show."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, calls = _split_execute(
+            RuntimeError("Command failed (255): clGetPlatformIDs(): CL_PLATFORM_NOT_FOUND_KHR"),
+            "svc_sql:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            with pytest.raises(RuntimeError, match=r"Use john_the_ripper"):
+                await cracking_mod.Cracking().hashcat(
+                    hashcat_mode=1000,
+                    hash_file=str(hash_file),
+                    wordlist_path=str(wordlist),
+                )
+        assert not any("--show" in cmd for cmd in calls)
+
+
+class TestJohnResultsSurviveNonCleanExit:
+    """John is time-boxed by a timeout kill — results must survive that — but a
+    non-zero john exit is a genuine failure and must surface."""
+
+    def _write_inputs(self, tmp_path):
+        hash_file = tmp_path / "hashes.txt"
+        hash_file.write_text("$krb5asrep$23$user@REALM:abc...\n")
+        wordlist = tmp_path / "wordlist.txt"
+        wordlist.write_text("Password1\n")
+        return hash_file, wordlist
+
+    @pytest.mark.asyncio
+    async def test_timeout_kill_returns_recovered(self, tmp_path):
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, calls = _split_execute(
+            TimeoutError("Command 'john ...' timed out after 630 seconds."),
+            "user:Password1",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            result = await cracking_mod.Cracking().john_the_ripper(
+                hash_format="krb5asrep",
+                hash_file=str(hash_file),
+                wordlist_path=str(wordlist),
+            )
+        assert result == "user:Password1"
+        assert any("--show" in cmd for cmd in calls)
+
+    @pytest.mark.asyncio
+    async def test_nonzero_exit_surfaces_instead_of_empty(self, tmp_path):
+        """A john failure (e.g. unknown format) surfaces rather than returning an
+        empty, successful-looking result."""
+        hash_file, wordlist = self._write_inputs(tmp_path)
+        fake_execute, calls = _split_execute(
+            RuntimeError("Command failed (1):\nUnknown ciphertext format name"),
+            "",
+        )
+        with patch.object(cracking_mod, "execute", side_effect=fake_execute):
+            with pytest.raises(RuntimeError, match=r"Unknown ciphertext format"):
+                await cracking_mod.Cracking().john_the_ripper(
+                    hash_format="krb5asrep",
+                    hash_file=str(hash_file),
+                    wordlist_path=str(wordlist),
+                )
+        assert not any("--show" in cmd for cmd in calls)

capabilities/network-ops/tools/cracking.py

@@ -1,5 +1,6 @@
 import asyncio
 import os
+import re
 import tempfile
 
 from dreadnode import Config
@@ -18,6 +19,22 @@
 
 _NO_BACKEND_MARKERS = ("CL_PLATFORM_NOT_FOUND_KHR", "No devices found/left")
 
+# hashcat exit statuses that mean the run started and then stopped for an
+# expected reason: 1 = keyspace exhausted, 2/3 = aborted, 4 = halted by the
+# --runtime time box this tool always sets. In each the potfile may already
+# hold cracked hashes, so results are collected rather than treated as failure.
+# Anything else (255, negative) means hashcat could not run and is surfaced.
+_HASHCAT_RAN_EXIT_CODES = frozenset({1, 2, 3, 4})
+
+_EXECUTE_EXIT_CODE_RE = re.compile(r"Command failed \((-?\d+)\)")
+
+
+def _execute_exit_code(message: str) -> int | None:
+    """Recover the process exit code the platform execute() helper embeds in its
+    failure text ("Command failed (<code>): ..."), or None when it is absent."""
+    match = _EXECUTE_EXIT_CODE_RE.search(message)
+    return int(match.group(1)) if match else None
+
 
 class Cracking(Toolset):
     """
@@ -65,15 +82,17 @@ async def hashcat(
                 raise FileNotFoundError(f"Hash file {hash_file_path} does not exist.")
 
             if not os.path.exists(wordlist_path):
-                raise FileNotFoundError(
-                    f"Wordlist file {wordlist_path} does not exist."
-                )
+                raise FileNotFoundError(f"Wordlist file {wordlist_path} does not exist.")
 
-            logger.info(
-                f"Cracking {hash_file_path} with mode {hashcat_mode} using wordlist {wordlist_path}"
-            )
+            logger.info(f"Cracking {hash_file_path} with mode {hashcat_mode} using wordlist {wordlist_path}")
 
-            # Execute the cracking command
+            # Run the crack, then always read the potfile with --show below. hashcat
+            # records each recovered hash to the potfile as it goes and a time-boxed
+            # crack normally exits non-zero (keyspace exhausted, or halted by the
+            # --runtime box), so an expected non-zero exit, or a timeout that kills the
+            # process, must not abort before --show collects what landed. A crack that
+            # could not run (no compute backend, or any other genuine failure) is
+            # surfaced instead, so an empty result never masks a broken invocation.
             try:
                 await execute(
                     [
@@ -90,7 +109,9 @@ async def hashcat(
                     ],
                     timeout=(max_time_minutes * 60) + 30,
                 )
-            except Exception as e:
+            except TimeoutError:
+                logger.info("hashcat crack step timed out; collecting recovered hashes from " "the potfile via --show")
+            except RuntimeError as e:
                 msg = str(e)
                 if any(marker in msg for marker in _NO_BACKEND_MARKERS):
                     john_fmt = HASHCAT_TO_JOHN.get(hashcat_mode)
@@ -104,12 +125,14 @@ async def hashcat(
                         f"Use john_the_ripper instead (hashcat mode {hashcat_mode} "
                         f"has no automatic john format mapping — consult john documentation)."
                     ) from e
-                raise
+                if _execute_exit_code(msg) not in _HASHCAT_RAN_EXIT_CODES:
+                    raise
+                logger.info(
+                    "hashcat crack step exited non-zero (exhausted or aborted); "
+                    "collecting recovered hashes from the potfile via --show"
+                )
 
-            # Execute the --show command to get the cracked results
-            return await execute(
-                ["hashcat", "-m", str(hashcat_mode), hash_file_path, "--show"]
-            )
+            return await execute(["hashcat", "-m", str(hashcat_mode), hash_file_path, "--show"])
 
     @tool_method(catch=True, variants=["hashcat", "john", "all"])
     async def john_the_ripper(
@@ -150,23 +173,24 @@ async def john_the_ripper(
         if not os.path.exists(wordlist_path):
             raise FileNotFoundError(f"Wordlist file {wordlist_path} does not exist.")
 
-        logger.info(
-            f"Cracking {hash_file_path} with format {hash_format} using wordlist {wordlist_path}"
-        )
-
-        # Execute the cracking command
-        # Note: John's --max-run-time is not a standard feature, we rely on the timeout
-        await execute(
-            [
-                "john",
-                f"--wordlist={wordlist_path}",
-                f"--format={hash_format}",
-                hash_file_path,
-            ],
-            timeout=(max_time_minutes * 60) + 30,
-        )
-
-        # Execute the --show command to get the cracked results
-        return await execute(
-            ["john", "--show", f"--format={hash_format}", hash_file_path]
-        )
+        logger.info(f"Cracking {hash_file_path} with format {hash_format} using wordlist {wordlist_path}")
+
+        # Run the crack, then read the pot file with --show below. John writes each
+        # recovered password to its pot file as it goes and is time-boxed here by the
+        # run timeout, so a run cut short by the time box still has results to collect.
+        # A non-zero exit means john itself failed (bad format, unreadable hashes) and
+        # is left to surface rather than reported as an empty, successful result.
+        try:
+            await execute(
+                [
+                    "john",
+                    f"--wordlist={wordlist_path}",
+                    f"--format={hash_format}",
+                    hash_file_path,
+                ],
+                timeout=(max_time_minutes * 60) + 30,
+            )
+        except TimeoutError:
+            logger.info("john crack step timed out; collecting recovered hashes from the " "pot file via --show")
+
+        return await execute(["john", "--show", f"--format={hash_format}", hash_file_path])