From 098688d3679bd0bb40c20fc3ed4c35fd76a07022 Mon Sep 17 00:00:00 2001 From: lullah Date: Thu, 17 Sep 2026 09:41:24 +0100 Subject: [PATCH 1/2] Keep processing libraries out of the API --- requirements.in | 6 +-- requirements.txt | 6 +-- scripts/README.md | 24 +++++++++++ scripts/profile_web_imports.py | 71 ++++++++++++++++++++++++++++++++ stixify/classifier/utils.py | 7 ++-- tests/src/test_worker_imports.py | 37 ++++++++++++----- 6 files changed, 131 insertions(+), 20 deletions(-) create mode 100644 scripts/README.md create mode 100644 scripts/profile_web_imports.py diff --git a/requirements.in b/requirements.in index fbad8dd..32a11b8 100644 --- a/requirements.in +++ b/requirements.in @@ -17,9 +17,9 @@ xlrd>=2.0.2 playwright==1.58.0 ## dogesec -file2txt -txt2stix[llms]>=1.7.1 -dogesec_commons[stixifier] +file2txt>=1.1.0 +txt2stix[llms]>=1.7.2 +dogesec_commons[stixifier]>=1.5.1 stix2arango diff --git a/requirements.txt b/requirements.txt index 5746480..efbb323 100644 --- a/requirements.txt +++ b/requirements.txt @@ -136,13 +136,13 @@ djangorestframework==3.16.1 # drf-spectacular docstring-parser==0.17.0 # via anthropic -dogesec-commons[stixifier]==1.5.0 +dogesec-commons[stixifier]==1.5.1 # via -r requirements.in drf-spectacular==0.29.0 # via # -r requirements.in # dogesec-commons -file2txt==1.0.13 +file2txt==1.1.0 # via # -r requirements.in # dogesec-commons @@ -607,7 +607,7 @@ tqdm==4.67.3 # transformers transformers==4.57.6 # via llama-index-llms-openai-like -txt2stix[llms]==1.7.1 +txt2stix[llms]==1.7.2 # via # -r requirements.in # dogesec-commons diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..1e349a1 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,24 @@ +# Web import memory + +With the application's environment configured, run a fresh process: + +```sh +python scripts/profile_web_imports.py --skip-external-startup +``` + +This skips ArangoDB startup, then measures WSGI, URL loading, extractor metadata, +and AI configuration validation. Older dependency releases construct AI clients +during validation; use `--skip-profile-validation` for that baseline. + +The fix requires txt2stix 1.7.2, dogesec-commons 1.5.1 and file2txt 1.1.0. +Publish these dependency releases before building Stixify from the updated pins. +Local wheels can be used for verification before publication. + +In the same cached Linux image, route-load RSS fell from 406 MiB to 149 MiB +with the fixed dependency wheels. After metadata and model validation it was +151 MiB, with no LlamaIndex, Transformers, pandas, OpenAI or phone geography +modules loaded. Configuration validation does not check provider credentials; +processing workers still construct the actual clients. + +These are single-process measurements, not production container totals. +Gunicorn workers share some mappings; use PSS rather than summing their RSS. diff --git a/scripts/profile_web_imports.py b/scripts/profile_web_imports.py new file mode 100644 index 0000000..2f27b58 --- /dev/null +++ b/scripts/profile_web_imports.py @@ -0,0 +1,71 @@ +"""Measure a fresh web process, including URL loading on the first request. + +Run with the application's environment configured: + python scripts/profile_web_imports.py --skip-external-startup + +This is a single-process measurement, not total Gunicorn/container memory. +""" + +import argparse +from contextlib import nullcontext +import json +import os +from pathlib import Path +import resource +import sys +import time + + +def snapshot(stage, started): + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + result = { + "stage": stage, + "peak_rss_mib": round(peak / (1024 ** 2 if sys.platform == "darwin" else 1024), 2), + "elapsed_seconds": round(time.monotonic() - started, 2), + "module_count": len(sys.modules), + } + # Linux PSS accounts for shared mappings; do not sum worker RSS values. + rollup = Path("/proc/self/smaps_rollup") + if rollup.exists(): + for line in rollup.read_text().splitlines(): + key, _, value = line.partition(":") + if key in ("Rss", "Pss", "Private_Dirty", "Private_Clean"): + result[key.lower() + "_mib"] = round(int(value.split()[0]) / 1024, 2) + print(json.dumps(result), flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--skip-external-startup", action="store_true", + help="Skip the shared app's ArangoDB initialization hook.") + parser.add_argument("--skip-profile-validation", action="store_true", + help="Compare older releases whose model validation constructs AI clients.") + args = parser.parse_args() + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "stixify.settings") + started = time.monotonic() + snapshot("baseline", started) + from unittest.mock import patch + + context = patch("dogesec_commons.objects.db_view_creator.startup_func") if args.skip_external_startup else nullcontext() + with context: + from stixify.wsgi import application # noqa: F401 + snapshot("wsgi", started) + import stixify.urls # noqa: F401 + snapshot("urls", started) + from dogesec_commons.stixifier.serializers import validate_model, Txt2stixExtractorSerializer + + Txt2stixExtractorSerializer.all_extractors(("pattern", "lookup", "ai")) + if not args.skip_profile_validation: + for provider in ("openai", "anthropic", "gemini", "deepseek", "openrouter"): + validate_model(provider) + snapshot("profile_metadata_and_validation", started) + prefixes = ("txt2stix", "llama_index", "pandas", "transformers", "torch", "openai", "phonenumbers.geodata") + print(json.dumps({"loaded_modules": { + prefix: sum(name == prefix or name.startswith(prefix + ".") for name in sys.modules) + for prefix in prefixes + }}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/stixify/classifier/utils.py b/stixify/classifier/utils.py index b4eaca1..dc60a50 100644 --- a/stixify/classifier/utils.py +++ b/stixify/classifier/utils.py @@ -1,9 +1,6 @@ import os from typing import List -import openai - - from .models import DocumentEmbedding @@ -12,6 +9,8 @@ class ClusteringCancelled(Exception): def _openai_client(): + import openai + openai.api_key = os.getenv("OPENAI_API_KEY") return openai.Client() @@ -40,4 +39,4 @@ def create_embedding_text(*texts: List[str]) -> str: """Create a single string to embed from multiple text fields.""" # simple concat with separator, could be improved with field weighting or truncation texts = [t.strip() for t in texts if t and t.strip()] - return " | ".join(texts) \ No newline at end of file + return " | ".join(texts) diff --git a/tests/src/test_worker_imports.py b/tests/src/test_worker_imports.py index 416e5a9..8d297c8 100644 --- a/tests/src/test_worker_imports.py +++ b/tests/src/test_worker_imports.py @@ -17,7 +17,7 @@ def _run_in_clean_process(code): ) -def test_django_and_task_imports_do_not_load_worker_dependencies(): +def test_web_routes_and_profile_validation_do_not_load_worker_dependencies(): _run_in_clean_process( """ import django @@ -28,16 +28,33 @@ def test_django_and_task_imports_do_not_load_worker_dependencies(): django.setup() import stixify.worker.tasks + import stixify.wsgi + import stixify.urls + from django.test import Client + from dogesec_commons.stixifier.serializers import ( + validate_model, validate_extractor, uses_ai, Txt2stixExtractorSerializer, + ) - # assert not any( - # name == "txt2stix" or name.startswith("txt2stix.") - # for name in sys.modules - # ) - assert "stixify.worker.process_post" not in sys.modules - assert "stixify.worker.pdf_converter" not in sys.modules - assert "stixify.classifier.tasks" not in sys.modules - assert "sklearn.metrics.pairwise" not in sys.modules - assert "joblib" not in sys.modules + assert Client().get('/api/healthcheck/').status_code == 204 + assert Txt2stixExtractorSerializer.all_extractors(('pattern',)) + validate_extractor('extractor', ['pattern'], 'pattern_ipv4_address_only') + uses_ai(['pattern_ipv4_address_only']) + for provider in ('openai', 'anthropic', 'gemini', 'deepseek', 'openrouter'): + assert validate_model(provider) == provider + assert validate_model(provider + ':example-model') == provider + ':example-model' + + forbidden = ( + 'stixify.worker.process_post', 'stixify.worker.pdf_converter', + 'stixify.classifier.tasks', 'txt2stix.txt2stix', 'txt2stix.bundler', + 'txt2stix.indicator', 'txt2stix.pattern', 'llama_index', + 'transformers', 'torch', 'pandas', 'sklearn', 'joblib', + 'openai', 'anthropic', 'google.genai', 'phonenumbers.geodata', + ) + loaded = [ + name for name in sys.modules + if any(name == prefix or name.startswith(prefix + '.') for prefix in forbidden) + ] + assert not loaded, loaded """ ) From 73026ff9e26029bf880498efa7db276a47380aab Mon Sep 17 00:00:00 2001 From: lullah Date: Thu, 17 Sep 2026 09:54:46 +0100 Subject: [PATCH 2/2] Use lightweight txt2stix dependencies --- requirements.in | 2 +- requirements.txt | 4 ++-- scripts/README.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 32a11b8..4ddb40f 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,7 @@ playwright==1.58.0 ## dogesec file2txt>=1.1.0 -txt2stix[llms]>=1.7.2 +txt2stix[llms]>=1.7.3 dogesec_commons[stixifier]>=1.5.1 stix2arango diff --git a/requirements.txt b/requirements.txt index efbb323..9f46f66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -387,7 +387,7 @@ pathvalidate==3.3.1 # via txt2stix pgvector==0.4.2 # via -r requirements.in -phonenumbers==9.0.26 +phonenumberslite==9.0.26 # via txt2stix pillow==10.4.0 # via @@ -607,7 +607,7 @@ tqdm==4.67.3 # transformers transformers==4.57.6 # via llama-index-llms-openai-like -txt2stix[llms]==1.7.2 +txt2stix[llms]==1.7.3 # via # -r requirements.in # dogesec-commons diff --git a/scripts/README.md b/scripts/README.md index 1e349a1..031d892 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,7 +10,7 @@ This skips ArangoDB startup, then measures WSGI, URL loading, extractor metadata and AI configuration validation. Older dependency releases construct AI clients during validation; use `--skip-profile-validation` for that baseline. -The fix requires txt2stix 1.7.2, dogesec-commons 1.5.1 and file2txt 1.1.0. +The fix requires txt2stix 1.7.3, dogesec-commons 1.5.1 and file2txt 1.1.0. Publish these dependency releases before building Stixify from the updated pins. Local wheels can be used for verification before publication.