Reduce API memory usage

requirements.in

@@ -17,9 +17,9 @@ xlrd>=2.0.2
 playwright==1.58.0
 
 ## dogesec
-file2txt
-txt2stix[llms]>=1.7.1
-dogesec_commons[stixifier]
+file2txt>=1.1.0
+txt2stix[llms]>=1.7.3
+dogesec_commons[stixifier]>=1.5.1
 stix2arango
 
 

requirements.txt

@@ -136,13 +136,13 @@ djangorestframework==3.16.1
     #   drf-spectacular
 docstring-parser==0.17.0
     # via anthropic
-dogesec-commons[stixifier]==1.5.0
+dogesec-commons[stixifier]==1.5.1
     # via -r requirements.in
 drf-spectacular==0.29.0
     # via
     #   -r requirements.in
     #   dogesec-commons
-file2txt==1.0.13
+file2txt==1.1.0
     # via
     #   -r requirements.in
     #   dogesec-commons
@@ -387,7 +387,7 @@ pathvalidate==3.3.1
     # via txt2stix
 pgvector==0.4.2
     # via -r requirements.in
-phonenumbers==9.0.26
+phonenumberslite==9.0.26
     # via txt2stix
 pillow==10.4.0
     # via
@@ -607,7 +607,7 @@ tqdm==4.67.3
     #   transformers
 transformers==4.57.6
     # via llama-index-llms-openai-like
-txt2stix[llms]==1.7.1
+txt2stix[llms]==1.7.3
     # via
     #   -r requirements.in
     #   dogesec-commons

scripts/README.md

@@ -0,0 +1,24 @@
+# Web import memory
+
+With the application's environment configured, run a fresh process:
+
+```sh
+python scripts/profile_web_imports.py --skip-external-startup
+```
+
+This skips ArangoDB startup, then measures WSGI, URL loading, extractor metadata,
+and AI configuration validation. Older dependency releases construct AI clients
+during validation; use `--skip-profile-validation` for that baseline.
+
+The fix requires txt2stix 1.7.3, dogesec-commons 1.5.1 and file2txt 1.1.0.
+Publish these dependency releases before building Stixify from the updated pins.
+Local wheels can be used for verification before publication.
+
+In the same cached Linux image, route-load RSS fell from 406 MiB to 149 MiB
+with the fixed dependency wheels. After metadata and model validation it was
+151 MiB, with no LlamaIndex, Transformers, pandas, OpenAI or phone geography
+modules loaded. Configuration validation does not check provider credentials;
+processing workers still construct the actual clients.
+
+These are single-process measurements, not production container totals.
+Gunicorn workers share some mappings; use PSS rather than summing their RSS.

scripts/profile_web_imports.py

@@ -0,0 +1,71 @@
+"""Measure a fresh web process, including URL loading on the first request.
+
+Run with the application's environment configured:
+    python scripts/profile_web_imports.py --skip-external-startup
+
+This is a single-process measurement, not total Gunicorn/container memory.
+"""
+
+import argparse
+from contextlib import nullcontext
+import json
+import os
+from pathlib import Path
+import resource
+import sys
+import time
+
+
+def snapshot(stage, started):
+    peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    result = {
+        "stage": stage,
+        "peak_rss_mib": round(peak / (1024 ** 2 if sys.platform == "darwin" else 1024), 2),
+        "elapsed_seconds": round(time.monotonic() - started, 2),
+        "module_count": len(sys.modules),
+    }
+    # Linux PSS accounts for shared mappings; do not sum worker RSS values.
+    rollup = Path("/proc/self/smaps_rollup")
+    if rollup.exists():
+        for line in rollup.read_text().splitlines():
+            key, _, value = line.partition(":")
+            if key in ("Rss", "Pss", "Private_Dirty", "Private_Clean"):
+                result[key.lower() + "_mib"] = round(int(value.split()[0]) / 1024, 2)
+    print(json.dumps(result), flush=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--skip-external-startup", action="store_true",
+                        help="Skip the shared app's ArangoDB initialization hook.")
+    parser.add_argument("--skip-profile-validation", action="store_true",
+                        help="Compare older releases whose model validation constructs AI clients.")
+    args = parser.parse_args()
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "stixify.settings")
+    started = time.monotonic()
+    snapshot("baseline", started)
+    from unittest.mock import patch
+
+    context = patch("dogesec_commons.objects.db_view_creator.startup_func") if args.skip_external_startup else nullcontext()
+    with context:
+        from stixify.wsgi import application  # noqa: F401
+    snapshot("wsgi", started)
+    import stixify.urls  # noqa: F401
+    snapshot("urls", started)
+    from dogesec_commons.stixifier.serializers import validate_model, Txt2stixExtractorSerializer
+
+    Txt2stixExtractorSerializer.all_extractors(("pattern", "lookup", "ai"))
+    if not args.skip_profile_validation:
+        for provider in ("openai", "anthropic", "gemini", "deepseek", "openrouter"):
+            validate_model(provider)
+    snapshot("profile_metadata_and_validation", started)
+    prefixes = ("txt2stix", "llama_index", "pandas", "transformers", "torch", "openai", "phonenumbers.geodata")
+    print(json.dumps({"loaded_modules": {
+        prefix: sum(name == prefix or name.startswith(prefix + ".") for name in sys.modules)
+        for prefix in prefixes
+    }}), flush=True)
+
+
+if __name__ == "__main__":
+    main()

stixify/classifier/utils.py

@@ -1,9 +1,6 @@
 import os
 from typing import List
 
-import openai
-
-
 from .models import DocumentEmbedding
 
 
@@ -12,6 +9,8 @@ class ClusteringCancelled(Exception):
 
 
 def _openai_client():
+    import openai
+
     openai.api_key = os.getenv("OPENAI_API_KEY")
     return openai.Client()
 
@@ -40,4 +39,4 @@ def create_embedding_text(*texts: List[str]) -> str:
     """Create a single string to embed from multiple text fields."""
     # simple concat with separator, could be improved with field weighting or truncation
     texts = [t.strip() for t in texts if t and t.strip()]
-    return " | ".join(texts)
+    return " | ".join(texts)

tests/src/test_worker_imports.py

@@ -17,7 +17,7 @@ def _run_in_clean_process(code):
     )
 
 
-def test_django_and_task_imports_do_not_load_worker_dependencies():
+def test_web_routes_and_profile_validation_do_not_load_worker_dependencies():
     _run_in_clean_process(
         """
         import django
@@ -28,16 +28,33 @@ def test_django_and_task_imports_do_not_load_worker_dependencies():
             django.setup()
 
         import stixify.worker.tasks
+        import stixify.wsgi
+        import stixify.urls
+        from django.test import Client
+        from dogesec_commons.stixifier.serializers import (
+            validate_model, validate_extractor, uses_ai, Txt2stixExtractorSerializer,
+        )
 
-        # assert not any(
-        #     name == "txt2stix" or name.startswith("txt2stix.")
-        #     for name in sys.modules
-        # )
-        assert "stixify.worker.process_post" not in sys.modules
-        assert "stixify.worker.pdf_converter" not in sys.modules
-        assert "stixify.classifier.tasks" not in sys.modules
-        assert "sklearn.metrics.pairwise" not in sys.modules
-        assert "joblib" not in sys.modules
+        assert Client().get('/api/healthcheck/').status_code == 204
+        assert Txt2stixExtractorSerializer.all_extractors(('pattern',))
+        validate_extractor('extractor', ['pattern'], 'pattern_ipv4_address_only')
+        uses_ai(['pattern_ipv4_address_only'])
+        for provider in ('openai', 'anthropic', 'gemini', 'deepseek', 'openrouter'):
+            assert validate_model(provider) == provider
+            assert validate_model(provider + ':example-model') == provider + ':example-model'
+
+        forbidden = (
+            'stixify.worker.process_post', 'stixify.worker.pdf_converter',
+            'stixify.classifier.tasks', 'txt2stix.txt2stix', 'txt2stix.bundler',
+            'txt2stix.indicator', 'txt2stix.pattern', 'llama_index',
+            'transformers', 'torch', 'pandas', 'sklearn', 'joblib',
+            'openai', 'anthropic', 'google.genai', 'phonenumbers.geodata',
+        )
+        loaded = [
+            name for name in sys.modules
+            if any(name == prefix or name.startswith(prefix + '.') for prefix in forbidden)
+        ]
+        assert not loaded, loaded
         """
     )