From e89718d29c5548a13c24eb00f7dece106cb22e1b Mon Sep 17 00:00:00 2001 From: Ehtesham Alam Date: Tue, 25 Aug 2026 07:39:10 +0000 Subject: [PATCH] feat: make AI Moderation provider-agnostic --- docs/how-tos/configure_ai_moderation.rst | 104 ++++ docs/how-tos/index.rst | 5 + forum/__init__.py | 2 +- forum/ai_moderation/__init__.py | 8 + forum/ai_moderation/backends/__init__.py | 14 + forum/ai_moderation/backends/base.py | 188 +++++++ forum/ai_moderation/backends/xpert.py | 84 ++++ forum/ai_moderation/defaults.py | 83 ++++ .../service.py} | 234 +++++---- forum/api/comments.py | 2 +- forum/api/threads.py | 2 +- forum/settings/common.py | 38 ++ tests/test_ai_moderation.py | 286 +++++++---- tests/test_ai_moderation_backends.py | 465 ++++++++++++++++++ 14 files changed, 1322 insertions(+), 193 deletions(-) create mode 100644 docs/how-tos/configure_ai_moderation.rst create mode 100644 forum/ai_moderation/__init__.py create mode 100644 forum/ai_moderation/backends/__init__.py create mode 100644 forum/ai_moderation/backends/base.py create mode 100644 forum/ai_moderation/backends/xpert.py create mode 100644 forum/ai_moderation/defaults.py rename forum/{ai_moderation.py => ai_moderation/service.py} (66%) create mode 100644 tests/test_ai_moderation_backends.py diff --git a/docs/how-tos/configure_ai_moderation.rst b/docs/how-tos/configure_ai_moderation.rst new file mode 100644 index 00000000..7591272f --- /dev/null +++ b/docs/how-tos/configure_ai_moderation.rst @@ -0,0 +1,104 @@ +Configure AI moderation +####################### + +Forum can classify new threads and comments as spam and flag them, using an AI +provider of your choosing. Forum ships the interface; you supply a backend that +wraps your provider. + + +1. Write a backend +****************** + +Subclass ``HTTPModerationBackend`` and describe three things: how to +authenticate, what the request body looks like, and where the verdict sits in +the response. + +.. code-block:: python + + from django.conf import settings + + from forum.ai_moderation.backends import HTTPModerationBackend + + + class MyProviderBackend(HTTPModerationBackend): + def classify(self, content): + headers = { + "content-type": "application/json", + "Authorization": f"Bearer {settings.MY_PROVIDER_API_KEY}", + } + payload = { + "model": settings.MY_PROVIDER_MODEL, + "messages": [ + {"role": "system", "content": self.system_message}, + {"role": "user", "content": content}, + ], + } + + response = self.post(payload, headers) + if response is None: + return None + + answer = response["choices"][0]["message"]["content"] + return self.parse_moderation_payload(answer, response) + +``self.post()`` and ``self.parse_moderation_payload()`` handle the endpoint, +timeouts, network failures and parsing; ``self.system_message`` is the prompt. +See ``forum.ai_moderation.backends.base`` for the interface and what +``classify()`` must return. + +If your provider is not a JSON HTTP API, subclass ``BaseModerationBackend`` and +implement ``classify()`` however you like. + +Put the class anywhere the LMS can import. + + +2. Configure it +*************** + +.. code-block:: python + + AI_MODERATION_BACKEND = "myproject.moderation.MyProviderBackend" + AI_MODERATION_API_URL = "https://api.myprovider.example.com/v1/chat/completions" + AI_MODERATION_USER_ID = 42 + + # Read by your backend, named however you like. + MY_PROVIDER_API_KEY = "..." + MY_PROVIDER_MODEL = "..." + +All three forum settings are required and have no defaults. +``AI_MODERATION_USER_ID`` is the user that flagging and deletion are attributed +to; no user is created for you. + +Set them however your deployment sets Django settings -- with Tutor, a plugin +patching ``openedx-lms-production-settings``. Forum declares these settings in +its own plugin settings, so there is nothing to add to edx-platform. + +Optional: + +``AI_MODERATION_SYSTEM_MESSAGE`` + Your own prompt. Defaults to forum's, which asks for the JSON that + ``parse_moderation_payload()`` expects. + +``AI_MODERATION_CONNECTION_TIMEOUT``, ``AI_MODERATION_READ_TIMEOUT`` + Default to 1.0 and 30 seconds. Moderation runs inline with posting, so keep + them low. + +``AI_MODERATION_FLAGGED_CACHE_TTL``, ``AI_MODERATION_FLAGGED_CACHE_PREFIX`` + Spam verdicts are cached by content hash for 24 hours, so an identical + repost costs no second API call. Clean verdicts are never cached. + + +3. Turn it on +************* + +Two course waffle flags, both off by default: + +``discussions.enable_ai_moderation`` + Classify new threads and comments, and flag what comes back spam. + +``discussions.enable_ai_auto_delete_spam`` + Also soft delete what was flagged. Has no effect on its own. + +Enable them site-wide in Django admin under ``waffle/flag``, or per course with +a "Waffle flag course override" at +``/admin/waffle_utils/waffleflagcourseoverridemodel/``. diff --git a/docs/how-tos/index.rst b/docs/how-tos/index.rst index 5147f808..f2bf79d7 100644 --- a/docs/how-tos/index.rst +++ b/docs/how-tos/index.rst @@ -1,2 +1,7 @@ How-tos ####### + +.. toctree:: + :maxdepth: 1 + + configure_ai_moderation diff --git a/forum/__init__.py b/forum/__init__.py index 33827dd1..07a4c898 100644 --- a/forum/__init__.py +++ b/forum/__init__.py @@ -2,4 +2,4 @@ Openedx forum app. """ -__version__ = "0.7.1" +__version__ = "0.7.2" diff --git a/forum/ai_moderation/__init__.py b/forum/ai_moderation/__init__.py new file mode 100644 index 00000000..ca6b809f --- /dev/null +++ b/forum/ai_moderation/__init__.py @@ -0,0 +1,8 @@ +""" +AI moderation for forum content. + +Classifies each new thread and comment with the configured AI provider, flags +what comes back spam, and soft deletes it when auto-delete is enabled. Both +steps are gated on course waffle flags, and every spam verdict is recorded on a +moderation audit log. +""" diff --git a/forum/ai_moderation/backends/__init__.py b/forum/ai_moderation/backends/__init__.py new file mode 100644 index 00000000..3070989a --- /dev/null +++ b/forum/ai_moderation/backends/__init__.py @@ -0,0 +1,14 @@ +""" +AI moderation backends. + +Each backend adapts one AI provider to the common moderation interface. The +backend in use is chosen with the ``AI_MODERATION_BACKEND`` setting, so nothing +outside this package needs to know which provider is answering. +""" + +from forum.ai_moderation.backends.base import ( + BaseModerationBackend, + HTTPModerationBackend, +) + +__all__ = ["BaseModerationBackend", "HTTPModerationBackend"] diff --git a/forum/ai_moderation/backends/base.py b/forum/ai_moderation/backends/base.py new file mode 100644 index 00000000..b51eff2b --- /dev/null +++ b/forum/ai_moderation/backends/base.py @@ -0,0 +1,188 @@ +""" +Provider-agnostic interface for AI moderation backends. + +A moderation backend is the only part of AI moderation that knows how to talk to +a particular AI provider. It takes a piece of forum content and returns the +common moderation result described by :meth:`BaseModerationBackend.classify`; +everything downstream of that -- caching, flagging, deletion, audit logging -- +is provider independent and lives in :mod:`forum.ai_moderation.service`. +""" + +import json +import logging +from typing import Any, Dict, Optional, Tuple + +import requests +from django.conf import settings + +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_READ_TIMEOUT, + DEFAULT_REASONING, + DEFAULT_SYSTEM_MESSAGE, +) + +log = logging.getLogger(__name__) + +CLASSIFICATION_SPAM = "spam_or_scam" +CLASSIFICATION_NOT_SPAM = "not_spam" + +# Classifications that count as spam. "spam" is accepted alongside the +# documented "spam_or_scam" because prompts in the wild return either. +SPAM_CLASSIFICATIONS = ("spam", CLASSIFICATION_SPAM) + + +class BaseModerationBackend: + """ + Interface implemented by every AI moderation backend. + """ + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """ + Classify a piece of forum content. + + Args: + content: The text content to classify. + + Returns: + A moderation result:: + + { + "classification": "spam_or_scam" | "not_spam", + "reasoning": "...", + "confidence_score": float | None, + "full_api_response": , + } + + or None if the provider could not be reached or answered with + something that could not be understood. Returning None must never + raise: a failing classifier degrades moderation, it does not break + posting. + """ + raise NotImplementedError + + +class HTTPModerationBackend(BaseModerationBackend): # pylint: disable=abstract-method + """ + Base class for backends that call an HTTP moderation API. + + It owns the concerns that are the same whichever provider is in use -- + reading the endpoint, prompt and timeouts from Django settings, POSTing + JSON, and turning the classifier's JSON payload into the common moderation + result. Subclasses only describe the provider's request and response shape. + """ + + @property + def api_url(self) -> Optional[str]: + """Endpoint the classifier is served from.""" + return getattr(settings, "AI_MODERATION_API_URL", None) + + @property + def system_message(self) -> str: + """Prompt describing the classification task and its output format.""" + return ( + getattr(settings, "AI_MODERATION_SYSTEM_MESSAGE", None) + or DEFAULT_SYSTEM_MESSAGE + ) + + @property + def timeout(self) -> Tuple[float, float]: + """Connection and read timeouts, in seconds.""" + return ( + getattr( + settings, "AI_MODERATION_CONNECTION_TIMEOUT", DEFAULT_CONNECTION_TIMEOUT + ), + getattr(settings, "AI_MODERATION_READ_TIMEOUT", DEFAULT_READ_TIMEOUT), + ) + + def post(self, payload: Dict[str, Any], headers: Dict[str, str]) -> Optional[Any]: + """ + POST a JSON payload to the configured endpoint and decode the response. + + Returns the decoded JSON body, or None if the request failed. + """ + if not self.api_url: + log.error("AI_MODERATION_API_URL setting is not configured") + return None + + try: + response = requests.post( + self.api_url, + headers=headers, + json=payload, + timeout=self.timeout, + ) + response.raise_for_status() + return response.json() + except ( + requests.RequestException, + requests.Timeout, + requests.ConnectionError, + ) as e: + log.error(f"AI moderation API request failed: {e}") + return None + except ValueError as e: + log.error(f"AI moderation API returned a non-JSON response: {e}") + return None + + def parse_moderation_payload( + self, raw_content: Any, full_api_response: Any + ) -> Optional[Dict[str, Any]]: + """ + Turn the JSON document produced by the classifier into a moderation result. + + Args: + raw_content: The classifier's answer, as a JSON string. Models + routinely wrap it in a Markdown code fence, which is stripped. + full_api_response: The provider's whole response, kept for auditing. + + Returns: + The common moderation result, or None if it could not be parsed. + """ + if not isinstance(raw_content, str) or not raw_content.strip(): + log.error("AI moderation response did not contain any content") + return None + + try: + parsed = json.loads(strip_code_fence(raw_content)) + except json.JSONDecodeError as e: + log.error(f"Failed to parse AI moderation response JSON: {e}") + return None + + if not isinstance(parsed, dict): + log.error( + f"Expected a JSON object from the AI moderation API, got {type(parsed)}" + ) + return None + + return normalize_moderation_result(parsed, full_api_response) + + +def normalize_moderation_result( + parsed: Dict[str, Any], full_api_response: Any +) -> Dict[str, Any]: + """ + Fill in the keys the rest of AI moderation relies on. + + Any additional keys the classifier returned are preserved: they end up on + the audit log, where they are worth having. + """ + result = dict(parsed) + result["classification"] = parsed.get("classification", CLASSIFICATION_NOT_SPAM) + result["reasoning"] = parsed.get("reasoning", DEFAULT_REASONING) + result["confidence_score"] = parsed.get("confidence_score") + result["full_api_response"] = full_api_response + return result + + +def strip_code_fence(text: str) -> str: + """Remove a surrounding Markdown code fence, if the model added one.""" + stripped = text.strip() + if not stripped.startswith("```"): + return stripped + + # Drop the opening fence, which may carry a language hint such as ```json. + lines = stripped.splitlines()[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + return "\n".join(lines).strip() diff --git a/forum/ai_moderation/backends/xpert.py b/forum/ai_moderation/backends/xpert.py new file mode 100644 index 00000000..5f511e2f --- /dev/null +++ b/forum/ai_moderation/backends/xpert.py @@ -0,0 +1,84 @@ +""" +AI moderation backend for the edX XPert classifier. + +This backend is edX specific: it exists so that the deployment which already +runs against XPert keeps doing so now that AI moderation itself is provider +agnostic. It is also the worked example of what a provider backend looks like -- +see ``docs/how-tos/configure_ai_moderation.rst`` for writing your own. + +XPert's dialect: it authorises on a ``client_id`` carried in the request body +rather than an ``Authorization`` header, it takes the prompt as a top level +``system_message``, and it answers with a list whose first element holds the +classifier JSON. +""" + +import logging +from typing import Any, Dict, Optional + +from django.conf import settings + +from forum.ai_moderation.backends.base import HTTPModerationBackend + +log = logging.getLogger(__name__) + + +class XPertModerationBackend(HTTPModerationBackend): + """ + Classify content with the edX XPert API. + + Configured with:: + + AI_MODERATION_API_URL # e.g. https://xpert-api-services.../v1/message + AI_MODERATION_CLIENT_ID + AI_MODERATION_SYSTEM_MESSAGE + AI_MODERATION_CONNECTION_TIMEOUT + AI_MODERATION_READ_TIMEOUT + """ + + @property + def client_id(self) -> Optional[str]: + """XPert client the request is billed and authorised against.""" + return getattr(settings, "AI_MODERATION_CLIENT_ID", None) + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """Classify content, returning the common moderation result.""" + headers = { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9", + "content-type": "application/json", + "user-agent": "Mozilla/5.0 (compatible; edX-Forum-AI-Moderation/1.0)", + } + + payload: Dict[str, Any] = { + "messages": [{"role": "user", "content": content}], + "client_id": self.client_id, + "system_message": self.system_message, + } + + response_data = self.post(payload, headers) + if response_data is None: + return None + + message_content = self._extract_message_content(response_data) + if message_content is None: + return None + + return self.parse_moderation_payload(message_content, response_data) + + def _extract_message_content(self, response_data: Any) -> Optional[Any]: + """Pull the classifier answer out of an XPert response.""" + if not isinstance(response_data, list): + log.error( + f"Expected list response from XPert API, got {type(response_data)}" + ) + return None + + if len(response_data) == 0: + log.error("Empty response list from XPert API") + return None + + if not isinstance(response_data[0], dict): + log.error(f"Expected dict in response list, got {type(response_data[0])}") + return None + + return response_data[0].get("content", "") diff --git a/forum/ai_moderation/defaults.py b/forum/ai_moderation/defaults.py new file mode 100644 index 00000000..397a793e --- /dev/null +++ b/forum/ai_moderation/defaults.py @@ -0,0 +1,83 @@ +""" +Provider-independent defaults for AI moderation. + +Every value here is the fallback used when the matching Django setting is not +configured. Nothing in this module may encode the behaviour of a single AI +provider: which provider answers is chosen entirely by AI_MODERATION_BACKEND, +which has no default -- forum ships an interface, not a provider. + +The one thing forum does supply is the prompt, so that standing up a backend +does not also mean writing a spam classifier prompt from scratch. +""" + +# Seconds to wait for the connection to be established, and then for the +# classifier to answer. Moderation runs inline with thread/comment creation, so +# the connect timeout is deliberately short. +DEFAULT_CONNECTION_TIMEOUT = 1.0 +DEFAULT_READ_TIMEOUT = 30 + +# Spam verdicts are cached by content hash so an identical repost does not cost +# a second API call. Clean verdicts are never cached. +DEFAULT_FLAGGED_CACHE_TTL = 60 * 60 * 24 +DEFAULT_FLAGGED_CACHE_PREFIX = "ai_moderation:flagged:v1" + +# Value used for a missing `reasoning` field in a classifier response. +DEFAULT_REASONING = "No reasoning provided" + +DEFAULT_SYSTEM_MESSAGE = """\ +Filter posts from a discussion forum platform to identify and flag content that is likely to be spam or a scam. + +**Instructions**: +- Carefully analyze each post's text for language, links, or patterns typical of spam or scams. +- Use clear reasoning to identify suspicious indicators such as: + * Promotional language or unsolicited commercial content + * Misleading claims or "too good to be true" offers + * Excessive external links (especially non-educational domains) + * Requests for personal information (phone numbers, email, social media) + * Suspicious offers (money, investment, guaranteed results) + * Impersonation of authority figures (course staff, professors) + * Directing users to external communication platforms (WhatsApp, Telegram) + * Cryptocurrency, forex, or investment scheme language + * Urgent pressure tactics ("act now", "limited time") + +- After thoroughly explaining your reasoning and highlighting specific suspicious features, + classify the post as either "spam_or_scam" or "not_spam". +- **Do not make a classification before detailing your reasoning.** Always present your + analysis of the post's content before your final determination. +- If uncertainty exists, explain which factors made detection difficult before concluding. +- Consider legitimate use cases: Course-related external links (.edu domains), genuine help + requests, study group formation. + +**Output Format** (strict JSON, and nothing else): +{ + "reasoning": "[Detailed explanation of why this post may or may not be spam/scam, + referencing specific features of the post. Minimum 2 sentences.]", + "classification": "[spam_or_scam | not_spam]" +} + +**Examples**: + +Example 1 (Spam): +Post: "Hi everyone! I'm Professor Johnson. Contact me on WhatsApp +1-555-0123 for +guaranteed A+ grades. Limited slots!" +Output: +{ + "reasoning": "This post exhibits multiple red flags: (1) Impersonation of a professor + with no verification, (2) request to contact via WhatsApp with phone + number, (3) unrealistic promise of 'guaranteed A+ grades', (4) urgency + tactic 'limited slots'. These are classic patterns of academic scams + targeting students.", + "classification": "spam_or_scam" +} + +Example 2 (Not Spam): +Post: "Can someone explain the difference between merge sort and quick sort? I'm +struggling with the time complexity analysis." +Output: +{ + "reasoning": "This is a legitimate academic question about sorting algorithms. The post + contains no suspicious links, no requests for external contact, no + promotional language, and is directly related to course content. The tone + is appropriate for a learner seeking help.", + "classification": "not_spam" +}""" diff --git a/forum/ai_moderation.py b/forum/ai_moderation/service.py similarity index 66% rename from forum/ai_moderation.py rename to forum/ai_moderation/service.py index a3c56798..9e85da35 100644 --- a/forum/ai_moderation.py +++ b/forum/ai_moderation/service.py @@ -2,20 +2,29 @@ AI Moderation utilities for forum content. """ -import json -import logging import hashlib -from typing import Dict, Optional, Any +import logging +from typing import Any, Dict, Optional -import requests from django.conf import settings from django.contrib.auth import get_user_model from django.core.cache import cache -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist from django.utils import timezone +from django.utils.module_loading import import_string from opaque_keys.edx.keys import CourseKey from rest_framework.serializers import ValidationError +from forum.ai_moderation.backends.base import ( + SPAM_CLASSIFICATIONS, + BaseModerationBackend, + CLASSIFICATION_NOT_SPAM, +) +from forum.ai_moderation.defaults import ( + DEFAULT_FLAGGED_CACHE_PREFIX, + DEFAULT_FLAGGED_CACHE_TTL, + DEFAULT_REASONING, +) from forum.backends.mysql.models import ModerationAuditLog from forum.utils import ForumV2RequestError @@ -88,7 +97,7 @@ def create_moderation_audit_log( timestamp=timezone.now(), body=content_body, # Store full body content classifier_output=enhanced_moderation_result, - reasoning=moderation_result.get("reasoning", "No reasoning provided"), + reasoning=moderation_result.get("reasoning", DEFAULT_REASONING), classification=moderation_result.get("classification", "spam"), actions_taken=actions_taken, confidence_score=moderation_result.get("confidence_score"), @@ -105,28 +114,109 @@ class AIModerationService: Waffle Flag "discussion.enable_ai_moderation" controls whether AI moderation is active. - XPERT AI Moderation API is used to classify content as spam or not spam. + Content is classified by the moderation backend named in the + AI_MODERATION_BACKEND setting. There is no default: forum defines the + interface and leaves the choice of provider to the deployment. This service + is provider agnostic -- everything it does with a verdict, from caching to + flagging to soft deletion to audit logging, is the same whichever backend + produced it. """ - def __init__(self): # type: ignore[no-untyped-def] + def __init__(self) -> None: """Initialize the AI moderation service.""" - self.api_url = getattr(settings, "AI_MODERATION_API_URL", None) - self.client_id = getattr(settings, "AI_MODERATION_CLIENT_ID", None) - self.system_message = getattr(settings, "AI_MODERATION_SYSTEM_MESSAGE", None) - self.connection_timeout = getattr( - settings, "AI_MODERATION_CONNECTION_TIMEOUT", 30 - ) # seconds - self.read_timeout = getattr( - settings, "AI_MODERATION_READ_TIMEOUT", 30 - ) # seconds - self.ai_moderation_user_id = getattr(settings, "AI_MODERATION_USER_ID", None) - self.flagged_cache_ttl = getattr( - settings, "AI_MODERATION_FLAGGED_CACHE_TTL", 60 * 60 * 24 + self._moderation_backend: Optional[BaseModerationBackend] = None + self._moderation_backend_path: Optional[str] = None + + @property + def ai_moderation_user_id(self) -> Optional[Any]: + """User the moderation actions are attributed to.""" + return getattr(settings, "AI_MODERATION_USER_ID", None) + + @property + def flagged_cache_ttl(self) -> int: + """How long a spam verdict stays cached, in seconds.""" + return getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_TTL", DEFAULT_FLAGGED_CACHE_TTL ) - self.flagged_cache_prefix = getattr( - settings, "AI_MODERATION_FLAGGED_CACHE_PREFIX", "ai_moderation:flagged:v1" + + @property + def flagged_cache_prefix(self) -> str: + """Key prefix for cached spam verdicts.""" + return getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_PREFIX", DEFAULT_FLAGGED_CACHE_PREFIX ) + @property + def moderation_backend_path(self) -> Optional[str]: + """Dotted path of the configured moderation backend, if one is configured.""" + return getattr(settings, "AI_MODERATION_BACKEND", None) + + @property + def moderation_backend(self) -> BaseModerationBackend: + """ + The configured moderation backend. + + Loaded on first use rather than in __init__ so that the module level + service instance does not freeze the setting at import time, and cached + until the configured path changes. + + Raises: + ImproperlyConfigured: if AI_MODERATION_BACKEND is unset, or does not + name a usable BaseModerationBackend. + """ + backend_path = self.moderation_backend_path + if not backend_path: + raise ImproperlyConfigured( + "AI_MODERATION_BACKEND is not configured. Forum provides the " + "moderation interface but no provider: set this to the dotted path " + "of a BaseModerationBackend subclass." + ) + if ( + self._moderation_backend is None + or self._moderation_backend_path != backend_path + ): + self._moderation_backend = self._load_moderation_backend(backend_path) + self._moderation_backend_path = backend_path + return self._moderation_backend + + @staticmethod + def _load_moderation_backend(backend_path: str) -> BaseModerationBackend: + """Import and instantiate the moderation backend at ``backend_path``.""" + try: + backend_class = import_string(backend_path) + except ImportError as e: + raise ImproperlyConfigured( + f"AI_MODERATION_BACKEND '{backend_path}' could not be imported: {e}" + ) from e + + backend = backend_class() + if not isinstance(backend, BaseModerationBackend): + raise ImproperlyConfigured( + f"AI_MODERATION_BACKEND '{backend_path}' is not a subclass of " + f"{BaseModerationBackend.__module__}.{BaseModerationBackend.__name__}" + ) + return backend + + def _classify(self, content: str) -> Optional[Dict[str, Any]]: + """ + Ask the configured backend to classify content. + + Returns the moderation result, or None if the backend is unusable or + failed. Moderation runs inline with posting, so no backend problem is + allowed to propagate out of here. + """ + try: + return self.moderation_backend.classify(content) + except ImproperlyConfigured as e: + log.error(f"AI moderation backend is not usable: {e}") + return None + except Exception: # pylint: disable=broad-except + log.exception( + f"AI moderation backend '{self.moderation_backend_path}' " + f"raised an unexpected error" + ) + return None + def _cache_key_for_content(self, content: str) -> str: """Return the cache key for a given message content.""" normalized = (content or "").strip() @@ -155,78 +245,6 @@ def _set_cached_flagged_result( except Exception: # pylint: disable=broad-except log.exception("AI moderation cache write failed") - def _make_api_request(self, content: str) -> Optional[Dict[str, Any]]: - """ - Make API request to XPert Service. - - Args: - content: The text content to moderate - - Returns: - Dictionary with 'reasoning' and 'classification' keys, or None if failed - """ - if not self.api_url: - log.error("AI_MODERATION_API_URL setting is not configured") - return None - - headers = { - "accept": "*/*", - "accept-language": "en-US,en;q=0.9", - "content-type": "application/json", - "user-agent": "Mozilla/5.0 (compatible; edX-Forum-AI-Moderation/1.0)", - } - - payload = { - "messages": [{"role": "user", "content": content}], - "client_id": self.client_id, - "system_message": self.system_message, - } - - try: - response = requests.post( - self.api_url, - headers=headers, - json=payload, - timeout=(self.connection_timeout, self.read_timeout), - ) - response.raise_for_status() - - response_data = response.json() - # Validate response data structure - if not isinstance(response_data, list): - log.error( - f"Expected list response from XPert API, got {type(response_data)}" - ) - return None - - if len(response_data) == 0: - log.error("Empty response list from XPert API") - return None - - if not isinstance(response_data[0], dict): - log.error( - f"Expected dict in response list, got {type(response_data[0])}" - ) - return None - - assistant_content = response_data[0].get("content", "") - # Parse the JSON content from the assistant response - try: - moderation_result = json.loads(assistant_content) - # full API response for audit purposes - moderation_result["full_api_response"] = response_data - return moderation_result - except json.JSONDecodeError as e: - log.error(f"Failed to parse AI moderation response JSON: {e}") - return None - except ( - requests.RequestException, - requests.Timeout, - requests.ConnectionError, - ) as e: - log.error(f"AI moderation API request failed: {e}") - return None - def moderate_and_flag_content( self, content: str, @@ -241,7 +259,9 @@ def moderate_and_flag_content( content: The text content to check content_instance: The content model instance (Thread or Comment) course_id: Optional course ID for waffle flag checking - backend: Backend instance for database operations + backend: Forum storage backend used for the database operations. + This is not the AI moderation backend, which is chosen by the + AI_MODERATION_BACKEND setting. Returns: Dictionary with moderation results and actions taken @@ -249,7 +269,7 @@ def moderate_and_flag_content( result = { "is_spam": False, "reasoning": "AI moderation disabled or unavailable", - "classification": "not_spam", + "classification": CLASSIFICATION_NOT_SPAM, "actions_taken": ["no_action"], "flagged": False, } @@ -267,18 +287,20 @@ def moderate_and_flag_content( # If we've already flagged this exact content before, reuse the cached result moderation_result = self._get_cached_flagged_result(content) if moderation_result is None: - moderation_result = self._make_api_request(content) + moderation_result = self._classify(content) if moderation_result is None: result["reasoning"] = "AI moderation API failed" log.warning("AI moderation API failed") return result - classification = moderation_result.get("classification", "not_spam") - reasoning = moderation_result.get("reasoning", "No reasoning provided") - is_spam = classification in ["spam", "spam_or_scam"] + classification = moderation_result.get( + "classification", CLASSIFICATION_NOT_SPAM + ) + reasoning = moderation_result.get("reasoning", DEFAULT_REASONING) + is_spam = classification in SPAM_CLASSIFICATIONS - # Cache only flagged (spam) results to avoid repeated XPert calls + # Cache only flagged (spam) results to avoid repeated classifier calls if is_spam: self._set_cached_flagged_result(content, moderation_result) @@ -299,6 +321,9 @@ def moderate_and_flag_content( self._mark_as_spam_and_moderate(content_instance, backend) result["actions_taken"] = ["flagged"] result["flagged"] = True + except ImproperlyConfigured as e: + log.error(f"Cannot act on AI moderation verdict: {e}") + result["actions_taken"] = ["no_action"] except (AttributeError, ValueError, TypeError) as e: log.error(f"Failed to flag content as spam: {e}") result["actions_taken"] = ["no_action"] @@ -333,7 +358,10 @@ def _mark_as_spam_and_moderate(self, content_instance: Any, backend: Any) -> Non ) } if not self.ai_moderation_user_id: - raise ValueError("AI_MODERATION_USER_ID setting is not configured.") + raise ImproperlyConfigured( + "AI_MODERATION_USER_ID setting is not configured, so there is no user " + "to attribute AI moderation actions to." + ) backend.flag_content_as_spam(content_type, content_id) backend.flag_as_abuse(str(self.ai_moderation_user_id), content_id, **extra_data) @@ -374,7 +402,7 @@ def _delete_content(self, content_instance: Any) -> None: # Global instance -ai_moderation_service = AIModerationService() # type: ignore[no-untyped-call] +ai_moderation_service = AIModerationService() def moderate_and_flag_spam( diff --git a/forum/api/comments.py b/forum/api/comments.py index 30debd45..e65a5b15 100644 --- a/forum/api/comments.py +++ b/forum/api/comments.py @@ -9,7 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.serializers import ValidationError -from forum.ai_moderation import moderate_and_flag_spam +from forum.ai_moderation.service import moderate_and_flag_spam from forum.backend import get_backend from forum.serializers.comment import CommentSerializer from forum.utils import ForumV2RequestError diff --git a/forum/api/threads.py b/forum/api/threads.py index b92aecca..caba96e7 100644 --- a/forum/api/threads.py +++ b/forum/api/threads.py @@ -10,7 +10,7 @@ from django.core.exceptions import ObjectDoesNotExist from rest_framework.serializers import ValidationError -from forum.ai_moderation import moderate_and_flag_spam +from forum.ai_moderation.service import moderate_and_flag_spam from forum.api.users import mark_thread_as_read from forum.backend import get_backend from forum.serializers.thread import ThreadSerializer diff --git a/forum/settings/common.py b/forum/settings/common.py index dbecd9cd..c620aec1 100644 --- a/forum/settings/common.py +++ b/forum/settings/common.py @@ -4,6 +4,14 @@ from typing import Any +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_FLAGGED_CACHE_PREFIX, + DEFAULT_FLAGGED_CACHE_TTL, + DEFAULT_READ_TIMEOUT, + DEFAULT_SYSTEM_MESSAGE, +) + def plugin_settings(settings: Any) -> None: """ @@ -44,3 +52,33 @@ def plugin_settings(settings: Any) -> None: # Timezone-awareness is required for mysql fields settings.USE_TZ = getattr(settings, "USE_TZ", True) + + # AI moderation. These run after the deployment's own configuration has been + # read, so every one of them defers to an already configured value; they are + # here to declare the settings and their defaults, not to impose them. + # + # AI_MODERATION_BACKEND has no default on purpose: forum defines the moderation + # interface and ships no provider, so a deployment must name the backend it + # wants. Neither does AI_MODERATION_API_URL, nor AI_MODERATION_USER_ID, which + # decides who flagging and deletion are attributed to -- no user is created for + # you. Whatever else a backend needs is read by that backend and deliberately + # not declared here: the XPert backend's AI_MODERATION_CLIENT_ID, or the + # credential setting of whichever provider you wrap. + settings.AI_MODERATION_BACKEND = getattr(settings, "AI_MODERATION_BACKEND", None) + settings.AI_MODERATION_API_URL = getattr(settings, "AI_MODERATION_API_URL", None) + settings.AI_MODERATION_USER_ID = getattr(settings, "AI_MODERATION_USER_ID", None) + settings.AI_MODERATION_SYSTEM_MESSAGE = getattr( + settings, "AI_MODERATION_SYSTEM_MESSAGE", DEFAULT_SYSTEM_MESSAGE + ) + settings.AI_MODERATION_CONNECTION_TIMEOUT = getattr( + settings, "AI_MODERATION_CONNECTION_TIMEOUT", DEFAULT_CONNECTION_TIMEOUT + ) + settings.AI_MODERATION_READ_TIMEOUT = getattr( + settings, "AI_MODERATION_READ_TIMEOUT", DEFAULT_READ_TIMEOUT + ) + settings.AI_MODERATION_FLAGGED_CACHE_TTL = getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_TTL", DEFAULT_FLAGGED_CACHE_TTL + ) + settings.AI_MODERATION_FLAGGED_CACHE_PREFIX = getattr( + settings, "AI_MODERATION_FLAGGED_CACHE_PREFIX", DEFAULT_FLAGGED_CACHE_PREFIX + ) diff --git a/tests/test_ai_moderation.py b/tests/test_ai_moderation.py index 2699b6f6..cc9c3ea5 100644 --- a/tests/test_ai_moderation.py +++ b/tests/test_ai_moderation.py @@ -1,14 +1,15 @@ """Tests for AI moderation functionality.""" import sys -from typing import Any +from typing import Any, Generator from unittest.mock import Mock, MagicMock, patch import pytest from django.contrib.auth import get_user_model from django.core.cache import cache +from django.test import override_settings -from forum.ai_moderation import AIModerationService, moderate_and_flag_spam +from forum.ai_moderation.service import AIModerationService, moderate_and_flag_spam from forum.backends.mysql.models import ModerationAuditLog from forum.utils import ForumV2RequestError @@ -41,16 +42,57 @@ def is_enabled(self, _course_key: Any) -> bool: sys.modules["openedx.core.djangoapps.waffle_utils"] = mock_waffle_utils +XPERT_BACKEND = "forum.ai_moderation.backends.xpert.XPertModerationBackend" + + +def classifier_response(payload: str) -> Mock: + """ + Build a mocked classifier response, in the shape the configured backend reads. + + Args: + payload: The JSON document the classifier answered with. + """ + response = Mock() + response.status_code = 200 + response.json.return_value = [{"content": payload}] + return response + + +SPAM_RESPONSE = ( + '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' +) +NOT_SPAM_RESPONSE = ( + '{"classification": "not_spam", "reasoning": "This is legitimate content", ' + '"confidence_score": 0.9}' +) + + +@pytest.fixture(autouse=True) +def clear_moderation_cache() -> Generator[None, None, None]: + """Keep cached spam verdicts from leaking between tests.""" + cache.clear() + yield + cache.clear() + + @pytest.fixture def mock_ai_moderation_settings() -> Any: - """Mock AI moderation settings.""" - with patch("forum.ai_moderation.settings") as mock_settings: - mock_settings.AI_MODERATION_API_URL = "http://test-api.example.com" - mock_settings.AI_MODERATION_API_KEY = "test-api-key" - mock_settings.AI_MODERATION_USER_ID = "999" - mock_settings.AI_MODERATION_FLAGGED_CACHE_TTL = 60 * 60 - mock_settings.AI_MODERATION_FLAGGED_CACHE_PREFIX = "ai_moderation:flagged:v1" - yield mock_settings + """ + Configure AI moderation against a provider backend. + + Which backend does not matter to any test in this module -- they are about + the provider-agnostic workflow -- but one has to be named, because forum + ships no default. + """ + with override_settings( + AI_MODERATION_BACKEND=XPERT_BACKEND, + AI_MODERATION_API_URL="http://test-api.example.com", + AI_MODERATION_CLIENT_ID="test-client-id", + AI_MODERATION_USER_ID="999", + AI_MODERATION_FLAGGED_CACHE_TTL=60 * 60, + AI_MODERATION_FLAGGED_CACHE_PREFIX="ai_moderation:flagged:v1", + ): + yield @pytest.fixture @@ -73,7 +115,7 @@ def ai_service( mock_ai_moderation_settings: Any, # pylint: disable=redefined-outer-name,unused-argument ) -> AIModerationService: """Create an AI moderation service instance.""" - return AIModerationService() # type: ignore[no-untyped-call] + return AIModerationService() @pytest.fixture @@ -115,13 +157,7 @@ def test_auto_delete_triggered_when_enabled( ) -> None: """Test that auto-delete is triggered when waffle flag is enabled.""" # Mock API response indicating spam - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "This content is spam", "confidence_score": 0.95}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() @@ -155,13 +191,7 @@ def test_auto_delete_not_triggered_when_disabled( mock_waffle_flags["auto_delete"].return_value = False # Mock API response indicating spam - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "This content is spam", "confidence_score": 0.95}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() @@ -192,15 +222,7 @@ def test_auto_delete_not_triggered_for_non_spam( ) -> None: """Test that auto-delete is NOT triggered for non-spam content.""" # Mock API response indicating NOT spam - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "not_spam", ' - '"reasoning": "This is legitimate content", ' - '"confidence_score": 0.9}' - } - ] + mock_response = classifier_response(NOT_SPAM_RESPONSE) backend = Mock() @@ -232,13 +254,7 @@ def test_actions_taken_reflects_flagged_only_when_delete_disabled( # Disable auto-delete mock_waffle_flags["auto_delete"].return_value = False - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() @@ -260,13 +276,7 @@ def test_actions_taken_reflects_both_when_delete_enabled( sample_comment_content: dict[str, Any], ) -> None: """Test that actions_taken correctly reflects both flagging and deletion.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() @@ -286,6 +296,140 @@ def test_actions_taken_reflects_both_when_delete_enabled( assert len(result["actions_taken"]) == 2 +class TestAIModerationBackendDelegation: # pylint: disable=redefined-outer-name,unused-argument + """Tests that the service delegates classification to the configured backend.""" + + def test_service_calls_backend_classify( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """The service asks the backend to classify, and acts on what it returns.""" + classify = Mock( + return_value={ + "classification": "spam_or_scam", + "reasoning": "Spam detected", + "confidence_score": 0.9, + } + ) + mock_waffle_flags["auto_delete"].return_value = False + + with patch.object(ai_service.moderation_backend, "classify", classify): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=Mock(), + ) + + classify.assert_called_once_with("spam content") + assert result["is_spam"] is True + assert result["actions_taken"] == ["flagged"] + + def test_service_has_no_provider_specific_request_code(self) -> None: + """The service no longer talks to any provider itself.""" + assert not hasattr(AIModerationService, "_make_api_request") + + def test_backend_failure_leaves_content_alone( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """A backend that cannot classify degrades moderation, it does not raise.""" + backend = Mock() + + with patch.object( + ai_service.moderation_backend, "classify", Mock(return_value=None) + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["is_spam"] is False + assert result["actions_taken"] == ["no_action"] + assert result["reasoning"] == "AI moderation API failed" + backend.flag_content_as_spam.assert_not_called() + + def test_unexpected_backend_error_is_contained( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """An exception from a third party backend must not break posting.""" + with patch.object( + ai_service.moderation_backend, + "classify", + Mock(side_effect=RuntimeError("boom")), + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=Mock(), + ) + + assert result["is_spam"] is False + assert result["actions_taken"] == ["no_action"] + + +class TestAIModerationUserId: # pylint: disable=redefined-outer-name,unused-argument + """Tests for attributing moderation actions to AI_MODERATION_USER_ID.""" + + def test_actions_are_attributed_to_configured_user( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + ) -> None: + """Flagging is performed as the configured moderation user.""" + mock_waffle_flags["auto_delete"].return_value = False + backend = Mock() + + with patch("requests.post", return_value=classifier_response(SPAM_RESPONSE)): + ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + backend.flag_as_abuse.assert_called_once_with( + "999", "thread123", entity_type="CommentThread" + ) + + def test_missing_user_id_reports_a_configuration_error( + self, + ai_service: AIModerationService, + mock_waffle_flags: dict[str, Mock], + sample_thread_content: dict[str, Any], + caplog: pytest.LogCaptureFixture, + ) -> None: + """Without AI_MODERATION_USER_ID nothing is moderated, and it is logged.""" + backend = Mock() + + with override_settings(AI_MODERATION_USER_ID=None), patch( + "requests.post", return_value=classifier_response(SPAM_RESPONSE) + ): + result = ai_service.moderate_and_flag_content( + "spam content", + sample_thread_content, + course_id="course-v1:edX+DemoX+Demo", + backend=backend, + ) + + assert result["is_spam"] is True + assert result["flagged"] is False + assert result["actions_taken"] == ["no_action"] + backend.flag_as_abuse.assert_not_called() + assert "AI_MODERATION_USER_ID" in caplog.text + + class TestAIModerationCaching: # pylint: disable=redefined-outer-name,unused-argument """Tests for caching of flagged moderation results.""" @@ -299,18 +443,12 @@ def test_flagged_result_is_cached_and_reused( mock_waffle_flags["auto_delete"].return_value = False # Mock API response indicating spam - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() with patch("requests.post", return_value=mock_response) as mock_post: - # First call should hit XPert and then cache + # First call should hit the classifier and then cache first = ai_service.moderate_and_flag_content( "spam content", sample_thread_content, @@ -341,13 +479,7 @@ def test_deletion_failure_after_successful_flagging( sample_thread_content: dict[str, Any], ) -> None: """Test that flagging succeeds even if deletion fails.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() @@ -377,13 +509,7 @@ def test_flagging_failure_prevents_deletion( sample_thread_content: dict[str, Any], ) -> None: """Test that if flagging fails, deletion is not attempted.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() backend.flag_content_as_spam.side_effect = ValueError("Flag failed") @@ -461,21 +587,13 @@ def test_moderate_and_flag_spam_with_auto_delete( # pylint: disable=unused-argu sample_thread_content: dict[str, Any], ) -> None: """Test the module-level function with auto-delete enabled.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() - # Create instance with mocked settings already active - test_service: AIModerationService = AIModerationService() # type: ignore[no-untyped-call] with patch("requests.post", return_value=mock_response), patch( "forum.api.threads.delete_thread" - ), patch("forum.ai_moderation.ai_moderation_service", test_service): + ): result = moderate_and_flag_spam( "spam content", @@ -500,13 +618,7 @@ def test_audit_log_created_for_auto_deleted_content( sample_thread_content: dict[str, Any], ) -> None: """Test that audit log is created with correct actions for auto-deleted content.""" - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = [ - { - "content": '{"classification": "spam", "reasoning": "Spam detected", "confidence_score": 0.9}' - } - ] + mock_response = classifier_response(SPAM_RESPONSE) backend = Mock() user = User.objects.create(username="testuser") diff --git a/tests/test_ai_moderation_backends.py b/tests/test_ai_moderation_backends.py new file mode 100644 index 00000000..38563bfc --- /dev/null +++ b/tests/test_ai_moderation_backends.py @@ -0,0 +1,465 @@ +"""Tests for the AI moderation backend interface and backend selection.""" + +# Fixtures are passed to tests by name, which pylint reads as shadowing; one +# applied only for its side effects reads as an unused argument. +# pylint: disable=redefined-outer-name,unused-argument + +import os +import subprocess +import sys +from types import SimpleNamespace +from typing import Any, Dict, Generator, Optional +from unittest.mock import Mock, patch + +import pytest +import requests +from django.conf import settings as django_settings +from django.core.exceptions import ImproperlyConfigured +from django.test import override_settings + +from forum.ai_moderation.backends import BaseModerationBackend, HTTPModerationBackend +from forum.ai_moderation.backends.base import strip_code_fence +from forum.ai_moderation.backends.xpert import XPertModerationBackend +from forum.ai_moderation.defaults import ( + DEFAULT_CONNECTION_TIMEOUT, + DEFAULT_READ_TIMEOUT, + DEFAULT_SYSTEM_MESSAGE, +) +from forum.ai_moderation.service import AIModerationService +from forum.settings.common import plugin_settings + +SPAM_PAYLOAD = ( + '{"classification": "spam_or_scam", "reasoning": "Spam detected", ' + '"confidence_score": 0.9}' +) + +XPERT_BACKEND = "forum.ai_moderation.backends.xpert.XPertModerationBackend" + + +class StubProviderBackend(HTTPModerationBackend): + """ + A provider backend written the way an Open edX operator would write one. + + It exists to prove the extension point works from outside forum: a class + that subclasses HTTPModerationBackend, describes one provider's request and + response, and is selected by dotted path. Its provider is imaginary -- a + bearer-token JSON API answering ``{"verdict": {"text": ""}}``. + """ + + @property + def api_key(self) -> Optional[str]: + """Credential for the provider.""" + return getattr(django_settings, "AI_MODERATION_API_KEY", None) + + def classify(self, content: str) -> Optional[Dict[str, Any]]: + """Classify content, returning the common moderation result.""" + headers = {"content-type": "application/json"} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + + response_data = self.post( + {"prompt": self.system_message, "input": content}, headers + ) + if response_data is None: + return None + + return self.parse_moderation_payload( + response_data.get("verdict", {}).get("text"), response_data + ) + + +@pytest.fixture +def xpert_settings() -> Generator[None, None, None]: + """Configure the XPert backend.""" + with override_settings( + AI_MODERATION_BACKEND=XPERT_BACKEND, + AI_MODERATION_API_URL="http://xpert.example.com/v1/message", + AI_MODERATION_CLIENT_ID="test-client-id", + AI_MODERATION_SYSTEM_MESSAGE="classify this", + AI_MODERATION_CONNECTION_TIMEOUT=0.5, + AI_MODERATION_READ_TIMEOUT=20, + ): + yield + + +@pytest.fixture +def stub_provider_settings() -> Generator[None, None, None]: + """Configure the out-of-tree stub provider backend.""" + with override_settings( + AI_MODERATION_BACKEND=f"{StubProviderBackend.__module__}.StubProviderBackend", + AI_MODERATION_API_URL="http://provider.example.com/v1/moderate", + AI_MODERATION_API_KEY="test-api-key", + AI_MODERATION_SYSTEM_MESSAGE="classify this", + AI_MODERATION_CONNECTION_TIMEOUT=0.5, + AI_MODERATION_READ_TIMEOUT=20, + ): + yield + + +def xpert_response(payload: str) -> Mock: + """Build a mocked XPert response.""" + response = Mock() + response.status_code = 200 + response.json.return_value = [{"content": payload}] + return response + + +def stub_provider_response(payload: str) -> Mock: + """Build a mocked response from the stub provider.""" + response = Mock() + response.status_code = 200 + response.json.return_value = {"verdict": {"text": payload}} + return response + + +class TestBaseModerationBackend: + """Tests for the backend interface itself.""" + + def test_classify_is_not_implemented(self) -> None: + """The interface carries no behaviour of its own.""" + with pytest.raises(NotImplementedError): + BaseModerationBackend().classify("some content") + + def test_shipped_backend_implements_the_interface(self) -> None: + """XPert is usable wherever a moderation backend is expected.""" + assert isinstance(XPertModerationBackend(), BaseModerationBackend) + + def test_http_base_reads_only_provider_neutral_settings(self) -> None: + """ + The shared HTTP base has no opinion about credentials. + + Auth belongs to the subclass -- XPert puts a client_id in the body, + another provider might send a bearer token or an x-api-key header. + """ + assert not hasattr(HTTPModerationBackend, "api_key") + + with override_settings( + AI_MODERATION_API_URL="http://provider.example.com", + AI_MODERATION_SYSTEM_MESSAGE=None, + ): + backend = XPertModerationBackend() + assert backend.api_url == "http://provider.example.com" + assert backend.system_message == DEFAULT_SYSTEM_MESSAGE + assert backend.timeout == (DEFAULT_CONNECTION_TIMEOUT, DEFAULT_READ_TIMEOUT) + + @pytest.mark.parametrize( + "raw,expected", + [ + ('{"a": 1}', '{"a": 1}'), + ('```json\n{"a": 1}\n```', '{"a": 1}'), + ('```\n{"a": 1}\n```', '{"a": 1}'), + (' {"a": 1} ', '{"a": 1}'), + ], + ) + def test_strip_code_fence(self, raw: str, expected: str) -> None: + """Models routinely wrap their JSON answer in a Markdown fence.""" + assert strip_code_fence(raw) == expected + + +class TestPluginSettings: + """ + Forum declares the AI moderation settings itself, through its plugin settings, + so that no edx-platform change is needed to configure the feature. + """ + + def test_defaults_are_declared(self) -> None: + """The settings a deployment must fill in are declared, and left empty.""" + site = SimpleNamespace(FEATURES={}) + plugin_settings(site) + + assert site.AI_MODERATION_SYSTEM_MESSAGE == DEFAULT_SYSTEM_MESSAGE + assert site.AI_MODERATION_CONNECTION_TIMEOUT == DEFAULT_CONNECTION_TIMEOUT + assert site.AI_MODERATION_READ_TIMEOUT == DEFAULT_READ_TIMEOUT + + # Nothing is guessed on a deployment's behalf -- least of all a provider. + assert site.AI_MODERATION_BACKEND is None + assert site.AI_MODERATION_API_URL is None + assert site.AI_MODERATION_USER_ID is None + + def test_no_provider_specific_settings_are_declared(self) -> None: + """ + Provider settings belong to the backend that reads them. + + Declaring, say, a bearer token here would bake one provider's auth + scheme into the generic layer. + """ + site = SimpleNamespace(FEATURES={}) + plugin_settings(site) + + assert not hasattr(site, "AI_MODERATION_API_KEY") + assert not hasattr(site, "AI_MODERATION_MODEL") + assert not hasattr(site, "AI_MODERATION_CLIENT_ID") + + def test_configured_values_are_never_overridden(self) -> None: + """ + Plugin settings are applied after a deployment's own configuration is + read, so a site that already chose XPert keeps it. + """ + site = SimpleNamespace( + FEATURES={}, + AI_MODERATION_BACKEND=XPERT_BACKEND, + AI_MODERATION_API_URL="https://xpert.example.com/v1/message", + AI_MODERATION_SYSTEM_MESSAGE="the deployed prompt", + AI_MODERATION_CONNECTION_TIMEOUT=0.5, + AI_MODERATION_READ_TIMEOUT=20, + AI_MODERATION_USER_ID=758316, + ) + plugin_settings(site) + + assert site.AI_MODERATION_BACKEND == XPERT_BACKEND + assert site.AI_MODERATION_API_URL == "https://xpert.example.com/v1/message" + assert site.AI_MODERATION_SYSTEM_MESSAGE == "the deployed prompt" + assert site.AI_MODERATION_CONNECTION_TIMEOUT == 0.5 + assert site.AI_MODERATION_READ_TIMEOUT == 20 + assert site.AI_MODERATION_USER_ID == 758316 + + def test_settings_module_imports_before_apps_are_ready(self) -> None: + """ + Plugin settings are imported while Django settings are still being + assembled, so nothing on that path may reach a Django model. A fresh + interpreter is the only honest way to check: this process has long since + imported the service. + """ + result = subprocess.run( + [sys.executable, "-c", "import forum.settings.common"], + env={**os.environ, "DJANGO_SETTINGS_MODULE": "forum.settings.test"}, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +class TestBackendSelection: + """Tests for choosing a backend with AI_MODERATION_BACKEND.""" + + def test_no_provider_is_shipped_by_default(self) -> None: + """ + Forum defines the interface and no provider, so an unconfigured + deployment is told exactly what to set. + """ + service = AIModerationService() + with override_settings(AI_MODERATION_BACKEND=None): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "AI_MODERATION_BACKEND" in str(excinfo.value) + assert "BaseModerationBackend" in str(excinfo.value) + + def test_configured_backend_is_loaded(self, xpert_settings: None) -> None: + """The configured dotted path decides which provider is used.""" + service = AIModerationService() + assert isinstance(service.moderation_backend, XPertModerationBackend) + + def test_out_of_tree_backend_is_loaded(self, stub_provider_settings: None) -> None: + """A backend that does not ship with forum plugs in the same way.""" + service = AIModerationService() + assert isinstance(service.moderation_backend, StubProviderBackend) + + def test_backend_is_reloaded_when_the_setting_changes( + self, xpert_settings: None + ) -> None: + """The cached backend does not outlive the setting that chose it.""" + service = AIModerationService() + assert isinstance(service.moderation_backend, XPertModerationBackend) + + with override_settings( + AI_MODERATION_BACKEND=( + f"{StubProviderBackend.__module__}.StubProviderBackend" + ) + ): + assert isinstance(service.moderation_backend, StubProviderBackend) + + assert isinstance(service.moderation_backend, XPertModerationBackend) + + def test_unimportable_backend_reports_a_clear_error(self) -> None: + """A typo in AI_MODERATION_BACKEND says exactly what is wrong.""" + service = AIModerationService() + with override_settings(AI_MODERATION_BACKEND="forum.nope.NoSuchBackend"): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "AI_MODERATION_BACKEND" in str(excinfo.value) + assert "forum.nope.NoSuchBackend" in str(excinfo.value) + + def test_backend_must_implement_the_interface(self) -> None: + """Pointing the setting at some other class is a configuration error.""" + service = AIModerationService() + with override_settings( + AI_MODERATION_BACKEND="forum.ai_moderation.service.AIModerationService" + ): + with pytest.raises(ImproperlyConfigured) as excinfo: + _ = service.moderation_backend + + assert "BaseModerationBackend" in str(excinfo.value) + + @pytest.mark.parametrize("backend_path", [None, "forum.nope.NoSuchBackend"]) + def test_misconfiguration_does_not_break_moderation( + self, backend_path: Optional[str] + ) -> None: + """Unset or wrong, a configuration error is logged, not raised at the caller.""" + service = AIModerationService() + classify = service._classify # pylint: disable=protected-access + with override_settings(AI_MODERATION_BACKEND=backend_path): + assert classify("content") is None + + +class TestXPertModerationBackend: + """Tests for the edX-specific XPert backend.""" + + def test_request_format(self, xpert_settings: None) -> None: + """XPert authorises on client_id and takes the prompt at the top level.""" + with patch( + "requests.post", return_value=xpert_response(SPAM_PAYLOAD) + ) as mock_post: + XPertModerationBackend().classify("check me") + + args, kwargs = mock_post.call_args + assert args[0] == "http://xpert.example.com/v1/message" + assert "Authorization" not in kwargs["headers"] + assert kwargs["timeout"] == (0.5, 20) + assert kwargs["json"] == { + "messages": [{"role": "user", "content": "check me"}], + "client_id": "test-client-id", + "system_message": "classify this", + } + + def test_response_is_normalized(self, xpert_settings: None) -> None: + """The first element's content holds the classifier JSON.""" + raw = xpert_response(SPAM_PAYLOAD) + with patch("requests.post", return_value=raw): + result = XPertModerationBackend().classify("check me") + + assert result is not None + assert result["classification"] == "spam_or_scam" + assert result["reasoning"] == "Spam detected" + assert result["confidence_score"] == 0.9 + assert result["full_api_response"] == raw.json.return_value + + def test_default_system_message_is_used_when_unset(self) -> None: + """Standing up a backend does not also mean writing a prompt.""" + with override_settings( + AI_MODERATION_API_URL="http://xpert.example.com/v1/message", + AI_MODERATION_SYSTEM_MESSAGE=None, + ), patch( + "requests.post", return_value=xpert_response(SPAM_PAYLOAD) + ) as mock_post: + XPertModerationBackend().classify("check me") + + assert mock_post.call_args.kwargs["json"]["system_message"] == ( + DEFAULT_SYSTEM_MESSAGE + ) + + def test_missing_api_url_is_a_configuration_error(self) -> None: + """No endpoint means no request at all.""" + with override_settings(AI_MODERATION_API_URL=None), patch( + "requests.post" + ) as mock_post: + assert XPertModerationBackend().classify("check me") is None + + mock_post.assert_not_called() + + @pytest.mark.parametrize( + "body", + [ + {}, + [], + ["not a dict"], + [{}], + [{"content": "not json"}], + [{"content": "[1, 2, 3]"}], + ], + ) + def test_unusable_responses_return_none( + self, body: Any, xpert_settings: None + ) -> None: + """Anything that is not a parsable verdict fails closed.""" + response = Mock() + response.status_code = 200 + response.json.return_value = body + + with patch("requests.post", return_value=response): + assert XPertModerationBackend().classify("check me") is None + + def test_request_failure_returns_none(self, xpert_settings: None) -> None: + """A network failure degrades moderation instead of raising.""" + with patch("requests.post", side_effect=requests.ConnectionError("refused")): + assert XPertModerationBackend().classify("check me") is None + + def test_timeout_returns_none(self, xpert_settings: None) -> None: + """A slow classifier degrades moderation instead of raising.""" + with patch("requests.post", side_effect=requests.Timeout("too slow")): + assert XPertModerationBackend().classify("check me") is None + + def test_http_error_returns_none(self, xpert_settings: None) -> None: + """A non-2xx answer degrades moderation instead of raising.""" + failed = requests.Response() + failed.status_code = 500 + response = Mock() + response.raise_for_status.side_effect = requests.HTTPError( + "500", response=failed + ) + + with patch("requests.post", return_value=response): + assert XPertModerationBackend().classify("check me") is None + + +class TestCustomProviderBackend: + """ + Tests for what an operator gets from the shared HTTP base. + + These exercise StubProviderBackend, whose only provider-specific code is the + request body, the auth header and where the verdict lives in the response. + Everything else -- timeouts, error handling, fence stripping, normalization + -- is inherited. + """ + + def test_request_format(self, stub_provider_settings: None) -> None: + """The backend decides its own body and its own auth scheme.""" + with patch( + "requests.post", return_value=stub_provider_response(SPAM_PAYLOAD) + ) as mock_post: + StubProviderBackend().classify("check me") + + args, kwargs = mock_post.call_args + assert args[0] == "http://provider.example.com/v1/moderate" + assert kwargs["headers"]["Authorization"] == "Bearer test-api-key" + assert kwargs["timeout"] == (0.5, 20) + assert kwargs["json"] == {"prompt": "classify this", "input": "check me"} + + def test_response_is_normalized(self, stub_provider_settings: None) -> None: + """A verdict from anywhere in the response reaches the common result.""" + raw = stub_provider_response(SPAM_PAYLOAD) + with patch("requests.post", return_value=raw): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["classification"] == "spam_or_scam" + assert result["reasoning"] == "Spam detected" + assert result["confidence_score"] == 0.9 + assert result["full_api_response"] == raw.json.return_value + + def test_fenced_response_is_parsed(self, stub_provider_settings: None) -> None: + """A JSON answer wrapped in a Markdown fence is still understood.""" + fenced = f"```json\n{SPAM_PAYLOAD}\n```" + with patch("requests.post", return_value=stub_provider_response(fenced)): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["classification"] == "spam_or_scam" + + def test_extra_keys_are_preserved(self, stub_provider_settings: None) -> None: + """Anything else the classifier returned is kept for the audit log.""" + payload = '{"classification": "not_spam", "categories": ["none"]}' + with patch("requests.post", return_value=stub_provider_response(payload)): + result = StubProviderBackend().classify("check me") + + assert result is not None + assert result["categories"] == ["none"] + assert result["confidence_score"] is None + + def test_request_failure_returns_none(self, stub_provider_settings: None) -> None: + """Error handling is inherited, not reimplemented per provider.""" + with patch("requests.post", side_effect=requests.ConnectionError("refused")): + assert StubProviderBackend().classify("check me") is None