diff --git a/k8s/welearn-api/values.yaml b/k8s/welearn-api/values.yaml index 231ade39..a83132a7 100644 --- a/k8s/welearn-api/values.yaml +++ b/k8s/welearn-api/values.yaml @@ -39,6 +39,7 @@ config: AZURE_API_VERSION: "2024-08-01-preview" LLM_MODEL_NAME: gpt-oss-120b LLM_TEMPERATURE: 0.2 + TUTOR_SINGLE_PASS: "true" MISTRAL_LLM_MODEL_NAME: mistral-small-latest AZURE_BAML_LLM_FRANCE_BASE_URL: https://welearn-mistral.services.ai.azure.com/openai/v1/ AZURE_BAML_LLM_SWEDEN_BASE_URL: https://welearn-mistral-sweden.services.ai.azure.com/openai/v1/ diff --git a/src/app/core/config.py b/src/app/core/config.py index 59c28570..7c3d21ae 100644 --- a/src/app/core/config.py +++ b/src/app/core/config.py @@ -48,6 +48,8 @@ def get_api_version(self) -> dict: LLM_MODEL_NAME: str LLM_TEMPERATURE: float + # old tutor: one structured LLM call instead of the 3-agent chain (False = revert) + TUTOR_SINGLE_PASS: bool = True ENV: str # LANGSMITH / LANGCHAIN TRACING diff --git a/src/app/shared/utils/utils.py b/src/app/shared/utils/utils.py index b69a9841..1f25ea6b 100644 --- a/src/app/shared/utils/utils.py +++ b/src/app/shared/utils/utils.py @@ -39,9 +39,9 @@ def extract_doc_info(documents: list[ScoredPoint]) -> list[dict]: """ return [ { - "title": getattr(doc.payload, "document_title", ""), # type: ignore - "url": getattr(doc.payload, "document_url", ""), # type: ignore - "content": getattr(doc.payload, "slice_content", ""), # type: ignore + "title": doc.payload.get("document_title", ""), + "url": doc.payload.get("document_url", ""), + "content": doc.payload.get("slice_content", ""), } for doc in documents if doc.payload is not None diff --git a/src/app/tests/services/tutor/test_syllabus.py b/src/app/tests/services/tutor/test_syllabus.py new file mode 100644 index 00000000..a5e07cc3 --- /dev/null +++ b/src/app/tests/services/tutor/test_syllabus.py @@ -0,0 +1,139 @@ +from src.app.tutor.service.models import ( + DraftAssessment, + DraftCompetency, + DraftOutcome, + DraftSession, + DraftSyllabus, +) +from src.app.tutor.service.syllabus import ( + compute_limits, + detect_syllabus_lang, + render_markdown, + render_references, + split_references, + trim_chatter, +) + + +def make_draft() -> DraftSyllabus: + return DraftSyllabus( + course_title="Économie circulaire", + description="Desc", + objectives=["O1", "O2", "O3", "O4", "O5"], + outcomes=[ + DraftOutcome(text=f"Out {i}", objective_numbers=[1]) for i in range(1, 10) + ], + competencies=[ + DraftCompetency(text="Travail en équipe", outcome_numbers=[1]), + DraftCompetency( + text="Systémique", greencomp_code="2.1", outcome_numbers=[2] + ), + DraftCompetency(text="Non liée", greencomp_code="2.2", outcome_numbers=[]), + DraftCompetency(text="Critique", greencomp_code="2.2", outcome_numbers=[3]), + DraftCompetency(text="Doublon", greencomp_code="2.2", outcome_numbers=[1]), + DraftCompetency( + text="Faux code", greencomp_code="9.9", outcome_numbers=[1] + ), + ], + assessment=[ + DraftAssessment(method="Projet", weight="100%", outcome_numbers=[1, 99]) + ], + schedule=[DraftSession(topics="A | B", outcome_numbers=[1], class_plan="x\ny")], + ) + + +def test_compute_limits(): + assert compute_limits(None).sessions is None + assert compute_limits(None).outcomes == 5 # sized like a semester course + assert compute_limits("4 semaines").outcomes == 3 + assert compute_limits("8 semaines").outcomes == 4 # grows with duration + assert compute_limits("8 semaines").objectives == 3 + assert compute_limits("12 semaines").sessions == 12 + assert compute_limits("30h").sessions == 10 + assert compute_limits("1 semestre").sessions == 12 + assert compute_limits("40 weeks").outcomes == 7 # ceiling + assert compute_limits("40 weeks").objectives == 4 + + +def test_render_markdown_fr(): + md = render_markdown(make_draft(), "fr", compute_limits("12 semaines")) + assert md.startswith("# Économie circulaire") + assert "## 3. Résultats d'apprentissage" in md and "Learning" not in md + assert "**RA5**" in md and "**RA6**" not in md # capped at 5 outcomes + assert "4. O4" not in md # capped at 3 objectives + assert "Non liée" not in md and "Doublon" not in md # unlinked / duplicate code + # official name in the syllabus language, GreenComp listed first + assert "**GreenComp 2.1 – Pensée systémique** : Systémique *(RA2)*" in md + assert "**GreenComp 2.2 – Pensée critique** : Critique" in md + assert md.index("GreenComp 2.1") < md.index("Travail en équipe") + assert "- Faux code" in md # unknown code -> plain competency + assert "RA99" not in md + assert "| 1 | A / B | RA1 | x
y |" in md + + +def test_references_are_deterministic(): + refs = render_references( + [ + {"title": "Doc A", "url": "https://a.org"}, + {"title": "Doc A", "url": "https://a.org"}, # second slice of same doc + {"title": "Doc B", "url": "https://b.org"}, + ], + "fr", + ) + assert refs.startswith("## 7. Références") + assert refs.count("https://a.org") == 1 + assert '[lien]' in refs + assert "Aucune" in render_references([], "fr") + # a resource without title or url is never rendered + assert "Aucune" in render_references( + [{"title": "", "url": "https://c.org"}, {"title": "Doc C", "url": ""}], "fr" + ) + + +def test_split_and_trim(): + md = ( + "Voici votre syllabus :\n# T\n## 6. Programme\n| a |\n|---|\n" + "Ce syllabus est aligné avec GreenComp.\n## 7. Références\n- Doc x" + ) + body, refs = split_references(md) + assert refs == "## 7. Références\n- Doc x" + assert trim_chatter(body) == "# T\n## 6. Programme\n| a |\n|---|" + assert split_references("# T\nno refs") == ("# T\nno refs", "") + + +def test_detect_syllabus_lang(): + assert ( + detect_syllabus_lang(render_markdown(make_draft(), "fr", compute_limits(None))) + == "fr" + ) + assert ( + detect_syllabus_lang(render_markdown(make_draft(), "en", compute_limits(None))) + == "en" + ) + assert detect_syllabus_lang("# free text") == "unknown" + + +def test_generate_syllabus_retries_once_then_raises(): + import asyncio + + import pytest + from langchain_core.runnables import RunnableConfig, RunnableLambda + + from src.app.tutor.service.models import MessageWithResources + from src.app.tutor.service.syllabus import generate_syllabus + + answers = [None, make_draft()] # first answer invalid, second valid + + class FakeModel: + def with_structured_output(self, schema): + return RunnableLambda(lambda _: answers.pop(0)) + + msg = MessageWithResources( + lang="fr", content=[], themes=[], summary=[], resources=[], duration=None + ) + md = asyncio.run(generate_syllabus(msg, FakeModel(), [], RunnableConfig())) # type: ignore + assert md.startswith("# Économie circulaire") and not answers + + answers[:] = [None, None] + with pytest.raises(Exception): + asyncio.run(generate_syllabus(msg, FakeModel(), [], RunnableConfig())) # type: ignore diff --git a/src/app/tests/services/tutor/test_utils.py b/src/app/tests/services/tutor/test_utils.py index 74925e79..8ce3e2c6 100644 --- a/src/app/tests/services/tutor/test_utils.py +++ b/src/app/tests/services/tutor/test_utils.py @@ -42,18 +42,18 @@ def test_build_system_message_without_optional_params(self): def test_extract_doc_info(self): # Create mock documents doc1 = Mock(spec=ScoredPoint) - doc1.payload = Mock( - document_title="Test Doc 1", - document_url="http://test1.com", - slice_content="Content 1", - ) + doc1.payload = { + "document_title": "Test Doc 1", + "document_url": "http://test1.com", + "slice_content": "Content 1", + } doc2 = Mock(spec=ScoredPoint) - doc2.payload = Mock( - document_title="Test Doc 2", - document_url="http://test2.com", - slice_content="Content 2", - ) + doc2.payload = { + "document_title": "Test Doc 2", + "document_url": "http://test2.com", + "slice_content": "Content 2", + } doc3 = Mock(spec=ScoredPoint) doc3.payload = None # Test case for None payload diff --git a/src/app/tutor/api/router.py b/src/app/tutor/api/router.py index cb246521..bd48453a 100644 --- a/src/app/tutor/api/router.py +++ b/src/app/tutor/api/router.py @@ -6,9 +6,11 @@ BackgroundTasks, Depends, File, + HTTPException, Request, Response, UploadFile, + status, ) from src.app.baml_client.async_client import b, types @@ -22,9 +24,6 @@ from src.app.shared.utils.dependencies import get_settings from src.app.shared.utils.requests import extract_session_cookie from src.app.shared.utils.utils import get_files_content - -# from src.app.tutor.service.agents import TEMPLATES -from src.app.tutor.service.agents import TEMPLATES from src.app.tutor.service.models import ( CompetencyMappingRequest, CourseDescriptionRequest, @@ -48,7 +47,7 @@ extractor_user_prompt, summaries_schema, ) -from src.app.tutor.service.tutor import tutor_manager +from src.app.tutor.service.tutor import apply_feedback, tutor_manager from src.app.utils.logger import logger as utils_logger logger = utils_logger(__name__) @@ -197,7 +196,6 @@ async def tutor_search_extract( return resp -@with_backoff() @router.post("/syllabus") async def create_syllabus( request: Request, @@ -214,14 +212,26 @@ async def create_syllabus( "query_extracts_count": len(body.extracts), "documents_count": len(body.documents), } - results = await tutor_manager(body, lang, settings, trace_context=trace_context) + try: + results = await tutor_manager(body, lang, settings, trace_context=trace_context) + except Exception as e: + logger.error(f"Syllabus generation failed: {e}") + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail={ + "message": "Syllabus generation failed, please retry", + "code": "SYLLABUS_GENERATION_FAILED", + }, + ) - # TODO: handle errors + # the final syllabus is the last item: the only one in single-pass mode, + # the PedagogicalEngineerAgent's in the legacy 3-agent chain + final_syllabus = results[-1] message_id = await data_collection.register_syllabus_data( session_id=session_id, input_data=body, - agent_answer=results[0].content if results else "", + agent_answer=final_syllabus.content, feature="syllabus_creation", ) @@ -233,81 +243,24 @@ async def create_syllabus( ) -feedback_prompt = """ -You are a pedagogical engineer and are given a syllabus and a feedback. by the teacher that will teach the course. -Your responsibility is to analyze the syllabus and return an improved version of it in a markdown format. Do not add the backticks and the markdown mention. -It is important to take into account the feedback given by the teacher and to keep the syllabus structure. -The syllabus structure is: - {syllabus_structure} - -To be able to do that, the assistant gives you: - - the syllabus of the course - - the feedback given by the teacher - - a list of documents related to the course - - a list of extracts from a document of interest - - the themes that the course is related to - -You will respond with the syllabus. Do not provide explanations or notes -""" - -feedback_assistant_prompt = """ -IMPORTANT: you must follow the syllabus structure given by the system message. -and the respect the format of the original syllabus. -Keep the same language as the original syllabus. - -here is the original syllabus: - {syllabus} - -take into account the user feedback: - {feedback} - -keep the references section with the format document.title, references are based on these documents: - {documents} - -for more context, here are the extracts of the original document the user sent to build the syllabus from. Extracts: - {extracts} - -and the themes extracted from those documents: - {themes} -""" - - -@with_backoff() @router.post("/syllabus/feedback") async def handle_syllabus_feedback( request: Request, body: SyllabusFeedback, - chatfactory=Depends(get_chat_service), data_collection=Depends(get_data_collection_service), + settings: Settings = Depends(get_settings), ): session_id = extract_session_cookie(request) - messages = [ - { - "role": "system", - "content": feedback_prompt.format(syllabus_structure=TEMPLATES), - }, - { - "role": "user", - "content": feedback_assistant_prompt.format( - syllabus=body.syllabus[0], - feedback=body.feedback, - documents=body.documents, - extracts="\n".join([extract.summary for extract in body.extracts]), - themes=(", ").join( - [ - (", ").join([theme["theme"] for theme in extract.themes]) - for extract in body.extracts - ] - ), - ), - }, - ] - try: - syllabus = await chatfactory.syllabus_feedback_completion( - max_tokens=20000, - messages=messages, + syllabus = await apply_feedback( + body, + settings, + trace_context={ + "endpoint": request.url.path, + "feature": "syllabus_feedback", + "session_id": str(session_id) if session_id else None, + }, ) await data_collection.register_syllabus_data( diff --git a/src/app/tutor/domain/template.md b/src/app/tutor/domain/template.md index e33804bb..5aab077a 100644 --- a/src/app/tutor/domain/template.md +++ b/src/app/tutor/domain/template.md @@ -22,4 +22,4 @@ Course title: Small title for the course table structure: | week | Topics | Learning Outcomes | Class Plan | 7. References: In this section should be added all sources used to generate the syllabus, including any source documents or WeLearn documents used to construct the syllabus. IMPORTANT: Do not remove any existent references in this section - Example of references: Document title [link] + Example of references: Document title [link] diff --git a/src/app/tutor/service/agents.py b/src/app/tutor/service/agents.py index b046733c..ea02fcb6 100644 --- a/src/app/tutor/service/agents.py +++ b/src/app/tutor/service/agents.py @@ -9,7 +9,12 @@ from langchain_core.runnables import RunnableConfig from src.app.shared.utils.utils import build_system_message -from src.app.tutor.service.models import MessageWithResources, SyllabusResponseAgent +from src.app.tutor.service.models import ( + ExtractorOutput, + MessageWithResources, + SyllabusResponseAgent, +) +from src.app.tutor.service.syllabus import COUNTS_GUIDE, GREENCOMP_NAMES_GUIDE from src.app.utils.logger import logger as utils_logger logger = utils_logger(__name__) @@ -115,8 +120,13 @@ def __init__( ) async def generate(self, message: MessageWithResources) -> SyllabusResponseAgent: - DISCIPLINARY_SKILLS = get_disciplinary_skills() - disciplinary_skills_sentences = "\n\nThe syllabus should also contribute to build the following disciplinary skills:\n-" + skills = get_disciplinary_skills().get(message.discipline, []) + disciplinary_skills_sentences = ( + "\n\nThe syllabus should also contribute to build the following disciplinary skills:\n- " + + "\n- ".join(skills) + if skills + else "" + ) contents = "summary :".join(message.summary) themes = ",".join([theme["theme"] for theme in message.themes]) prompt = ( @@ -125,7 +135,7 @@ async def generate(self, message: MessageWithResources) -> SyllabusResponseAgent f"The syllabus should be written in lang: {message.lang} the section names must also be written in {message.lang}, this is important \n\nTEXT CONTENTS:\n{contents}\n\n" f"THEMES:\n{themes} \n\nTake into account the users input courses title, level, duration and " f"description: {message.course_title}, {message.level}, {message.duration}, {message.description}." - f"{(disciplinary_skills_sentences.join(DISCIPLINARY_SKILLS[message.discipline])) if message.discipline in DISCIPLINARY_SKILLS.keys() else ''}" + f"{disciplinary_skills_sentences}" ) response = await self.run(prompt) return SyllabusResponseAgent(content=response, source=self.name) @@ -240,3 +250,64 @@ async def refine(self, syllabus: SyllabusResponseAgent) -> SyllabusResponseAgent ) response = await self.run(prompt) return SyllabusResponseAgent(content=response, source=self.name) + + +class FeedbackAgent(TutorChatAgent): + """Applies the teacher's modification request to an existing syllabus.""" + + agent_name = "FeedbackAgent" + agent_tag = "feedback" + + def __init__( + self, + model: BaseChatModel, + greencomp_competencies: str, + trace_tags: list[str] | None = None, + trace_metadata: dict[str, Any] | None = None, + ) -> None: + system_prompt = build_system_message( + role="a pedagogical engineer who revises a syllabus according to the feedback of the teacher who will teach the course", + backstory="You are an expert in competency-based course design, active learning and the EU GreenComp framework. You make precise, targeted edits and keep everything else intact.", + goal="Apply the teacher's feedback to the syllabus and return the full revised syllabus in markdown", + instructions=( + "1. Apply the teacher's feedback exactly; change only what the feedback requires. " + "2. Keep the same structure, the same section headings word for word, the same numbering " + "(e.g. LO1/AA1) and the course schedule as a markdown table. " + "3. Keep the language of the syllabus. " + "4. Keep the number of learning objectives, learning outcomes and competencies unless the feedback " + "requires otherwise. If the feedback changes the course duration or number of sessions, rescale the " + "course: one schedule row per session, and adjust the objectives and outcomes (renumbering all " + f"references to them) to this guide: {COUNTS_GUIDE}. " + "5. Keep at least one GreenComp competency. A competency that corresponds to a GreenComp " + "competency is written as '**GreenComp – ** : *()*'. Official names (English / French): " + f"{GREENCOMP_NAMES_GUIDE}. " + "6. Keep class plans student-centred and active (project/problem-based learning, case studies, " + "debates, peer instruction, workshops); teacher input stays short. " + "7. Never comment on your changes or design choices inside the syllabus. " + "8. The references section is handled separately: do not write any references or links. " + f"Here is the GreenComp framework for reference: {greencomp_competencies}" + ), + expected_output="Only the revised syllabus in markdown, starting with its title heading. No backticks, explanations, notes or comments before or after it.", + ) + super().__init__( + model, + system_prompt, + trace_tags=trace_tags, + trace_metadata=trace_metadata, + ) + + async def apply( + self, syllabus: str, feedback: str, extracts: list[ExtractorOutput] + ) -> str: + summaries = "\n".join(extract.summary for extract in extracts) + themes = ", ".join( + theme["theme"] for extract in extracts for theme in extract.themes + ) + prompt = ( + f"SYLLABUS:\n{syllabus}\n\n" + f"TEACHER'S FEEDBACK:\n{feedback}\n\n" + f"Context - summaries of the teacher's documents:\n{summaries}\n\n" + f"Context - themes: {themes}" + ) + return await self.run(prompt) diff --git a/src/app/tutor/service/models.py b/src/app/tutor/service/models.py index 1933846e..cb7056a6 100644 --- a/src/app/tutor/service/models.py +++ b/src/app/tutor/service/models.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field from qdrant_client.models import ScoredPoint from src.app.baml_client import types @@ -57,6 +57,76 @@ class SyllabusUserUpdate(BaseModel): syllabus: str +@dataclass +class Limits: + """Target counts for a syllabus, derived from the course duration.""" + + sessions: int | None + objectives: int + outcomes: int + competencies: int + + +class DraftOutcome(BaseModel): + text: str = Field( + description="Student-centred, measurable outcome starting with an action verb, " + "including how it is demonstrated (the class activity or assessment)." + ) + objective_numbers: list[int] = Field( + description="1-based numbers of the learning objectives this outcome serves." + ) + + +class DraftCompetency(BaseModel): + text: str = Field(description="Transferable competency developed by the course.") + greencomp_code: str | None = Field( + default=None, + description="Code of the GreenComp competency this corresponds to, like '2.1'; " + "null only if it has no GreenComp equivalent.", + ) + outcome_numbers: list[int] = Field( + description="1-based numbers of the learning outcomes that develop it." + ) + + +class DraftAssessment(BaseModel): + method: str = Field(description="Assessment method and what students produce.") + weight: str = Field(description="Share of the final grade, e.g. '40%'.") + outcome_numbers: list[int] = Field( + description="1-based numbers of the learning outcomes it evaluates." + ) + + +class DraftSession(BaseModel): + topics: str = Field(description="Topics covered in this session.") + outcome_numbers: list[int] = Field( + description="1-based numbers of the learning outcomes targeted." + ) + class_plan: str = Field( + description="Student-centred plan for the session: the active-learning " + "activities students do, with timing; teacher input kept short. Never a " + "placeholder or ellipsis." + ) + + +class DraftSyllabus(BaseModel): + """Structured syllabus returned by the LLM; references are added in code.""" + + course_title: str + description: str = Field( + description="Course overview addressed to students: content, relevance to the " + "academic program and to sustainability challenges. No comments about how the " + "syllabus was designed." + ) + objectives: list[str] = Field( + description="Broad, teacher-centred learning objectives." + ) + outcomes: list[DraftOutcome] + competencies: list[DraftCompetency] + assessment: list[DraftAssessment] + schedule: list[DraftSession] + + class MessageWithAnalysis(BaseModel): content: Dict source: str = "default" diff --git a/src/app/tutor/service/syllabus.py b/src/app/tutor/service/syllabus.py new file mode 100644 index 00000000..adbbe940 --- /dev/null +++ b/src/app/tutor/service/syllabus.py @@ -0,0 +1,364 @@ +"""Single-pass syllabus generation (old tutor). + +The LLM fills a DraftSyllabus; headings, limits, GreenComp filtering and the +references section are handled here in code, so they can't be hallucinated. +""" + +import re +from math import ceil + +from langchain_core.exceptions import OutputParserException +from langchain_core.language_models import BaseChatModel +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnableConfig +from pydantic import ValidationError + +from src.app.tutor.service.models import DraftSyllabus, Limits, MessageWithResources +from src.app.utils.logger import logger as utils_logger + +logger = utils_logger(__name__) + +GREENCOMP_COMPETENCIES = ( + "Here are the GreenComp competencies: " + "url: https://joint-research-centre.ec.europa.eu/greencomp-european-sustainability-competence-framework_en " + "1.1 Valuing sustainability: To reflect on personal values; identify and explain how values vary among people " + "and over time, while critically evaluating how they align with sustainability values. " + "1.2 Supporting fairness: To support equity and justice for current and future generations and learn from previous " + "generations for sustainability. " + "1.3 Promoting nature: To acknowledge that humans are part of nature; and to respect the needs and rights of other " + "species and of nature itself in order to restore and regenerate healthy and resilient ecosystems. " + "2.1 Systems thinking: To approach a sustainability problem from all sides; to consider time, space and context in " + "order to understand how elements interact within and between systems. " + "2.2 Critical thinking: To assess information and arguments, identify assumptions, challenge the status quo, and " + "reflect on how personal, social and cultural backgrounds influence thinking and conclusions. " + "2.3 Problem framing: To formulate current or potential challenges as a sustainability problem in terms of " + "difficulty, people involved, time and geographical scope, in order to identify suitable approaches to anticipating " + "and preventing problems, and to mitigating and adapting to already existing problems. " + "3.1 Futures literacy: To envision alternative sustainable futures by imagining and developing alternative scenarios " + "and identifying the steps needed to achieve a preferred sustainable future. " + "3.2 Adaptability: To manage transitions and challenges in complex sustainability situations and make decisions " + "related to the future in the face of uncertainty, ambiguity and risk. " + "3.3 Exploratory thinking: To adopt a relational way of thinking by exploring and linking different disciplines, " + "using creativity and experimentation with novel ideas or methods. " + "4.1 Political agency: To navigate the political system, identify political responsibility and accountability for " + "unsustainable behaviour, and demand effective policies for sustainability. " + "4.2 Collective action: To act for change in collaboration with others. " + "4.3 Individual initiative: To identify own potential for sustainability and to actively contribute to improving " + "prospects for the community and the planet." +) +# official names; French from the JRC's French edition of GreenComp (JRC128040) +GREENCOMP_NAMES = { + "1.1": { + "en": "Valuing sustainability", + "fr": "Accorder de la valeur à la durabilité", + }, + "1.2": {"en": "Supporting fairness", "fr": "Encourager l'équité"}, + "1.3": {"en": "Promoting nature", "fr": "Promouvoir la nature"}, + "2.1": {"en": "Systems thinking", "fr": "Pensée systémique"}, + "2.2": {"en": "Critical thinking", "fr": "Pensée critique"}, + "2.3": {"en": "Problem framing", "fr": "Cadrage des problèmes"}, + "3.1": {"en": "Futures literacy", "fr": "Littératie des futurs"}, + "3.2": {"en": "Adaptability", "fr": "Adaptabilité"}, + "3.3": {"en": "Exploratory thinking", "fr": "Pensée exploratoire"}, + "4.1": {"en": "Political agency", "fr": "Agentivité politique"}, + "4.2": {"en": "Collective action", "fr": "Action collective"}, + "4.3": {"en": "Individual initiative", "fr": "Initiative individuelle"}, +} +GREENCOMP_NAMES_GUIDE = "; ".join( + f"{code} {names['en']} / {names['fr']}" for code, names in GREENCOMP_NAMES.items() +) +MIN_GREENCOMP = 1 + +# (up to N sessions, objectives, outcomes, competencies incl. GreenComp). +# Teaching-centre guidance is 3-5 / 4-8 outcomes per full course, ECTS 6-8 per module, +# so outcomes grow slowly with length and stay under 8. +COUNTS_BY_SESSIONS = [ + (4, 2, 3, 3), + (8, 3, 4, 3), + (12, 3, 5, 4), + (16, 4, 6, 4), + (20, 4, 7, 4), +] +COUNTS_GUIDE = "; ".join( + f"up to {n} sessions: {o} objectives, {lo} outcomes, {c} competencies" + for n, o, lo, c in COUNTS_BY_SESSIONS +) + +HEADINGS = { + "en": { + "description": "1. Course Description", + "objectives": "2. Learning Objectives", + "outcomes": "3. Learning Outcomes", + "competencies": "4. Competencies Developed", + "assessment": "5. Assessment Methods", + "schedule": "6. Course Schedule", + "references": "7. References", + "table": "| Week | Topics | Learning Outcomes | Class Plan |", + "lo": "LO", + "objectives_ref": "Objectives:", + "link": "link", + "no_references": "No WeLearn resource was used.", + }, + "fr": { + "description": "1. Description du cours", + "objectives": "2. Objectifs d'apprentissage", + "outcomes": "3. Résultats d'apprentissage", + "competencies": "4. Compétences développées", + "assessment": "5. Modalités d'évaluation", + "schedule": "6. Programme du cours", + "references": "7. Références", + "table": "| Semaine | Thèmes | Résultats d'apprentissage | Plan de séance |", + "lo": "RA", + "objectives_ref": "Objectifs :", + "link": "lien", + "no_references": "Aucune ressource WeLearn n'a été utilisée.", + }, +} +LANG_NAMES = {"en": "English", "fr": "French"} + +# matches a references heading in LLM/user markdown: "## 7. Références", "**References**", ... +_REFERENCES_HEADING = re.compile( + r"^\s*(#+\s*|\*\*\s*)?(\d+\.?\s*)?(r[ée]f[ée]rences?|bibliograph)", re.IGNORECASE +) + + +def compute_limits(duration: str | None) -> Limits: + # ponytail: first number + unit in free text ("12 semaines", "30h"); a structured + # duration field in the client would remove the guessing. + match = re.search(r"(\d+)\s*([a-zA-Zéè]*)", duration or "") + sessions = None + if match: + sessions, unit = int(match[1]), match[2].lower() + if unit.startswith(("h", "hour", "heure")): + sessions = ceil(sessions / 3) + elif unit.startswith(("mo", "month")): + sessions *= 4 + elif unit.startswith("sem") and "semaine" not in unit: # semestre/semester + sessions *= 12 + sessions = max(1, min(sessions, 20)) + n = sessions or 12 # unknown duration: size it like a standard semester course + _, objectives, outcomes, competencies = next( + row for row in COUNTS_BY_SESSIONS if n <= row[0] + ) + return Limits(sessions, objectives, outcomes, competencies) + + +def detect_syllabus_lang(markdown: str) -> str: + """Language of a syllabus rendered by this module, from its headings.""" + for lang, h in HEADINGS.items(): + if f"## {h['objectives']}" in markdown: + return lang + return "unknown" + + +def split_references(markdown: str) -> tuple[str, str]: + """Split a syllabus into (body, references section). References may be ''.""" + lines = markdown.split("\n") + for i in range(len(lines) - 1, -1, -1): + if _REFERENCES_HEADING.match(lines[i]): + return "\n".join(lines[:i]).rstrip(), "\n".join(lines[i:]).strip() + return markdown.rstrip(), "" + + +def trim_chatter(body: str) -> str: + """Drop LLM chatter before the first heading and after the schedule table.""" + lines = body.strip().split("\n") + start = next((i for i, line in enumerate(lines) if line.startswith("#")), 0) + end = max( + (i for i, line in enumerate(lines) if line.lstrip().startswith("|")), + default=len(lines) - 1, + ) + return "\n".join(lines[start : end + 1]).strip() + + +def render_references(resources: list[dict], lang: str) -> str: + h = HEADINGS.get(lang, HEADINGS["en"]) + seen, items = set(), [] + for res in resources: + url, title = res.get("url", ""), res.get("title", "") + # WeLearn documents always have both; skip anything that would render a + # broken link or a bare URL + if not (url and title) or url in seen: + continue + seen.add(url) + items.append(f'- {title} [{h["link"]}]') + return f"## {h['references']}\n\n" + ("\n".join(items) or h["no_references"]) + + +def _cell(text: str) -> str: + return text.replace("|", "/").replace("\n", "
").strip() + + +def _refs(prefix: str, numbers: list[int], max_n: int) -> str: + return ", ".join(f"{prefix}{n}" for n in numbers if 1 <= n <= max_n) + + +def render_markdown(draft: DraftSyllabus, lang: str, limits: Limits) -> str: + h = HEADINGS.get(lang, HEADINGS["en"]) + objectives = draft.objectives[: limits.objectives] + outcomes = draft.outcomes[: limits.outcomes] + n_obj, n_lo, lo = len(objectives), len(outcomes), h["lo"] + + greencomp, others, seen_codes = [], [], set() + for comp in draft.competencies: + linked = _refs(lo, comp.outcome_numbers, n_lo) + suffix = f" *({linked})*" if linked else "" + code = (comp.greencomp_code or "").strip() + if code not in GREENCOMP_NAMES: + others.append(f"- {comp.text}{suffix}") + elif linked and code not in seen_codes: + # GreenComp only when it supports a listed outcome, each code once + seen_codes.add(code) + name = GREENCOMP_NAMES[code].get(lang, GREENCOMP_NAMES[code]["en"]) + greencomp.append(f"- **GreenComp {code} – {name}** : {comp.text}{suffix}") + if not greencomp: + logger.warning( + "Syllabus draft has no GreenComp competency linked to an outcome" + ) + # GreenComp has priority: if there are as many GreenComp competencies as the + # limit allows, the non-GreenComp ones are dropped + greencomp = greencomp[: limits.competencies] + competencies = greencomp + others[: limits.competencies - len(greencomp)] + + sections = [f"# {draft.course_title}", f"## {h['description']}", draft.description] + sections += [ + f"## {h['objectives']}", + "\n".join(f"{i}. {o}" for i, o in enumerate(objectives, 1)), + ] + sections += [ + f"## {h['outcomes']}", + "\n".join( + f"- **{lo}{i}** {o.text}" + + ( + f" *({h['objectives_ref']} {refs})*" + if (refs := _refs("", o.objective_numbers, n_obj)) + else "" + ) + for i, o in enumerate(outcomes, 1) + ), + ] + sections += [f"## {h['competencies']}", "\n".join(competencies)] + sections += [ + f"## {h['assessment']}", + "\n".join( + f"- **{a.method}** ({a.weight})" + + (f" — {refs}" if (refs := _refs(lo, a.outcome_numbers, n_lo)) else "") + for a in draft.assessment + ), + ] + table = [h["table"], "|---|---|---|---|"] + [ + f"| {i} | {_cell(s.topics)} | {_refs(lo, s.outcome_numbers, n_lo)} | {_cell(s.class_plan)} |" + for i, s in enumerate(draft.schedule, 1) + ] + sections += [f"## {h['schedule']}", "\n".join(table)] + return "\n\n".join(sections) + + +SYSTEM_PROMPT = ( + "You are an experienced university professor and pedagogical engineer, expert in " + "competency-based course design, active learning and sustainability education (UN SDGs, " + "EU GreenComp framework). You design one realistic, coherent syllabus for a real teacher. " + "Your institute's pedagogy is active and student-centred: students learn by doing, discussing, " + "investigating and creating, and the teacher acts as a facilitator.\n\n" + "Rules:\n" + "- Ground the course in the teacher's documents (summaries and themes). Use the WeLearn " + "resources only as supporting content; never invent sources, and do not write references.\n" + "- Weave sustainability into the course content where it connects to the discipline and topics.\n" + "- Write every field as the final syllabus the teacher hands to students. Never comment on your " + "design choices, these rules or the frameworks (no sentences like 'sustainability is not forced' or " + "'a GreenComp competency is integrated'); frameworks appear only in the competencies section.\n" + "- Learning objectives are broad and teacher-centred (what the course covers). Learning outcomes " + "are student-centred, observable and measurable, start with a strong action verb, and name the " + "activity or assessment through which they are demonstrated. Each outcome serves at least one objective.\n" + "- Competencies are transferable skills, each linked to the outcomes that develop it. Map each " + "competency to the GreenComp competency it corresponds to (e.g. critical analysis of discourses -> 2.2, " + "understanding interactions between systems -> 2.1) and set its code; at least one competency must be " + "GreenComp, each code at most once. Leave the code empty only for skills with no GreenComp equivalent " + "(e.g. a disciplinary method). The competency text describes how it applies in this course; do not " + "repeat the GreenComp code or name in it. Never list the whole framework.\n" + "- Assessment methods evaluate the outcomes; weights add up to 100%. Favour authentic assessment " + "(projects, case analyses, presentations, portfolios, peer and self-assessment) over exams alone.\n" + "- The schedule has one row per session; each session targets outcomes from the list. Every outcome " + "is targeted at least once. No placeholders or ellipses.\n" + "- Class plans are student-centred: most of each session is active work such as project- or " + "problem-based learning, case studies, debates and role plays, peer instruction, think-pair-share, " + "flipped classroom, fieldwork, workshops and co-construction. Teacher input is short (15 minutes " + "at most) and serves the activity; a session is never mainly a lecture.\n" + "- Quality over quantity: aim for exactly the counts given by the user, with sharp, distinct items.\n" + "- Write every field in the requested language." +) + + +def build_user_prompt( + message: MessageWithResources, limits: Limits, disciplinary_skills: list[str] +) -> str: + lang = LANG_NAMES.get(message.lang, message.lang) + themes = ", ".join(t["theme"] for t in message.themes) + summaries = "\n\n".join(message.summary) + # ponytail: slices truncated to 1500 chars to bound prompt size (latency) + resources = "\n\n".join( + f"- {r['title']}: {r['content'][:1500]}" + for r in message.resources + if r.get("title") + ) + sessions = ( + f"exactly {limits.sessions} sessions" + if limits.sessions + else "a realistic number of sessions (usually 10 to 12)" + ) + parts = [ + f"Language: {lang}. Every field must be written in {lang}.", + f"Course title: {message.course_title or 'propose a short title'}", + f"Level: {message.level or 'not specified'}", + f"Duration: {message.duration or 'not specified'}", + f"Teacher's description: {message.description or 'none'}", + f"Aim for exactly: {limits.objectives} learning objectives, {limits.outcomes} learning outcomes, " + f"{limits.competencies} competencies (at least {MIN_GREENCOMP} from GreenComp). " + f"Schedule: {sessions}.", + ] + if disciplinary_skills: + parts.append( + "The course should also build these disciplinary skills:\n- " + + "\n- ".join(disciplinary_skills) + ) + parts += [ + f"TEACHER'S DOCUMENTS (summaries):\n{summaries}", + f"THEMES:\n{themes}", + f"WELEARN RESOURCES:\n{resources or 'none'}", + GREENCOMP_COMPETENCIES, + ] + return "\n\n".join(parts) + + +async def generate_syllabus( + message: MessageWithResources, + model: BaseChatModel, + disciplinary_skills: list[str], + config: RunnableConfig, +) -> str: + limits = compute_limits(message.duration) + prompt = ChatPromptTemplate.from_messages( + [("system", SYSTEM_PROMPT), ("human", "{user_prompt}")] + ) + chain = prompt | model.with_structured_output(DraftSyllabus) + user_prompt = build_user_prompt(message, limits, disciplinary_skills) + + async def invoke() -> DraftSyllabus: + result = await chain.ainvoke({"user_prompt": user_prompt}, config=config) + # with_structured_output is typed dict | BaseModel: normalise to DraftSyllabus + # (also raises ValidationError when the LLM returned no/partial fields) + return DraftSyllabus.model_validate(result, from_attributes=True) + + try: + draft = await invoke() + except (OutputParserException, ValidationError) as e: + # ponytail: one retry for a malformed LLM answer; a second failure propagates + logger.warning("Invalid syllabus draft from LLM, retrying: %s", e) + draft = await invoke() + if message.course_title: + draft.course_title = message.course_title + return ( + render_markdown(draft, message.lang, limits) + + "\n\n" + + render_references(message.resources, message.lang) + ) diff --git a/src/app/tutor/service/tutor.py b/src/app/tutor/service/tutor.py index 0853c742..bede92be 100644 --- a/src/app/tutor/service/tutor.py +++ b/src/app/tutor/service/tutor.py @@ -1,48 +1,30 @@ +from langchain_core.runnables import RunnableConfig from langchain_mistralai import ChatMistralAI # type: ignore from src.app.core.config import Settings from src.app.shared.utils.utils import extract_doc_info from src.app.tutor.service.agents import ( + FeedbackAgent, PedagogicalEngineerAgent, SDGExpertAgent, UniversityTeacherAgent, + get_disciplinary_skills, ) from src.app.tutor.service.models import ( MessageWithResources, + SyllabusFeedback, SyllabusResponseAgent, TutorSyllabusRequest, ) - -GREENCOMP_COMPETENCIES = ( - "Here are the GreenComp competencies: " - "url: https://joint-research-centre.ec.europa.eu/greencomp-european-sustainability-competence-framework_en " - "1.1 Valuing sustainability: To reflect on personal values; identify and explain how values vary among people " - "and over time, while critically evaluating how they align with sustainability values. " - "1.2 Supporting fairness: To support equity and justice for current and future generations and learn from previous " - "generations for sustainability. " - "1.3 Promoting nature: To acknowledge that humans are part of nature; and to respect the needs and rights of other " - "species and of nature itself in order to restore and regenerate healthy and resilient ecosystems. " - "2.1 Systems thinking: To approach a sustainability problem from all sides; to consider time, space and context in " - "order to understand how elements interact within and between systems. " - "2.2 Critical thinking: To assess information and arguments, identify assumptions, challenge the status quo, and " - "reflect on how personal, social and cultural backgrounds influence thinking and conclusions. " - "2.3 Problem framing: To formulate current or potential challenges as a sustainability problem in terms of " - "difficulty, people involved, time and geographical scope, in order to identify suitable approaches to anticipating " - "and preventing problems, and to mitigating and adapting to already existing problems. " - "3.1 Futures literacy: To envision alternative sustainable futures by imagining and developing alternative scenarios " - "and identifying the steps needed to achieve a preferred sustainable future. " - "3.2 Adaptability: To manage transitions and challenges in complex sustainability situations and make decisions " - "related to the future in the face of uncertainty, ambiguity and risk. " - "3.3 Exploratory thinking: To adopt a relational way of thinking by exploring and linking different disciplines, " - "using creativity and experimentation with novel ideas or methods. " - "4.1 Political agency: To navigate the political system, identify political responsibility and accountability for " - "unsustainable behaviour, and demand effective policies for sustainability. " - "4.2 Collective action: To act for change in collaboration with others. " - "4.3 Individual initiative: To identify own potential for sustainability and to actively contribute to improving " - "prospects for the community and the planet.The weather should be in metric units" +from src.app.tutor.service.syllabus import ( + GREENCOMP_COMPETENCIES, + detect_syllabus_lang, + generate_syllabus, + render_references, + split_references, + trim_chatter, ) - chat_model: ChatMistralAI | None = None @@ -100,6 +82,22 @@ async def tutor_manager( if endpoint: base_tags.append(f"endpoint:{endpoint}") + if settings.TUTOR_SINGLE_PASS: + content_md = await generate_syllabus( + formatted_content, + chat_model, + get_disciplinary_skills().get(formatted_content.discipline, []), + RunnableConfig( + tags=base_tags + ["agent:single_pass"], + metadata={**base_metadata, "agent": "SinglePassSyllabus"}, + run_name="Tutor (SinglePassSyllabus)", + ), + ) + # source name kept: the client picks the "PedagogicalEngineerAgent" item + return [ + SyllabusResponseAgent(content=content_md, source="PedagogicalEngineerAgent") + ] + teacher_agent = UniversityTeacherAgent( chat_model, lang, @@ -126,5 +124,43 @@ async def tutor_manager( teacher_response, formatted_content.resources, lang ) pedagogical_response = await pedagogical_agent.refine(sdg_response) + # references are never trusted from the LLM: rebuilt from the selected documents + body, _ = split_references(pedagogical_response.content) + pedagogical_response.content = ( + trim_chatter(body) + + "\n\n" + + render_references(formatted_content.resources, lang) + ) return [teacher_response, sdg_response, pedagogical_response] + + +async def apply_feedback( + body: SyllabusFeedback, settings: Settings, trace_context: dict | None = None +) -> str: + """Apply the teacher's feedback to the syllabus body; references are kept verbatim.""" + if chat_model is None: + raise RuntimeError( + "Chat model not initialized. Call init_chat_model() at startup." + ) + syllabus_body, references = split_references(body.syllabus[0].content) + # same tags/metadata as generation; the client sends no lang here, so it is + # read from the syllabus's own headings + tags = ["welearn", "tutor", "syllabus"] + if trace_context and trace_context.get("endpoint"): + tags.append(f"endpoint:{trace_context['endpoint']}") + agent = FeedbackAgent( + chat_model, + GREENCOMP_COMPETENCIES, + trace_tags=tags, + trace_metadata={ + "component": "tutor_syllabus", + "environment": settings.ENV, + "language": detect_syllabus_lang(syllabus_body), + **(trace_context or {}), + }, + ) + new_body = trim_chatter( + await agent.apply(syllabus_body, body.feedback, body.extracts) + ) + return f"{new_body}\n\n{references}" if references else new_body