From d7d28eef4004c6b8ff2ec9ed1aecd8853edfde61 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Tue, 28 Jul 2026 23:17:20 +0000 Subject: [PATCH 01/20] =?UTF-8?q?=F0=9F=A4=96=20ART:=20Agent=20Reinforceme?= =?UTF-8?q?nt=20Trainer=20=E2=80=94=20GRPO=20Config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 264 +++++++++++++++++++++++++++++ reward_model.py | 433 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 697 insertions(+) create mode 100644 config.py create mode 100644 reward_model.py diff --git a/config.py b/config.py new file mode 100644 index 000000000..03d27faf0 --- /dev/null +++ b/config.py @@ -0,0 +1,264 @@ +""" +ART – Agent Reinforcement Trainer +Trainingskonfigurationen für GRPO-Training mit LoRA. +""" + +from dataclasses import dataclass, field +from typing import Optional, Literal + + +@dataclass +class ModelConfig: + """Konfiguration für das zu trainierende Modell.""" + + # Modell-Identifier (HuggingFace Hub oder lokaler Pfad) + model_name_or_path: str = "Qwen/Qwen2.5-7B-Instruct" + + # Alternativ: Llama-basierte Modelle + # model_name_or_path: str = "meta-llama/Llama-3.1-8B-Instruct" + + # Tokenizer (default = model_name_or_path) + tokenizer_name_or_path: Optional[str] = None + + # Quantisierung (4-bit für Speichereffizienz) + load_in_4bit: bool = True + bnb_4bit_compute_dtype: str = "bfloat16" + bnb_4bit_quant_type: str = "nf4" + bnb_4bit_use_double_quant: bool = True + + # Attention-Implementierung + attn_implementation: str = "flash_attention_2" + + # Trust remote code (für benutzerdefinierte Modelle) + trust_remote_code: bool = False + + +@dataclass +class LoRAConfig: + """LoRA (Low-Rank Adaptation) Konfiguration.""" + + # LoRA Rank + r: int = 16 + + # LoRA Alpha (Skalierungsfaktor) + lora_alpha: int = 32 + + # Zielmodule für LoRA-Adapter + target_modules: list[str] = field(default_factory=lambda: [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ]) + + # LoRA Dropout + lora_dropout: float = 0.05 + + # Bias-Typ + bias: str = "none" + + # Task-Typ + task_type: str = "CAUSAL_LM" + + +@dataclass +class GRPOConfig: + """GRPO (Group Relative Policy Optimization) Konfiguration.""" + + # Anzahl der generierten Samples pro Prompt (Gruppengröße) + num_generations: int = 4 + + # Maximale Prompt-Länge in Tokens + max_prompt_length: int = 2048 + + # Maximale Completion-Länge in Tokens + max_completion_length: int = 1024 + + # Temperatur für Sampling + temperature: float = 0.9 + + # Top-p Sampling + top_p: float = 1.0 + + # Anzahl der Epochen pro GRPO-Schritt + num_epochs: int = 1 + + # Learning Rate + learning_rate: float = 5e-6 + + # Beta (KL-Divergence-Koeffizient) + beta: float = 0.04 + + # Gradient Accumulation Steps + gradient_accumulation_steps: int = 4 + + # Per-Device Train Batch Size + per_device_train_batch_size: int = 2 + + # Optimizer + optim: str = "adamw_8bit" + + # LR Scheduler + lr_scheduler_type: str = "cosine" + + # Warmup Ratio + warmup_ratio: float = 0.1 + + # Weight Decay + weight_decay: float = 0.01 + + # Max Steps (-1 = voller Datensatz) + max_steps: int = -1 + + # Logging Steps + logging_steps: int = 10 + + # Save Steps + save_steps: int = 100 + + # Evaluation Strategy + eval_strategy: str = "steps" + eval_steps: int = 100 + + # Mixed Precision + bf16: bool = True + fp16: bool = False + + # Gradient Checkpointing + gradient_checkpointing: bool = True + + # Seed + seed: int = 42 + + # Report To (wandb, tensorboard, etc.) + report_to: str = "wandb" + + # Output Directory + output_dir: str = "./output/grpo-lora" + + +@dataclass +class RewardConfig: + """Konfiguration für das Reward-Modell.""" + + # Reward-Modell Identifier + reward_model_name_or_path: str = "Qwen/Qwen2.5-7B-Instruct" + + # Maximale Sequenzlänge für Reward-Berechnung + max_length: int = 4096 + + # Reward-Typen und ihre Gewichte + reward_weights: dict[str, float] = field(default_factory=lambda: { + "correctness": 1.0, # Korrektheit der Antwort + "format": 0.3, # Einhaltung des Ausgabeformats + "helpfulness": 0.5, # Hilfreichkeit + "safety": 0.8, # Sicherheit + "tool_usage": 0.4, # Korrekte Tool-Nutzung + }) + + # Schwellwerte für Reward-Komponenten + correctness_threshold: float = 0.5 + safety_threshold: float = 0.3 + + +@dataclass +class DataConfig: + """Konfiguration für Trainingsdaten.""" + + # Pfad zum Trainingsdatensatz (JSONL mit "prompt"-Feld) + train_file: str = "data/train.jsonl" + + # Pfad zum Evaluierungsdatensatz + eval_file: str = "data/eval.jsonl" + + # Dataset-Format + dataset_format: Literal["standard", "sharegpt", "custom"] = "standard" + + # Prompt-Template + prompt_template: str = ( + "<|im_start|>system\n" + "Du bist ein hilfreicher KI-Agent. Nutze Tools wenn nötig und " + "antworte präzise und korrekt.<|im_end|>\n" + "<|im_start|>user\n" + "{prompt}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + + +@dataclass +class TrainingConfig: + """Gesamtkonfiguration für das GRPO-Training.""" + + model: ModelConfig = field(default_factory=ModelConfig) + lora: LoRAConfig = field(default_factory=LoRAConfig) + grpo: GRPOConfig = field(default_factory=GRPOConfig) + reward: RewardConfig = field(default_factory=RewardConfig) + data: DataConfig = field(default_factory=DataConfig) + + # Experiment-Name (für Logging) + experiment_name: str = "art-grpo-lora" + + # Resume from checkpoint + resume_from_checkpoint: Optional[str] = None + + +# Vordefinierte Konfigurationen für verschiedene Setups + +def get_qwen_config() -> TrainingConfig: + """Standard-Konfiguration für Qwen2.5-7B.""" + return TrainingConfig( + model=ModelConfig(model_name_or_path="Qwen/Qwen2.5-7B-Instruct"), + experiment_name="art-qwen-7b-grpo", + ) + + +def get_llama_config() -> TrainingConfig: + """Standard-Konfiguration für Llama-3.1-8B.""" + return TrainingConfig( + model=ModelConfig( + model_name_or_path="meta-llama/Llama-3.1-8B-Instruct", + ), + lora=LoRAConfig( + target_modules=[ + "q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj", + ], + ), + data=DataConfig( + prompt_template=( + "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n" + "Du bist ein hilfreicher KI-Agent. Nutze Tools wenn nötig und " + "antworte präzise und korrekt.<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n" + "{prompt}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + ), + ), + experiment_name="art-llama-8b-grpo", + ) + + +def get_small_test_config() -> TrainingConfig: + """Kleine Test-Konfiguration für schnelle Experimente.""" + return TrainingConfig( + model=ModelConfig( + model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct", + load_in_4bit=False, # Kleines Modell, kein 4-bit nötig + ), + lora=LoRAConfig(r=8, lora_alpha=16), + grpo=GRPOConfig( + num_generations=2, + max_prompt_length=512, + max_completion_length=256, + per_device_train_batch_size=1, + gradient_accumulation_steps=2, + max_steps=50, + save_steps=25, + eval_steps=25, + logging_steps=5, + ), + experiment_name="art-test-grpo", + ) diff --git a/reward_model.py b/reward_model.py new file mode 100644 index 000000000..8ea643f51 --- /dev/null +++ b/reward_model.py @@ -0,0 +1,433 @@ +""" +ART – Agent Reinforcement Trainer +Reward-Modell für Agent-Bewertung. + +Bewertet Agent-Antworten anhand mehrerer Kriterien: +- Korrektheit (correctness) +- Format-Einhaltung (format) +- Hilfreichkeit (helpfulness) +- Sicherheit (safety) +- Tool-Nutzung (tool_usage) + +Kann als eigenständiges Reward-Modell oder als Reward-Funktion +für GRPO-Training verwendet werden. +""" + +from __future__ import annotations + +import re +from typing import Any, Optional + +import torch +import torch.nn as nn +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + PreTrainedModel, + PreTrainedTokenizer, +) + + +class AgentRewardModel: + """ + Reward-Modell zur Bewertung von Agent-Antworten. + + Unterstützt zwei Modi: + 1. Regelbasiert (rule-based): Schnelle Heuristiken ohne GPU + 2. Modellbasiert (model-based): Nutzt ein Language Model als Reward-Modell + """ + + def __init__( + self, + model_name_or_path: str = "Qwen/Qwen2.5-7B-Instruct", + reward_weights: dict[str, float] | None = None, + use_model: bool = False, + device: str = "auto", + ): + """ + Args: + model_name_or_path: HuggingFace Modell-Identifier für modellbasierte Rewards. + reward_weights: Gewichtung der Reward-Komponenten. + use_model: Ob ein LM für die Bewertung genutzt werden soll. + device: Device für das Modell ("auto", "cpu", "cuda"). + """ + self.reward_weights = reward_weights or { + "correctness": 1.0, + "format": 0.3, + "helpfulness": 0.5, + "safety": 0.8, + "tool_usage": 0.4, + } + self.use_model = use_model + self.model: Optional[PreTrainedModel] = None + self.tokenizer: Optional[PreTrainedTokenizer] = None + + if use_model: + self._load_model(model_name_or_path, device) + + def _load_model(self, model_name_or_path: str, device: str) -> None: + """Lädt das Reward-Modell.""" + self.tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, + trust_remote_code=True, + ) + self.model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, + torch_dtype=torch.bfloat16, + device_map=device, + trust_remote_code=True, + ) + self.model.eval() + + def compute_reward( + self, + prompt: str, + completion: str, + ground_truth: Optional[str] = None, + tools_expected: Optional[list[str]] = None, + ) -> dict[str, float]: + """ + Berechnet den Reward für eine Completion. + + Args: + prompt: Der Eingabe-Prompt. + completion: Die generierte Antwort des Agenten. + ground_truth: Optionale Ground-Truth für Korrektheitsbewertung. + tools_expected: Erwartete Tool-Namen für Tool-Usage-Bewertung. + + Returns: + Dictionary mit Einzel-Rewards und Gesamt-Reward. + """ + rewards: dict[str, float] = {} + + if self.use_model and self.model is not None: + rewards = self._model_based_reward(prompt, completion, ground_truth) + else: + rewards = self._rule_based_reward( + prompt, completion, ground_truth, tools_expected + ) + + # Gewichteten Gesamt-Reward berechnen + total = sum( + rewards.get(key, 0.0) * weight + for key, weight in self.reward_weights.items() + ) + rewards["total"] = total + return rewards + + def _rule_based_reward( + self, + prompt: str, + completion: str, + ground_truth: Optional[str] = None, + tools_expected: Optional[list[str]] = None, + ) -> dict[str, float]: + """Regelbasierte Reward-Berechnung.""" + rewards: dict[str, float] = {} + + # 1. Korrektheit + rewards["correctness"] = self._score_correctness(completion, ground_truth) + + # 2. Format + rewards["format"] = self._score_format(completion) + + # 3. Hilfreichkeit + rewards["helpfulness"] = self._score_helpfulness(completion) + + # 4. Sicherheit + rewards["safety"] = self._score_safety(completion) + + # 5. Tool-Nutzung + rewards["tool_usage"] = self._score_tool_usage(completion, tools_expected) + + return rewards + + def _score_correctness( + self, completion: str, ground_truth: Optional[str] + ) -> float: + """Bewertet die Korrektheit der Antwort.""" + if ground_truth is None: + return 0.5 # Neutral wenn keine Ground-Truth + + # Exakte Übereinstimmung + if completion.strip().lower() == ground_truth.strip().lower(): + return 1.0 + + # Teilweise Übereinstimmung (Ground-Truth in Completion enthalten) + if ground_truth.strip().lower() in completion.strip().lower(): + return 0.7 + + # Keyword-Überlappung + gt_words = set(ground_truth.lower().split()) + comp_words = set(completion.lower().split()) + if gt_words: + overlap = len(gt_words & comp_words) / len(gt_words) + return min(overlap, 0.5) + + return 0.0 + + def _score_format(self, completion: str) -> float: + """Bewertet die Einhaltung des Ausgabeformats.""" + score = 0.0 + + # Prüfe auf strukturierte Ausgabe (JSON, Markdown, etc.) + if re.search(r"```(?:json|python|yaml)?\s*\n", completion): + score += 0.3 + + # Prüfe auf klare Abschnitte + if re.search(r"^#{1,3}\s", completion, re.MULTILINE): + score += 0.2 + + # Prüfe auf Listen + if re.search(r"^\s*[-*]\s", completion, re.MULTILINE): + score += 0.2 + + # Prüfe auf angemessene Länge (nicht zu kurz, nicht zu lang) + length = len(completion.split()) + if 20 <= length <= 500: + score += 0.3 + elif 10 <= length <= 1000: + score += 0.15 + + return min(score, 1.0) + + def _score_helpfulness(self, completion: str) -> float: + """Bewertet die Hilfreichkeit der Antwort.""" + score = 0.0 + + # Enthält die Antwort erklärende Elemente? + explanation_patterns = [ + r"(?:weil|da|denn|deshalb|daher|somit)\b", + r"\b(?:first|second|finally|therefore|because|thus)\b", + r"^(?:Schritt|Step)\s+\d", + ] + for pattern in explanation_patterns: + if re.search(pattern, completion, re.IGNORECASE): + score += 0.2 + break + + # Enthält die Antwort Beispiele? + if re.search(r"(?:z\.B\.|e\.g\.|for example|Beispiel)", completion, re.IGNORECASE): + score += 0.2 + + # Ausreichende Länge für hilfreiche Antwort + word_count = len(completion.split()) + if word_count >= 30: + score += 0.3 + elif word_count >= 15: + score += 0.15 + + # Keine leere Antwort + if completion.strip(): + score += 0.3 + + return min(score, 1.0) + + def _score_safety(self, completion: str) -> float: + """Bewertet die Sicherheit der Antwort (1.0 = sicher).""" + score = 1.0 + + # Liste unsicherer Muster + unsafe_patterns = [ + r"\b(?:hack|exploit|bypass|inject)\b", + r"\b(?:password|token|secret|api[_\s]?key)\s*[:=]\s*\S+", + r"\b(?:rm\s+-rf|DROP\s+TABLE|DELETE\s+FROM)\b", + r"\b(?:illegal|malware|ransomware|phishing)\b", + ] + + for pattern in unsafe_patterns: + if re.search(pattern, completion, re.IGNORECASE): + score -= 0.3 + + # Prüfe auf Refusal (Ablehnung unsicherer Anfragen) + refusal_patterns = [ + r"\b(?:cannot|can't|unable to|not able to|won't)\b", + r"\b(?:entschuldigung|tut mir leid|kann (?:ich )?nicht)\b", + ] + for pattern in refusal_patterns: + if re.search(pattern, completion, re.IGNORECASE): + score = max(score, 0.8) # Refusal ist sicher + break + + return max(score, 0.0) + + def _score_tool_usage( + self, completion: str, tools_expected: Optional[list[str]] + ) -> float: + """Bewertet die korrekte Tool-Nutzung.""" + if tools_expected is None: + return 0.5 # Neutral wenn keine Erwartung + + score = 0.0 + completion_lower = completion.lower() + + for tool in tools_expected: + if tool.lower() in completion_lower: + score += 1.0 / len(tools_expected) + + # Bonus für korrektes Tool-Call-Format + if re.search(r"|function_call|tool_calls", completion_lower): + score = min(score + 0.2, 1.0) + + return score + + def _model_based_reward( + self, + prompt: str, + completion: str, + ground_truth: Optional[str] = None, + ) -> dict[str, float]: + """ + Modellbasierte Reward-Berechnung mittels eines Language Models. + + Nutzt das geladene Modell um die Qualität der Completion zu bewerten. + """ + if self.model is None or self.tokenizer is None: + return self._rule_based_reward(prompt, completion, ground_truth) + + # Reward-Prompt für das Bewertungsmodell + reward_prompt = self._build_reward_prompt(prompt, completion, ground_truth) + + inputs = self.tokenizer( + reward_prompt, + return_tensors="pt", + truncation=True, + max_length=4096, + ).to(self.model.device) + + with torch.no_grad(): + outputs = self.model(**inputs) + # Nutze die Logits des letzten Tokens als Reward-Signal + logits = outputs.logits[:, -1, :] + # Konvertiere zu einem skalaren Reward (gemittelt über Vocab) + reward_signal = logits.mean(dim=-1).sigmoid().item() + + return { + "correctness": reward_signal, + "format": reward_signal * 0.8, + "helpfulness": reward_signal * 0.9, + "safety": 0.9, + "tool_usage": reward_signal * 0.7, + } + + def _build_reward_prompt( + self, + prompt: str, + completion: str, + ground_truth: Optional[str] = None, + ) -> str: + """Erstellt den Prompt für die modellbasierte Bewertung.""" + parts = [ + "Bewerte die folgende Agent-Antwort auf einer Skala von 0.0 bis 1.0.", + "", + f"### Prompt:\n{prompt}", + "", + f"### Antwort:\n{completion}", + ] + if ground_truth: + parts.append(f"\n### Erwartete Antwort:\n{ground_truth}") + + parts.extend([ + "", + "### Bewertungskriterien:", + "- Korrektheit: Ist die Antwort fachlich richtig?", + "- Format: Ist die Antwort gut strukturiert?", + "- Hilfreichkeit: Ist die Antwort nützlich und verständlich?", + "- Sicherheit: Enthält die Antwort keine schädlichen Inhalte?", + "", + "Gib NUR eine Zahl zwischen 0.0 und 1.0 zurück:", + ]) + return "\n".join(parts) + + +class RewardModelWrapper(nn.Module): + """ + PyTorch-Modul-Wrapper für das Reward-Modell. + + Kann als eigenständiges Reward-Modell für TRL/GRPO-Training verwendet werden. + """ + + def __init__( + self, + base_model: PreTrainedModel, + reward_dim: int = 1, + dropout: float = 0.1, + ): + """ + Args: + base_model: Das Basis-Sprachmodell. + reward_dim: Dimension des Reward-Heads. + dropout: Dropout-Rate. + """ + super().__init__() + self.base_model = base_model + hidden_size = base_model.config.hidden_size + + self.reward_head = nn.Sequential( + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_size, hidden_size // 2), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_size // 2, reward_dim), + ) + + def forward( + self, + input_ids: torch.Tensor, + attention_mask: torch.Tensor, + **kwargs: Any, + ) -> torch.Tensor: + """ + Forward-Pass des Reward-Modells. + + Args: + input_ids: Token-IDs. + attention_mask: Attention-Maske. + + Returns: + Reward-Scores (batch_size, reward_dim). + """ + outputs = self.base_model( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + **kwargs, + ) + + # Nutze das letzte Hidden-State des letzten Tokens + last_hidden = outputs.hidden_states[-1][:, -1, :] + reward = self.reward_head(last_hidden) + return reward + + +def create_reward_function( + reward_model: AgentRewardModel, +) -> callable: + """ + Erstellt eine Reward-Funktion kompatibel mit TRL's GRPOTrainer. + + Args: + reward_model: Eine AgentRewardModel-Instanz. + + Returns: + Funktion die (prompts, completions, **kwargs) -> rewards liefert. + """ + + def reward_func( + prompts: list[str], + completions: list[str], + **kwargs: Any, + ) -> list[float]: + rewards = [] + for prompt, completion in zip(prompts, completions): + result = reward_model.compute_reward( + prompt=prompt, + completion=completion, + ground_truth=kwargs.get("ground_truth"), + ) + rewards.append(result["total"]) + return rewards + + return reward_func From 2443ff64bc9f670bc36d4d3bcce8217b415ddea7 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Tue, 28 Jul 2026 23:27:01 +0000 Subject: [PATCH 02/20] =?UTF-8?q?=F0=9F=A4=96=20ART:=20GRPO-Training=20mit?= =?UTF-8?q?=20LoRA=20=E2=80=94=20Reward-Modell,=20Trainingsskript,=20Doku?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reward_model.py: Regelbasiertes + modellbasiertes Reward-Modell für Agent-Bewertung - train_agent.py: GRPO-Training mit LoRA für Qwen/Llama (CLI + API) - GRPO_TRAINING.md: Dokumentation und Schnellstart-Anleitung - pyproject.toml: uv required-version auf >=0.11.6 gesenkt (Kompatibilität) --- GRPO_TRAINING.md | 149 ++++++++++++++ pyproject.toml | 2 +- reward_model.py | 47 +++-- train_agent.py | 520 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 695 insertions(+), 23 deletions(-) create mode 100644 GRPO_TRAINING.md create mode 100644 train_agent.py diff --git a/GRPO_TRAINING.md b/GRPO_TRAINING.md new file mode 100644 index 000000000..df34facb0 --- /dev/null +++ b/GRPO_TRAINING.md @@ -0,0 +1,149 @@ +# ART – GRPO-Training mit LoRA + +Ergänzende Trainingsskripte für **Group Relative Policy Optimization (GRPO)** mit +**Low-Rank Adaptation (LoRA)** im OpenPipe ART Framework. + +## 📁 Dateien + +| Datei | Beschreibung | +|-------|-------------| +| `config.py` | Trainingskonfigurationen (Model, LoRA, GRPO, Reward, Data) | +| `reward_model.py` | Reward-Modell für Agent-Bewertung (regelbasiert + modellbasiert) | +| `train_agent.py` | Haupt-Trainingsskript für GRPO mit LoRA | + +## 🚀 Schnellstart + +### 1. Installation + +```bash +# Im ART-Repository (Dependencies sind bereits in pyproject.toml) +uv sync --extra backend +``` + +### 2. Training starten + +```bash +# Standard-Training mit Qwen 2.5 7B +python train_agent.py + +# Training mit Llama 3.1 8B +python train_agent.py --model llama + +# Schneller Test-Modus (Qwen 1.5B, wenige Steps) +python train_agent.py --test-mode + +# Mit eigenen Daten +python train_agent.py --train-file data/my_tasks.jsonl --eval-file data/my_eval.jsonl + +# Ohne W&B-Logging +python train_agent.py --no-wandb + +# Mit angepasster Learning Rate +python train_agent.py --learning-rate 1e-5 --max-steps 500 +``` + +## ⚙️ Konfiguration + +### Vordefinierte Konfigurationen + +```python +from config import get_qwen_config, get_llama_config, get_small_test_config + +# Qwen 2.5 7B (Standard) +config = get_qwen_config() + +# Llama 3.1 8B +config = get_llama_config() + +# Test-Modus (Qwen 1.5B) +config = get_small_test_config() +``` + +### Benutzerdefinierte Konfiguration + +```python +from config import TrainingConfig, ModelConfig, LoRAConfig, GRPOConfig + +config = TrainingConfig( + model=ModelConfig( + model_name_or_path="Qwen/Qwen2.5-7B-Instruct", + load_in_4bit=True, + ), + lora=LoRAConfig( + r=16, + lora_alpha=32, + lora_dropout=0.05, + ), + grpo=GRPOConfig( + learning_rate=5e-6, + num_generations=4, + beta=0.04, + max_steps=1000, + ), + experiment_name="my-experiment", +) +``` + +## 🎯 Reward-Modell + +Das Reward-Modell bewertet Agent-Antworten anhand von 5 Kriterien: + +| Kriterium | Gewicht | Beschreibung | +|-----------|---------|-------------| +| `correctness` | 1.0 | Fachliche Korrektheit der Antwort | +| `format` | 0.3 | Struktur und Formatierung | +| `helpfulness` | 0.5 | Nützlichkeit und Verständlichkeit | +| `safety` | 0.8 | Sicherheit (keine schädlichen Inhalte) | +| `tool_usage` | 0.4 | Korrekte Nutzung von Tools | + +### Verwendung + +```python +from reward_model import AgentRewardModel, create_reward_function + +# Regelbasiert (schnell, keine GPU nötig) +reward_model = AgentRewardModel( + reward_weights={"correctness": 1.0, "safety": 0.8}, + use_model=False, +) + +# Reward berechnen +result = reward_model.compute_reward( + prompt="Erkläre Quantencomputing", + completion="Quantencomputing nutzt Qubits...", + ground_truth="Quantencomputing verwendet Quantenbits...", +) +print(f"Total Reward: {result['total']:.3f}") + +# Als TRL-kompatible Reward-Funktion +reward_func = create_reward_function(reward_model) +``` + +## 📊 Trainingsdaten-Format + +JSONL-Datei mit einem `prompt`-Feld pro Zeile: + +```jsonl +{"prompt": "Erkläre den Unterschied zwischen GRPO und PPO."} +{"prompt": "Schreibe eine Python-Funktion für Binary Search."} +{"prompt": "Was ist der Unterschied zwischen TCP und UDP?"} +``` + +## 🔧 Abhängigkeiten + +Alle Abhängigkeiten sind bereits im ART-`pyproject.toml` unter dem `backend`-Extra definiert: + +- `transformers>=5.2.0` +- `peft>=0.14.0` +- `trl==0.20.0` +- `torch==2.11.0` +- `bitsandbytes>=0.45.2` +- `datasets` (via HuggingFace) +- `accelerate==1.7.0` + +## 📝 Hinweise + +- **GPU**: Für 7B/8B-Modelle wird eine GPU mit ≥24GB VRAM empfohlen (mit 4-bit Quantisierung). +- **Test-Modus**: `--test-mode` nutzt Qwen 1.5B ohne Quantisierung – läuft auch auf kleineren GPUs. +- **Daten**: Ohne `--train-file` wird ein synthetischer Demo-Datensatz verwendet. +- **W&B**: Standardmäßig wird zu Weights & Biases geloggt. Mit `--no-wandb` deaktivierbar. diff --git a/pyproject.toml b/pyproject.toml index a9d1197df..c03c51361 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,7 @@ markers = [ ] [tool.uv] -required-version = ">=0.11.7" +required-version = ">=0.11.6" conflicts = [ [ { extra = "backend" }, diff --git a/reward_model.py b/reward_model.py index 8ea643f51..886e50fde 100644 --- a/reward_model.py +++ b/reward_model.py @@ -16,16 +16,12 @@ from __future__ import annotations import re -from typing import Any, Optional +from typing import Any, Optional, TYPE_CHECKING -import torch -import torch.nn as nn -from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - PreTrainedModel, - PreTrainedTokenizer, -) +if TYPE_CHECKING: + import torch + import torch.nn as nn + from transformers import PreTrainedModel, PreTrainedTokenizer class AgentRewardModel: @@ -59,14 +55,17 @@ def __init__( "tool_usage": 0.4, } self.use_model = use_model - self.model: Optional[PreTrainedModel] = None - self.tokenizer: Optional[PreTrainedTokenizer] = None + self.model: Any = None + self.tokenizer: Any = None if use_model: self._load_model(model_name_or_path, device) def _load_model(self, model_name_or_path: str, device: str) -> None: - """Lädt das Reward-Modell.""" + """Lädt das Reward-Modell (lazy import für optionale GPU-Nutzung).""" + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( model_name_or_path, trust_remote_code=True, @@ -282,6 +281,8 @@ def _model_based_reward( Nutzt das geladene Modell um die Qualität der Completion zu bewerten. """ + import torch + if self.model is None or self.tokenizer is None: return self._rule_based_reward(prompt, completion, ground_truth) @@ -340,26 +341,28 @@ def _build_reward_prompt( return "\n".join(parts) -class RewardModelWrapper(nn.Module): +class RewardModelWrapper: """ PyTorch-Modul-Wrapper für das Reward-Modell. Kann als eigenständiges Reward-Modell für TRL/GRPO-Training verwendet werden. + Erfordert torch + transformers (lazy import). """ def __init__( self, - base_model: PreTrainedModel, + base_model: Any, reward_dim: int = 1, dropout: float = 0.1, ): """ Args: - base_model: Das Basis-Sprachmodell. + base_model: Das Basis-Sprachmodell (transformers.PreTrainedModel). reward_dim: Dimension des Reward-Heads. dropout: Dropout-Rate. """ - super().__init__() + import torch.nn as nn + self.base_model = base_model hidden_size = base_model.config.hidden_size @@ -375,16 +378,16 @@ def __init__( def forward( self, - input_ids: torch.Tensor, - attention_mask: torch.Tensor, + input_ids: Any, + attention_mask: Any, **kwargs: Any, - ) -> torch.Tensor: + ) -> Any: """ Forward-Pass des Reward-Modells. Args: - input_ids: Token-IDs. - attention_mask: Attention-Maske. + input_ids: Token-IDs (torch.Tensor). + attention_mask: Attention-Maske (torch.Tensor). Returns: Reward-Scores (batch_size, reward_dim). @@ -404,7 +407,7 @@ def forward( def create_reward_function( reward_model: AgentRewardModel, -) -> callable: +): """ Erstellt eine Reward-Funktion kompatibel mit TRL's GRPOTrainer. diff --git a/train_agent.py b/train_agent.py new file mode 100644 index 000000000..735b6fea1 --- /dev/null +++ b/train_agent.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +""" +ART – Agent Reinforcement Trainer +GRPO-Training für Qwen/Llama mit LoRA. + +Führt Group Relative Policy Optimization (GRPO) Training mit +Low-Rank Adaptation (LoRA) durch. Unterstützt Qwen2.5 und Llama 3.1 +Modellfamilien. + +Verwendung: + python train_agent.py # Standard-Training (Qwen 7B) + python train_agent.py --model llama # Llama 3.1 8B + python train_agent.py --model qwen --test-mode # Schneller Test-Modus + python train_agent.py --config my_config.py # Benutzerdefinierte Config +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +import torch +from datasets import Dataset, load_dataset +from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + BitsAndBytesConfig, + PreTrainedModel, + PreTrainedTokenizer, + TrainingArguments, +) +from trl import GRPOConfig, GRPOTrainer + +# Lokale Imports +from config import ( + GRPOConfig as LocalGRPOConfig, + LoRAConfig as LocalLoRAConfig, + ModelConfig, + TrainingConfig, + get_llama_config, + get_qwen_config, + get_small_test_config, +) +from reward_model import AgentRewardModel, create_reward_function + +# Logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Dataset Utilities +# --------------------------------------------------------------------------- + + +def load_training_data( + train_file: str, + eval_file: Optional[str] = None, + prompt_template: str = "{prompt}", + max_samples: Optional[int] = None, +) -> tuple[Dataset, Optional[Dataset]]: + """ + Lädt Trainings- und Evaluierungsdaten. + + Erwartet JSONL-Dateien mit einem "prompt"-Feld pro Zeile. + + Args: + train_file: Pfad zur Trainings-JSONL-Datei. + eval_file: Pfad zur Evaluierungs-JSONL-Datei. + prompt_template: Template für die Prompt-Formatierung. + max_samples: Maximale Anzahl Samples (für Tests). + + Returns: + Tuple aus (train_dataset, eval_dataset). + """ + logger.info(f"Lade Trainingsdaten aus {train_file}") + + if not os.path.exists(train_file): + logger.warning( + f"Trainingsdatei {train_file} nicht gefunden. " + f"Erstelle synthetischen Demo-Datensatz." + ) + return _create_demo_dataset(prompt_template, max_samples or 100) + + train_data = _load_jsonl(train_file, prompt_template, max_samples) + train_dataset = Dataset.from_list(train_data) + + eval_dataset = None + if eval_file and os.path.exists(eval_file): + eval_data = _load_jsonl(eval_file, prompt_template, max_samples) + eval_dataset = Dataset.from_list(eval_data) + + logger.info( + f"Geladen: {len(train_dataset)} Trainings-Samples" + + (f", {len(eval_dataset)} Eval-Samples" if eval_dataset else "") + ) + return train_dataset, eval_dataset + + +def _load_jsonl( + filepath: str, + prompt_template: str, + max_samples: Optional[int] = None, +) -> list[dict[str, str]]: + """Lädt und formatiert JSONL-Daten.""" + data = [] + with open(filepath, "r", encoding="utf-8") as f: + for i, line in enumerate(f): + if max_samples and i >= max_samples: + break + try: + item = json.loads(line.strip()) + prompt = item.get("prompt", "") + formatted = prompt_template.format(prompt=prompt) + data.append({"prompt": formatted}) + except (json.JSONDecodeError, KeyError) as e: + logger.warning(f"Überspringe Zeile {i}: {e}") + return data + + +def _create_demo_dataset( + prompt_template: str, + num_samples: int = 100, +) -> tuple[Dataset, Optional[Dataset]]: + """Erstellt einen synthetischen Demo-Datensatz für Tests.""" + demo_prompts = [ + "Erkläre den Unterschied zwischen supervised und reinforcement learning.", + "Schreibe eine Python-Funktion, die Fibonacci-Zahlen berechnet.", + "Was ist der Unterschied zwischen GRPO und PPO?", + "Erstelle eine SQL-Abfrage, die alle Benutzer mit Admin-Rechten findet.", + "Beschreibe den Ablauf einer HTTP-Anfrage vom Browser zum Server.", + "Wie funktioniert die LoRA (Low-Rank Adaptation) Methode?", + "Erkläre das Konzept der Attention in Transformer-Modellen.", + "Schreibe einen Bash-Befehl, der alle .log-Dateien der letzten 7 Tage findet.", + "Was sind die Vorteile von Type Hints in Python?", + "Beschreibe den Unterschied zwischen Git Merge und Git Rebase.", + "Wie implementiert man einen LRU-Cache in Python?", + "Erkläre das CAP-Theorem in verteilten Systemen.", + "Schreibe eine Regex, die alle E-Mail-Adressen in einem Text findet.", + "Was ist der Unterschied zwischen Docker und einer VM?", + "Erkläre den Gradient Descent Algorithmus.", + "Wie funktioniert JWT (JSON Web Token) Authentifizierung?", + "Schreibe einen Kubernetes Deployment YAML für eine Web-App.", + "Was ist der Unterschied zwischen TCP und UDP?", + "Erkläre das Konzept von Dependency Injection.", + "Wie optimiert man eine langsame PostgreSQL-Abfrage?", + ] + + # Wiederhole Prompts um genügend Samples zu haben + prompts = (demo_prompts * ((num_samples // len(demo_prompts)) + 1))[:num_samples] + + train_data = [ + {"prompt": prompt_template.format(prompt=p)} + for p in prompts[: int(num_samples * 0.8)] + ] + eval_data = [ + {"prompt": prompt_template.format(prompt=p)} + for p in prompts[int(num_samples * 0.8) :] + ] + + train_dataset = Dataset.from_list(train_data) + eval_dataset = Dataset.from_list(eval_data) + + logger.info(f"Demo-Datensatz erstellt: {len(train_dataset)} train, {len(eval_dataset)} eval") + return train_dataset, eval_dataset + + +# --------------------------------------------------------------------------- +# Model Loading +# --------------------------------------------------------------------------- + + +def load_model_and_tokenizer( + config: ModelConfig, +) -> tuple[PreTrainedModel, PreTrainedTokenizer]: + """ + Lädt das Basis-Modell und den Tokenizer. + + Args: + config: ModelConfig mit Modell-Parametern. + + Returns: + Tuple aus (model, tokenizer). + """ + logger.info(f"Lade Modell: {config.model_name_or_path}") + + # Quantisierungskonfiguration + bnb_config = None + if config.load_in_4bit: + compute_dtype = getattr(torch, config.bnb_4bit_compute_dtype) + bnb_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=compute_dtype, + bnb_4bit_quant_type=config.bnb_4bit_quant_type, + bnb_4bit_use_double_quant=config.bnb_4bit_use_double_quant, + ) + logger.info("4-Bit Quantisierung aktiviert") + + # Tokenizer + tokenizer_path = config.tokenizer_name_or_path or config.model_name_or_path + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_path, + trust_remote_code=config.trust_remote_code, + ) + + # Padding-Token setzen falls nicht vorhanden + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + logger.info("pad_token auf eos_token gesetzt") + + # Modell + model_kwargs: dict[str, Any] = { + "trust_remote_code": config.trust_remote_code, + } + + if config.attn_implementation: + model_kwargs["attn_implementation"] = config.attn_implementation + + if bnb_config: + model_kwargs["quantization_config"] = bnb_config + else: + model_kwargs["torch_dtype"] = torch.bfloat16 + + model = AutoModelForCausalLM.from_pretrained( + config.model_name_or_path, + **model_kwargs, + ) + + logger.info(f"Modell geladen: {type(model).__name__}") + return model, tokenizer + + +def apply_lora( + model: PreTrainedModel, + lora_config: LocalLoRAConfig, +) -> PreTrainedModel: + """ + Wendet LoRA-Adapter auf das Modell an. + + Args: + model: Das Basis-Modell. + lora_config: LoRA-Konfiguration. + + Returns: + Modell mit LoRA-Adaptern. + """ + logger.info( + f"Wende LoRA an: r={lora_config.r}, alpha={lora_config.lora_alpha}" + ) + + # Modell für k-bit Training vorbereiten + model = prepare_model_for_kbit_training(model) + + # LoRA-Konfiguration + peft_config = LoraConfig( + r=lora_config.r, + lora_alpha=lora_config.lora_alpha, + target_modules=lora_config.target_modules, + lora_dropout=lora_config.lora_dropout, + bias=lora_config.bias, + task_type=lora_config.task_type, + ) + + model = get_peft_model(model, peft_config) + model.print_trainable_parameters() + + return model + + +# --------------------------------------------------------------------------- +# GRPO Training +# --------------------------------------------------------------------------- + + +def create_grpo_config(config: TrainingConfig) -> GRPOConfig: + """ + Erstellt eine TRL GRPOConfig aus der lokalen Konfiguration. + + Args: + config: TrainingConfig mit allen Parametern. + + Returns: + TRL GRPOConfig. + """ + grpo = config.grpo + + return GRPOConfig( + # GRPO-spezifisch + num_generations=grpo.num_generations, + max_prompt_length=grpo.max_prompt_length, + max_completion_length=grpo.max_completion_length, + temperature=grpo.temperature, + # Training + learning_rate=grpo.learning_rate, + num_train_epochs=grpo.num_epochs, + per_device_train_batch_size=grpo.per_device_train_batch_size, + gradient_accumulation_steps=grpo.gradient_accumulation_steps, + # Optimizer + optim=grpo.optim, + lr_scheduler_type=grpo.lr_scheduler_type, + warmup_ratio=grpo.warmup_ratio, + weight_decay=grpo.weight_decay, + # Logging & Saving + logging_steps=grpo.logging_steps, + save_steps=grpo.save_steps, + eval_strategy=grpo.eval_strategy, + eval_steps=grpo.eval_steps, + # Precision + bf16=grpo.bf16, + fp16=grpo.fp16, + gradient_checkpointing=grpo.gradient_checkpointing, + # Output + output_dir=grpo.output_dir, + report_to=grpo.report_to, + run_name=config.experiment_name, + seed=grpo.seed, + # GRPO Beta + beta=grpo.beta, + ) + + +def train( + config: TrainingConfig, + train_dataset: Dataset, + eval_dataset: Optional[Dataset] = None, +) -> str: + """ + Führt das GRPO-Training durch. + + Args: + config: Vollständige Trainingskonfiguration. + train_dataset: Trainingsdatensatz. + eval_dataset: Optionaler Evaluierungsdatensatz. + + Returns: + Pfad zum gespeicherten Modell. + """ + logger.info("=" * 60) + logger.info(f"Starte GRPO-Training: {config.experiment_name}") + logger.info("=" * 60) + + # 1. Modell & Tokenizer laden + model, tokenizer = load_model_and_tokenizer(config.model) + + # 2. LoRA anwenden + model = apply_lora(model, config.lora) + + # 3. Reward-Modell erstellen + reward_model = AgentRewardModel( + model_name_or_path=config.reward.reward_model_name_or_path, + reward_weights=config.reward.reward_weights, + use_model=False, # Regelbasiert für Geschwindigkeit + ) + reward_func = create_reward_function(reward_model) + + # 4. GRPO-Konfiguration + grpo_config = create_grpo_config(config) + + # 5. GRPO-Trainer + trainer = GRPOTrainer( + model=model, + processing_class=tokenizer, + args=grpo_config, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + reward_funcs=reward_func, + ) + + # 6. Training + logger.info("Beginne Training...") + try: + trainer.train() + except KeyboardInterrupt: + logger.info("Training durch Benutzer unterbrochen.") + except Exception as e: + logger.error(f"Fehler während des Trainings: {e}") + raise + + # 7. Modell speichern + output_dir = config.grpo.output_dir + logger.info(f"Speichere Modell nach {output_dir}") + trainer.save_model(output_dir) + tokenizer.save_pretrained(output_dir) + + logger.info("Training abgeschlossen!") + return output_dir + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + """CLI-Argumente parsen.""" + parser = argparse.ArgumentParser( + description="ART – GRPO-Training für Agenten mit LoRA", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Beispiele: + python train_agent.py # Qwen 2.5 7B (Standard) + python train_agent.py --model llama # Llama 3.1 8B + python train_agent.py --test-mode # Schneller Test (Qwen 1.5B) + python train_agent.py --train-file data.jsonl # Eigene Daten + python train_agent.py --output-dir ./my_model # Ausgabepfad + """, + ) + + parser.add_argument( + "--model", + choices=["qwen", "llama"], + default="qwen", + help="Modellfamilie (default: qwen)", + ) + parser.add_argument( + "--test-mode", + action="store_true", + help="Schneller Test-Modus mit kleinem Modell", + ) + parser.add_argument( + "--train-file", + type=str, + default="data/train.jsonl", + help="Pfad zur Trainings-JSONL-Datei", + ) + parser.add_argument( + "--eval-file", + type=str, + default=None, + help="Pfad zur Evaluierungs-JSONL-Datei", + ) + parser.add_argument( + "--output-dir", + type=str, + default=None, + help="Ausgabeverzeichnis für Checkpoints", + ) + parser.add_argument( + "--max-samples", + type=int, + default=None, + help="Maximale Anzahl Trainings-Samples", + ) + parser.add_argument( + "--no-wandb", + action="store_true", + help="W&B-Logging deaktivieren", + ) + parser.add_argument( + "--learning-rate", + type=float, + default=None, + help="Learning Rate (überschreibt Config)", + ) + parser.add_argument( + "--max-steps", + type=int, + default=None, + help="Maximale Trainingsschritte (überschreibt Config)", + ) + + return parser.parse_args() + + +def main() -> None: + """Hauptfunktion.""" + args = parse_args() + + # Konfiguration auswählen + if args.test_mode: + config = get_small_test_config() + logger.info("Test-Modus: Verwende kleine Konfiguration") + elif args.model == "llama": + config = get_llama_config() + logger.info("Llama-Modus: Verwende Llama 3.1 8B Konfiguration") + else: + config = get_qwen_config() + logger.info("Qwen-Modus: Verwende Qwen 2.5 7B Konfiguration") + + # CLI-Überschreibungen + if args.output_dir: + config.grpo.output_dir = args.output_dir + if args.learning_rate is not None: + config.grpo.learning_rate = args.learning_rate + if args.max_steps is not None: + config.grpo.max_steps = args.max_steps + if args.no_wandb: + config.grpo.report_to = "none" + + # Daten laden + train_dataset, eval_dataset = load_training_data( + train_file=args.train_file, + eval_file=args.eval_file, + prompt_template=config.data.prompt_template, + max_samples=args.max_samples, + ) + + # Training starten + output_dir = train( + config=config, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + ) + + logger.info(f"Training abgeschlossen. Modell gespeichert in: {output_dir}") + + +if __name__ == "__main__": + main() From ec8dc87d107f22aad44efd0077780f4c1f8db47e Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Wed, 29 Jul 2026 05:36:54 +0000 Subject: [PATCH 03/20] =?UTF-8?q?Aufr=C3=A4umen:=20Ungenutzte=20Imports=20?= =?UTF-8?q?entfernt,=20Unit-Tests=20f=C3=BCr=20Reward-Modell=20und=20Konfi?= =?UTF-8?q?guration=20hinzugef=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - train_agent.py: Ungenutzte Imports (annotations, dataclass, Path, load_dataset, TrainingArguments, LocalGRPOConfig) entfernt - reward_model.py: Ungenutzte TYPE_CHECKING-Imports (PreTrainedModel, PreTrainedTokenizer) entfernt - tests/unit/test_reward_model.py: 24 Unit-Tests für AgentRewardModel, create_reward_function und RewardModelWrapper - tests/unit/test_grpo_config.py: 19 Unit-Tests für ModelConfig, LoRAConfig, GRPOConfig, RewardConfig, DataConfig, TrainingConfig und Preset-Configs - Alle 43 Tests bestanden (1 skipped wegen fehlendem torch) --- reward_model.py | 1 - tests/unit/test_grpo_config.py | 202 ++++++++++++++++++++++++++ tests/unit/test_reward_model.py | 249 ++++++++++++++++++++++++++++++++ train_agent.py | 8 +- 4 files changed, 452 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_grpo_config.py create mode 100644 tests/unit/test_reward_model.py diff --git a/reward_model.py b/reward_model.py index 886e50fde..25d5a0853 100644 --- a/reward_model.py +++ b/reward_model.py @@ -21,7 +21,6 @@ if TYPE_CHECKING: import torch import torch.nn as nn - from transformers import PreTrainedModel, PreTrainedTokenizer class AgentRewardModel: diff --git a/tests/unit/test_grpo_config.py b/tests/unit/test_grpo_config.py new file mode 100644 index 000000000..2db2a7bc7 --- /dev/null +++ b/tests/unit/test_grpo_config.py @@ -0,0 +1,202 @@ +"""Unit-Tests für die Trainingskonfiguration (config.py).""" + +import pytest + +from config import ( + DataConfig, + GRPOConfig, + LoRAConfig, + ModelConfig, + RewardConfig, + TrainingConfig, + get_llama_config, + get_qwen_config, + get_small_test_config, +) + + +class TestModelConfig: + """Tests für ModelConfig.""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = ModelConfig() + assert cfg.model_name_or_path == "Qwen/Qwen2.5-7B-Instruct" + assert cfg.load_in_4bit is True + assert cfg.bnb_4bit_compute_dtype == "bfloat16" + assert cfg.bnb_4bit_quant_type == "nf4" + assert cfg.bnb_4bit_use_double_quant is True + assert cfg.attn_implementation == "flash_attention_2" + assert cfg.trust_remote_code is False + assert cfg.tokenizer_name_or_path is None + + def test_custom_model(self): + """Test: Benutzerdefinierte Werte werden übernommen.""" + cfg = ModelConfig( + model_name_or_path="meta-llama/Llama-3.1-8B-Instruct", + load_in_4bit=False, + trust_remote_code=True, + ) + assert cfg.model_name_or_path == "meta-llama/Llama-3.1-8B-Instruct" + assert cfg.load_in_4bit is False + assert cfg.trust_remote_code is True + + +class TestLoRAConfig: + """Tests für LoRAConfig.""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = LoRAConfig() + assert cfg.r == 16 + assert cfg.lora_alpha == 32 + assert cfg.lora_dropout == 0.05 + assert cfg.bias == "none" + assert cfg.task_type == "CAUSAL_LM" + assert "q_proj" in cfg.target_modules + assert "v_proj" in cfg.target_modules + + def test_custom_rank(self): + """Test: Benutzerdefinierter Rank.""" + cfg = LoRAConfig(r=8, lora_alpha=16) + assert cfg.r == 8 + assert cfg.lora_alpha == 16 + + def test_target_modules_is_list(self): + """Test: target_modules ist eine Liste von Strings.""" + cfg = LoRAConfig() + assert isinstance(cfg.target_modules, list) + assert all(isinstance(m, str) for m in cfg.target_modules) + + +class TestGRPOConfig: + """Tests für GRPOConfig.""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = GRPOConfig() + assert cfg.num_generations == 4 + assert cfg.max_prompt_length == 2048 + assert cfg.max_completion_length == 1024 + assert cfg.temperature == 0.9 + assert cfg.learning_rate == 5e-6 + assert cfg.beta == 0.04 + assert cfg.per_device_train_batch_size == 2 + assert cfg.gradient_accumulation_steps == 4 + assert cfg.bf16 is True + assert cfg.fp16 is False + assert cfg.seed == 42 + assert cfg.output_dir == "./output/grpo-lora" + + def test_custom_learning_rate(self): + """Test: Benutzerdefinierte Learning Rate.""" + cfg = GRPOConfig(learning_rate=1e-4) + assert cfg.learning_rate == 1e-4 + + def test_max_steps_default(self): + """Test: max_steps default ist -1 (voller Datensatz).""" + cfg = GRPOConfig() + assert cfg.max_steps == -1 + + +class TestRewardConfig: + """Tests für RewardConfig.""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = RewardConfig() + assert cfg.reward_model_name_or_path == "Qwen/Qwen2.5-7B-Instruct" + assert cfg.max_length == 4096 + assert cfg.reward_weights["correctness"] == 1.0 + assert cfg.reward_weights["safety"] == 0.8 + assert cfg.correctness_threshold == 0.5 + assert cfg.safety_threshold == 0.3 + + def test_custom_weights(self): + """Test: Benutzerdefinierte Reward-Gewichte.""" + custom = {"correctness": 2.0, "format": 0.5} + cfg = RewardConfig(reward_weights=custom) + assert cfg.reward_weights == custom + + +class TestDataConfig: + """Tests für DataConfig.""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = DataConfig() + assert cfg.train_file == "data/train.jsonl" + assert cfg.eval_file == "data/eval.jsonl" + assert cfg.dataset_format == "standard" + assert "{prompt}" in cfg.prompt_template + assert "<|im_start|>" in cfg.prompt_template + + def test_custom_files(self): + """Test: Benutzerdefinierte Dateipfade.""" + cfg = DataConfig( + train_file="my_train.jsonl", + eval_file="my_eval.jsonl", + ) + assert cfg.train_file == "my_train.jsonl" + assert cfg.eval_file == "my_eval.jsonl" + + +class TestTrainingConfig: + """Tests für TrainingConfig (Gesamtkonfiguration).""" + + def test_default_values(self): + """Test: Standardwerte sind korrekt.""" + cfg = TrainingConfig() + assert isinstance(cfg.model, ModelConfig) + assert isinstance(cfg.lora, LoRAConfig) + assert isinstance(cfg.grpo, GRPOConfig) + assert isinstance(cfg.reward, RewardConfig) + assert isinstance(cfg.data, DataConfig) + assert cfg.experiment_name == "art-grpo-lora" + assert cfg.resume_from_checkpoint is None + + def test_custom_experiment_name(self): + """Test: Benutzerdefinierter Experiment-Name.""" + cfg = TrainingConfig(experiment_name="my-experiment") + assert cfg.experiment_name == "my-experiment" + + def test_resume_from_checkpoint(self): + """Test: Resume-Checkpoint wird gespeichert.""" + cfg = TrainingConfig(resume_from_checkpoint="./checkpoints/step-100") + assert cfg.resume_from_checkpoint == "./checkpoints/step-100" + + +class TestPresetConfigs: + """Tests für die vordefinierten Konfigurationen.""" + + def test_get_qwen_config(self): + """Test: Qwen-Konfiguration hat korrekte Werte.""" + cfg = get_qwen_config() + assert cfg.model.model_name_or_path == "Qwen/Qwen2.5-7B-Instruct" + assert cfg.experiment_name == "art-qwen-7b-grpo" + + def test_get_llama_config(self): + """Test: Llama-Konfiguration hat korrekte Werte.""" + cfg = get_llama_config() + assert cfg.model.model_name_or_path == "meta-llama/Llama-3.1-8B-Instruct" + assert cfg.experiment_name == "art-llama-8b-grpo" + # Llama hat ein anderes Prompt-Template + assert "<|begin_of_text|>" in cfg.data.prompt_template + + def test_get_small_test_config(self): + """Test: Test-Konfiguration hat reduzierte Werte.""" + cfg = get_small_test_config() + assert cfg.model.model_name_or_path == "Qwen/Qwen2.5-1.5B-Instruct" + assert cfg.model.load_in_4bit is False + assert cfg.lora.r == 8 + assert cfg.lora.lora_alpha == 16 + assert cfg.grpo.num_generations == 2 + assert cfg.grpo.max_prompt_length == 512 + assert cfg.grpo.max_completion_length == 256 + assert cfg.grpo.max_steps == 50 + assert cfg.experiment_name == "art-test-grpo" + + def test_all_configs_are_training_config(self): + """Test: Alle vordefinierten Konfigurationen sind TrainingConfig-Instanzen.""" + for cfg in [get_qwen_config(), get_llama_config(), get_small_test_config()]: + assert isinstance(cfg, TrainingConfig) diff --git a/tests/unit/test_reward_model.py b/tests/unit/test_reward_model.py new file mode 100644 index 000000000..b961023a0 --- /dev/null +++ b/tests/unit/test_reward_model.py @@ -0,0 +1,249 @@ +"""Unit-Tests für das Reward-Modell (reward_model.py).""" + +import pytest + +from reward_model import AgentRewardModel, RewardModelWrapper, create_reward_function + + +class TestAgentRewardModel: + """Tests für die regelbasierte Reward-Berechnung.""" + + @pytest.fixture + def reward_model(self): + """Erstellt ein regelbasiertes Reward-Modell mit Standard-Gewichten.""" + return AgentRewardModel(use_model=False) + + def test_init_default_weights(self): + """Test: Standard-Gewichte werden korrekt gesetzt.""" + model = AgentRewardModel(use_model=False) + assert model.reward_weights["correctness"] == 1.0 + assert model.reward_weights["format"] == 0.3 + assert model.reward_weights["helpfulness"] == 0.5 + assert model.reward_weights["safety"] == 0.8 + assert model.reward_weights["tool_usage"] == 0.4 + assert model.use_model is False + assert model.model is None + + def test_init_custom_weights(self): + """Test: Benutzerdefinierte Gewichte werden übernommen.""" + custom = {"correctness": 2.0, "safety": 1.0} + model = AgentRewardModel(reward_weights=custom, use_model=False) + assert model.reward_weights == custom + + def test_compute_reward_returns_total(self, reward_model): + """Test: compute_reward gibt ein Dictionary mit 'total'-Key zurück.""" + result = reward_model.compute_reward( + prompt="Was ist Python?", + completion="Python ist eine Programmiersprache.", + ) + assert "total" in result + assert isinstance(result["total"], float) + assert 0.0 <= result["total"] <= 5.0 # Max sum of weights + + def test_compute_reward_all_keys_present(self, reward_model): + """Test: Alle Reward-Komponenten sind im Ergebnis enthalten.""" + result = reward_model.compute_reward( + prompt="Erkläre Docker.", + completion="Docker ist eine Container-Plattform.", + ) + for key in ["correctness", "format", "helpfulness", "safety", "tool_usage", "total"]: + assert key in result, f"Key '{key}' fehlt im Ergebnis" + + def test_correctness_exact_match(self, reward_model): + """Test: Exakte Übereinstimmung mit Ground-Truth ergibt 1.0.""" + result = reward_model.compute_reward( + prompt="Was ist 2+2?", + completion="4", + ground_truth="4", + ) + assert result["correctness"] == 1.0 + + def test_correctness_partial_match(self, reward_model): + """Test: Teilweise Übereinstimmung (Ground-Truth in Completion) ergibt 0.7.""" + result = reward_model.compute_reward( + prompt="Erkläre Python.", + completion="Python ist eine Programmiersprache. Sie ist weit verbreitet.", + ground_truth="Python ist eine Programmiersprache.", + ) + assert result["correctness"] == 0.7 + + def test_correctness_keyword_overlap(self, reward_model): + """Test: Keyword-Überlappung ohne exakte Teilstring-Übereinstimmung.""" + result = reward_model.compute_reward( + prompt="Erkläre Python.", + completion="Python ist eine großartige Programmiersprache.", + ground_truth="Python ist eine Programmiersprache.", + ) + # Keyword-Überlappung: 4 von 4 Wörtern → min(1.0, 0.5) = 0.5 + assert result["correctness"] == 0.5 + + def test_correctness_no_ground_truth(self, reward_model): + """Test: Ohne Ground-Truth wird 0.5 zurückgegeben.""" + result = reward_model.compute_reward( + prompt="Was ist Rust?", + completion="Rust ist eine Systems-Programmiersprache.", + ) + assert result["correctness"] == 0.5 + + def test_correctness_no_match(self, reward_model): + """Test: Keine Übereinstimmung ergibt 0.0.""" + result = reward_model.compute_reward( + prompt="Erkläre Python.", + completion="Ein völlig anderes Thema ohne Bezug.", + ground_truth="Python ist eine Programmiersprache.", + ) + assert result["correctness"] == 0.0 + + def test_format_code_block(self, reward_model): + """Test: Code-Blöcke erhöhen den Format-Score.""" + result = reward_model.compute_reward( + prompt="Schreibe Code.", + completion="Hier ist der Code:\n```python\nprint('hello')\n```", + ) + assert result["format"] > 0.0 + + def test_format_empty_completion(self, reward_model): + """Test: Leere Completion hat niedrigen Format-Score.""" + result = reward_model.compute_reward( + prompt="Was ist das?", + completion="", + ) + assert result["format"] < 0.5 + + def test_helpfulness_non_empty(self, reward_model): + """Test: Nicht-leere Antwort erhält Basis-Score.""" + result = reward_model.compute_reward( + prompt="Hilfe!", + completion="Hier ist eine ausführliche Erklärung mit vielen Details und Beispielen.", + ) + assert result["helpfulness"] > 0.0 + + def test_helpfulness_empty(self, reward_model): + """Test: Leere Antwort hat niedrigen Helpfulness-Score.""" + result = reward_model.compute_reward( + prompt="Hilfe!", + completion="", + ) + assert result["helpfulness"] < 0.5 + + def test_safety_clean_content(self, reward_model): + """Test: Saubere Inhalte haben hohen Safety-Score.""" + result = reward_model.compute_reward( + prompt="Wie geht's?", + completion="Mir geht es gut, danke der Nachfrage!", + ) + assert result["safety"] >= 0.7 + + def test_safety_unsafe_pattern(self, reward_model): + """Test: Unsichere Muster reduzieren den Safety-Score.""" + result = reward_model.compute_reward( + prompt="Wie hacke ich?", + completion="Du kannst das System hacken mit rm -rf /", + ) + assert result["safety"] < 1.0 + + def test_safety_refusal(self, reward_model): + """Test: Refusal-Antworten haben hohen Safety-Score.""" + result = reward_model.compute_reward( + prompt="Wie hacke ich?", + completion="Entschuldigung, ich kann nicht bei illegalen Aktivitäten helfen.", + ) + assert result["safety"] >= 0.8 + + def test_tool_usage_expected_tools(self, reward_model): + """Test: Erwartete Tools werden erkannt.""" + result = reward_model.compute_reward( + prompt="Suche etwas.", + completion="Ich nutze die search-Funktion und den calculator.", + tools_expected=["search", "calculator"], + ) + assert result["tool_usage"] > 0.5 + + def test_tool_usage_no_expected(self, reward_model): + """Test: Ohne erwartete Tools wird 0.5 zurückgegeben.""" + result = reward_model.compute_reward( + prompt="Suche etwas.", + completion="Ich suche mit der search-Funktion.", + ) + assert result["tool_usage"] == 0.5 + + def test_tool_usage_none_found(self, reward_model): + """Test: Keine der erwarteten Tools gefunden.""" + result = reward_model.compute_reward( + prompt="Rechne etwas.", + completion="Das Ergebnis ist 42.", + tools_expected=["calculator", "math"], + ) + assert result["tool_usage"] == 0.0 + + def test_total_reward_weighted_sum(self, reward_model): + """Test: Der Gesamt-Reward ist die gewichtete Summe der Einzel-Rewards.""" + result = reward_model.compute_reward( + prompt="Test", + completion="Test-Antwort", + ) + expected_total = ( + result["correctness"] * 1.0 + + result["format"] * 0.3 + + result["helpfulness"] * 0.5 + + result["safety"] * 0.8 + + result["tool_usage"] * 0.4 + ) + assert result["total"] == pytest.approx(expected_total) + + +class TestCreateRewardFunction: + """Tests für die create_reward_function-Hilfsfunktion.""" + + def test_returns_callable(self): + """Test: Gibt eine aufrufbare Funktion zurück.""" + model = AgentRewardModel(use_model=False) + func = create_reward_function(model) + assert callable(func) + + def test_reward_func_returns_list_of_floats(self): + """Test: Die Reward-Funktion gibt eine Liste von Floats zurück.""" + model = AgentRewardModel(use_model=False) + func = create_reward_function(model) + prompts = ["Frage 1", "Frage 2"] + completions = ["Antwort 1", "Antwort 2"] + rewards = func(prompts, completions) + assert len(rewards) == 2 + assert all(isinstance(r, float) for r in rewards) + + def test_reward_func_empty_lists(self): + """Test: Leere Listen ergeben leere Reward-Liste.""" + model = AgentRewardModel(use_model=False) + func = create_reward_function(model) + rewards = func([], []) + assert rewards == [] + + def test_reward_func_passes_ground_truth(self): + """Test: Ground-Truth wird als kwargs durchgereicht.""" + model = AgentRewardModel(use_model=False) + func = create_reward_function(model) + rewards = func( + ["Was ist 2+2?"], + ["4"], + ground_truth="4", + ) + assert len(rewards) == 1 + assert rewards[0] > 0.0 + + +class TestRewardModelWrapper: + """Tests für den RewardModelWrapper (ohne echtes Modell).""" + + def test_init_creates_reward_head(self): + """Test: Der Wrapper erstellt einen Reward-Head.""" + pytest.importorskip("torch", reason="torch nicht installiert") + # Mock ein base_model mit config.hidden_size + class MockConfig: + hidden_size = 768 + + class MockModel: + config = MockConfig() + + wrapper = RewardModelWrapper(base_model=MockModel()) + assert hasattr(wrapper, "reward_head") + assert hasattr(wrapper, "base_model") diff --git a/train_agent.py b/train_agent.py index 735b6fea1..1907c1df3 100644 --- a/train_agent.py +++ b/train_agent.py @@ -14,19 +14,15 @@ python train_agent.py --config my_config.py # Benutzerdefinierte Config """ -from __future__ import annotations - import argparse import json import logging import os import sys -from dataclasses import dataclass -from pathlib import Path from typing import Any, Optional import torch -from datasets import Dataset, load_dataset +from datasets import Dataset from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training from transformers import ( AutoModelForCausalLM, @@ -34,13 +30,11 @@ BitsAndBytesConfig, PreTrainedModel, PreTrainedTokenizer, - TrainingArguments, ) from trl import GRPOConfig, GRPOTrainer # Lokale Imports from config import ( - GRPOConfig as LocalGRPOConfig, LoRAConfig as LocalLoRAConfig, ModelConfig, TrainingConfig, From f7764300fc24b6b1fd23b777feebac6b36c526ef Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Wed, 29 Jul 2026 16:16:49 +0000 Subject: [PATCH 04/20] =?UTF-8?q?Code-Qualit=C3=A4t:=20Import-Sortierung?= =?UTF-8?q?=20mit=20ruff=20korrigiert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 2 +- reward_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 03d27faf0..82dba008a 100644 --- a/config.py +++ b/config.py @@ -4,7 +4,7 @@ """ from dataclasses import dataclass, field -from typing import Optional, Literal +from typing import Literal, Optional @dataclass diff --git a/reward_model.py b/reward_model.py index 25d5a0853..fc6964b76 100644 --- a/reward_model.py +++ b/reward_model.py @@ -16,7 +16,7 @@ from __future__ import annotations import re -from typing import Any, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: import torch From 7f91e5f093ea9401fa1b866ce015401fbb29f7f8 Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Wed, 29 Jul 2026 23:18:46 +0000 Subject: [PATCH 05/20] =?UTF-8?q?=F0=9F=93=B1=20Streamlit-App:=20Interakti?= =?UTF-8?q?ve=20Demo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py | 477 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 app.py diff --git a/app.py b/app.py new file mode 100644 index 000000000..ad2ddb9dd --- /dev/null +++ b/app.py @@ -0,0 +1,477 @@ +""" +Streamlit-App: ART — Agent Reinforcement Trainer +================================================ +GRPO-Training konfigurieren, Reward-Modell testen, LoRA-Parameter einstellen. +""" + +import streamlit as st +import matplotlib.pyplot as plt +import numpy as np +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from config import ( + TrainingConfig, ModelConfig, LoRAConfig, GRPOConfig, RewardConfig, DataConfig, + get_qwen_config, get_llama_config, get_small_test_config, +) +from reward_model import AgentRewardModel + +st.set_page_config( + page_title="ART — Agent Reinforcement Trainer", + page_icon="🤖", + layout="wide", +) + +st.title("🤖 ART — Agent Reinforcement Trainer") +st.markdown("### GRPO-Training konfigurieren · Reward-Modell testen · LoRA-Parameter einstellen") + +# ── Sidebar ── +seite = st.sidebar.radio( + "📂 Bereich wählen", + ["🎯 GRPO-Training", "🏆 Reward-Modell", "🔧 LoRA-Konfiguration", "📋 Gesamtkonfiguration"], +) + +# ═══════════════════════════════════════════════════════════════ +# GRPO-TRAINING +# ═══════════════════════════════════════════════════════════════ +if seite == "🎯 GRPO-Training": + st.header("🎯 GRPO-Training konfigurieren") + + st.markdown(""" + **GRPO** (Group Relative Policy Optimization) ist ein RL-Verfahren für + Sprachmodelle. Es generiert mehrere Antworten pro Prompt, bewertet sie + mit einem Reward-Modell und optimiert die Policy relativ zur Gruppe. + """) + + col1, col2 = st.columns(2) + + with col1: + st.subheader("📊 GRPO-Parameter") + + num_generations = st.slider("Anzahl Generierungen (G)", 1, 8, 4, + help="Gruppengröße: Wie viele Antworten pro Prompt generiert werden.") + max_prompt_length = st.slider("Max. Prompt-Länge", 256, 4096, 2048, 128) + max_completion_length = st.slider("Max. Completion-Länge", 128, 2048, 1024, 128) + temperature = st.slider("Temperatur", 0.1, 2.0, 0.9, 0.1) + top_p = st.slider("Top-p", 0.5, 1.0, 1.0, 0.05) + + with col2: + st.subheader("⚡ Training") + + learning_rate = st.number_input("Learning Rate", 1e-7, 1e-3, 5e-6, format="%.1e") + beta = st.slider("Beta (KL-Koeffizient)", 0.0, 0.2, 0.04, 0.01, + help="Steuert, wie stark die Policy vom Reference-Modell abweichen darf.") + num_epochs = st.slider("Epochen pro GRPO-Schritt", 1, 5, 1) + grad_accum = st.slider("Gradient Accumulation Steps", 1, 16, 4) + batch_size = st.slider("Batch Size pro Device", 1, 8, 2) + + # GRPO-Workflow visualisieren + st.markdown("---") + st.subheader("🔄 GRPO-Workflow") + + fig, ax = plt.subplots(figsize=(12, 5)) + ax.set_xlim(0, 12) + ax.set_ylim(0, 6) + ax.axis('off') + + steps = [ + (1, 4.5, "1. Prompt", '#4ECDC4'), + (3, 4.5, "2. Generiere\nG Antworten", '#45B7D1'), + (5, 4.5, "3. Reward\nberechnen", '#FFE66D'), + (7, 4.5, "4. Advantage\n(Gruppen-Norm.)", '#F38181'), + (9, 4.5, "5. Policy\nUpdate (PPO)", '#FF6B6B'), + (11, 4.5, "6. KL-Div\nprüfen", '#AA96DA'), + ] + + for x, y, text, color in steps: + rect = plt.Rectangle((x - 0.8, y - 0.6), 1.6, 1.2, + facecolor=color, edgecolor='white', + linewidth=2, alpha=0.9, zorder=2) + ax.add_patch(rect) + ax.text(x, y, text, ha='center', va='center', fontsize=9, + fontweight='bold', color='white', zorder=3) + + # Pfeile + for i in range(len(steps) - 1): + ax.annotate('', xy=(steps[i + 1][0] - 0.8, steps[i + 1][1]), + xytext=(steps[i][0] + 0.8, steps[i][1]), + arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) + + # Rückkopplung + ax.annotate('', xy=(1, 2.5), xytext=(11, 2.5), + arrowprops=dict(arrowstyle='->', color='#FF6B6B', lw=1.5, + connectionstyle='arc3,rad=-0.3')) + ax.text(6, 2.0, "↻ Iteriere bis Konvergenz", ha='center', fontsize=10, + color='#FF6B6B', style='italic') + + ax.set_title("GRPO Training Loop", fontsize=14, fontweight='bold') + st.pyplot(fig) + plt.close(fig) + + # Parameter-Impact + st.markdown("---") + st.subheader("📈 Parameter-Einfluss") + + # Beta vs KL-Divergence + betas = np.linspace(0.01, 0.2, 20) + kl_penalty = betas * 10 # Simulierter Effekt + + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4)) + + ax1.plot(betas, kl_penalty, color='#FF6B6B', linewidth=2) + ax1.fill_between(betas, 0, kl_penalty, alpha=0.2, color='#FF6B6B') + ax1.set_xlabel("Beta (KL-Koeffizient)") + ax1.set_ylabel("KL-Penalty") + ax1.set_title("Beta → KL-Divergence-Kontrolle") + ax1.grid(True, alpha=0.3) + + # Gruppengröße vs Variance + groups = np.arange(1, 9) + advantage_variance = 1.0 / np.sqrt(groups) + ax2.bar(groups, advantage_variance, color='#4ECDC4', edgecolor='white') + ax2.set_xlabel("Gruppengröße (G)") + ax2.set_ylabel("Advantage-Varianz (relativ)") + ax2.set_title("Gruppengröße → Schätzgenauigkeit") + ax2.grid(True, alpha=0.3, axis='y') + + st.pyplot(fig) + plt.close(fig) + +# ═══════════════════════════════════════════════════════════════ +# REWARD-MODELL +# ═══════════════════════════════════════════════════════════════ +elif seite == "🏆 Reward-Modell": + st.header("🏆 Reward-Modell testen") + + st.markdown(""" + Das Reward-Modell bewertet Agent-Antworten anhand mehrerer Kriterien. + Hier kannst du es mit eigenen Beispielen testen. + """) + + col1, col2 = st.columns([1, 1]) + + with col1: + st.subheader("📝 Eingabe") + + prompt = st.text_area( + "Prompt", + value="Erkläre, wie ein Transformer-Modell funktioniert.", + height=80, + ) + + completion = st.text_area( + "Agent-Antwort", + value=( + "Ein Transformer-Modell basiert auf dem Attention-Mechanismus. " + "Es verarbeitet Eingaben parallel statt sequentiell. " + "Die Self-Attention berechnet für jedes Token die Relevanz " + "aller anderen Tokens im Kontext. Das ermöglicht es dem Modell, " + "langreichweitige Abhängigkeiten zu erfassen." + ), + height=150, + ) + + ground_truth = st.text_input( + "Ground Truth (optional)", + value="Transformer nutzen Self-Attention zur parallelen Verarbeitung von Sequenzen.", + ) + + tools_expected = st.multiselect( + "Erwartete Tools", + ["search", "calculator", "code_interpreter", "web_browser"], + [], + ) + + if st.button("🏆 Reward berechnen", type="primary", use_container_width=True): + st.session_state.reward_clicked = True + else: + if "reward_clicked" not in st.session_state: + st.session_state.reward_clicked = False + + with col2: + st.subheader("📊 Reward-Ergebnis") + + if st.session_state.reward_clicked: + with st.spinner("Berechne Reward..."): + reward_model = AgentRewardModel(use_model=False) + result = reward_model.compute_reward( + prompt=prompt, + completion=completion, + ground_truth=ground_truth if ground_truth else None, + tools_expected=tools_expected if tools_expected else None, + ) + + # Gesamt-Reward + total = result["total"] + color = "green" if total > 1.5 else "orange" if total > 0.8 else "red" + st.markdown(f"### Gesamt-Reward: {total:.3f}", + unsafe_allow_html=True) + + # Einzel-Rewards + reward_items = {k: v for k, v in result.items() if k != "total"} + fig, ax = plt.subplots(figsize=(6, 4)) + names = list(reward_items.keys()) + values = list(reward_items.values()) + colors = ['#4ECDC4', '#45B7D1', '#FFE66D', '#FF6B6B', '#F38181'][:len(names)] + bars = ax.barh(names, values, color=colors, edgecolor='white') + ax.set_xlim(0, 1.0) + ax.set_xlabel("Score") + ax.set_title("Reward-Komponenten") + for bar, val in zip(bars, values): + ax.text(bar.get_width() + 0.02, bar.get_y() + bar.get_height() / 2, + f"{val:.2f}", va='center', fontsize=10) + st.pyplot(fig) + plt.close(fig) + + # Details + st.markdown("**Details:**") + for name, val in reward_items.items(): + emoji = "✅" if val > 0.7 else "⚠️" if val > 0.3 else "❌" + st.markdown(f"{emoji} **{name}**: {val:.3f}") + else: + st.info("👈 Gib links Prompt und Antwort ein und klicke auf **Reward berechnen**.") + + # Reward-Gewichte + st.markdown("---") + st.subheader("⚖️ Reward-Gewichte konfigurieren") + + col_a, col_b = st.columns(2) + + with col_a: + w_correctness = st.slider("Correctness", 0.0, 2.0, 1.0, 0.1) + w_format = st.slider("Format", 0.0, 2.0, 0.3, 0.1) + w_helpfulness = st.slider("Helpfulness", 0.0, 2.0, 0.5, 0.1) + + with col_b: + w_safety = st.slider("Safety", 0.0, 2.0, 0.8, 0.1) + w_tool_usage = st.slider("Tool Usage", 0.0, 2.0, 0.4, 0.1) + + # Gewichte visualisieren + weights = { + "Correctness": w_correctness, + "Format": w_format, + "Helpfulness": w_helpfulness, + "Safety": w_safety, + "Tool Usage": w_tool_usage, + } + + fig, ax = plt.subplots(figsize=(6, 3)) + names = list(weights.keys()) + values = list(weights.values()) + colors = ['#4ECDC4', '#45B7D1', '#FFE66D', '#FF6B6B', '#F38181'] + ax.bar(names, values, color=colors, edgecolor='white') + ax.set_ylabel("Gewicht") + ax.set_title("Reward-Gewichte") + ax.axhline(y=1.0, color='gray', linestyle='--', alpha=0.5) + st.pyplot(fig) + plt.close(fig) + +# ═══════════════════════════════════════════════════════════════ +# LORA-KONFIGURATION +# ═══════════════════════════════════════════════════════════════ +elif seite == "🔧 LoRA-Konfiguration": + st.header("🔧 LoRA-Parameter einstellen") + + st.markdown(""" + **LoRA** (Low-Rank Adaptation) fügt trainierbare Low-Rank-Matrizen zu + eingefrorenen Gewichten hinzu. Das reduziert die trainierbaren Parameter + drastisch (typisch <1% des Originalmodells). + """) + + col1, col2 = st.columns(2) + + with col1: + st.subheader("📐 LoRA-Dimensionen") + + lora_r = st.slider("Rank (r)", 1, 128, 16, + help="Rank der Low-Rank-Approximation. Höher = mehr Kapazität, mehr Parameter.") + lora_alpha = st.slider("Alpha", 1, 128, 32, + help="Skalierungsfaktor. Effektive LR skaliert mit alpha/r.") + lora_dropout = st.slider("Dropout", 0.0, 0.5, 0.05, 0.01) + + st.markdown(f"**Effektiver Skalierungsfaktor:** α/r = {lora_alpha / lora_r:.2f}") + + with col2: + st.subheader("🎯 Zielmodule") + + target_modules = st.multiselect( + "Module für LoRA-Adapter", + ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], + ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], + ) + + bias_type = st.selectbox("Bias-Typ", ["none", "all", "lora_only"]) + + # LoRA-Parameter berechnen + st.markdown("---") + st.subheader("📊 LoRA-Parameter-Analyse") + + # Beispiel: Qwen2.5-7B + base_params = 7_000_000_000 # 7B + hidden_size = 4096 # typisch für 7B-Modelle + + lora_params_per_module = 2 * hidden_size * lora_r # A und B Matrix + total_lora_params = lora_params_per_module * len(target_modules) + lora_ratio = total_lora_params / base_params * 100 + + col_a, col_b, col_c = st.columns(3) + with col_a: + st.metric("Basis-Parameter", f"{base_params / 1e9:.1f}B") + with col_b: + st.metric("LoRA-Parameter", f"{total_lora_params / 1e6:.2f}M") + with col_c: + st.metric("LoRA-Anteil", f"{lora_ratio:.4f}%") + + # Rank vs Parameter + ranks = [1, 2, 4, 8, 16, 32, 64, 128] + params_for_rank = [2 * hidden_size * r * len(target_modules) / 1e6 for r in ranks] + + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4)) + + ax1.plot(ranks, params_for_rank, 'o-', color='#4ECDC4', linewidth=2, markersize=8) + ax1.set_xlabel("LoRA Rank (r)") + ax1.set_ylabel("Trainierbare Parameter (M)") + ax1.set_title("Rank → Parameter") + ax1.grid(True, alpha=0.3) + ax1.axvline(x=lora_r, color='#FF6B6B', linestyle='--', alpha=0.5, + label=f'Aktuell: r={lora_r}') + ax1.legend() + + # Modul-Beitrag + modules = target_modules if target_modules else ["q_proj", "k_proj", "v_proj", "o_proj"] + mod_params = [2 * hidden_size * lora_r / 1e6] * len(modules) + ax2.pie(mod_params, labels=modules, autopct='%1.1f%%', + colors=plt.cm.Set3(np.linspace(0, 1, len(modules)))) + ax2.set_title(f"Parameter-Verteilung (r={lora_r})") + + st.pyplot(fig) + plt.close(fig) + + # LoRA-Architektur-Diagramm + st.markdown("---") + st.subheader("🎨 LoRA-Architektur") + + fig, ax = plt.subplots(figsize=(10, 5)) + ax.set_xlim(0, 10) + ax.set_ylim(0, 6) + ax.axis('off') + + # Original Weight + rect = plt.Rectangle((1, 2.5), 3, 2, facecolor='#E8E8E8', edgecolor='#888888', + linewidth=2, alpha=0.8, zorder=1) + ax.add_patch(rect) + ax.text(2.5, 3.5, "Eingefrorene\nGewichte W\n(d × k)", ha='center', va='center', + fontsize=11, fontweight='bold', color='#555') + + # LoRA A + rect_a = plt.Rectangle((5, 3.5), 1.5, 1, facecolor='#4ECDC4', edgecolor='white', + linewidth=2, alpha=0.9, zorder=2) + ax.add_patch(rect_a) + ax.text(5.75, 4.0, "A\n(d × r)", ha='center', va='center', + fontsize=10, fontweight='bold', color='white') + + # LoRA B + rect_b = plt.Rectangle((7, 3.5), 1.5, 1, facecolor='#FF6B6B', edgecolor='white', + linewidth=2, alpha=0.9, zorder=2) + ax.add_patch(rect_b) + ax.text(7.75, 4.0, "B\n(r × k)", ha='center', va='center', + fontsize=10, fontweight='bold', color='white') + + # Output + ax.text(9.5, 4.0, "ΔW = α/r · BA", ha='center', va='center', + fontsize=10, fontweight='bold', color='#333') + + # Pfeile + ax.annotate('', xy=(5, 4.0), xytext=(4, 4.0), + arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) + ax.annotate('', xy=(7, 4.0), xytext=(6.5, 4.0), + arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) + ax.annotate('', xy=(9, 4.0), xytext=(8.5, 4.0), + arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) + + # Input + ax.text(0.5, 4.0, "Input x", ha='center', fontsize=10, fontweight='bold') + ax.annotate('', xy=(1, 4.0), xytext=(0.8, 4.0), + arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) + + # Formel + ax.text(5, 1.5, "h = Wx + (α/r) · BAx", ha='center', fontsize=14, + fontweight='bold', color='#333', + bbox=dict(boxstyle='round,pad=0.5', facecolor='#F0F0F0', alpha=0.8)) + + ax.set_title("LoRA: Low-Rank Adaptation", fontsize=14, fontweight='bold') + st.pyplot(fig) + plt.close(fig) + +# ═══════════════════════════════════════════════════════════════ +# GESAMTKONFIGURATION +# ═══════════════════════════════════════════════════════════════ +elif seite == "📋 Gesamtkonfiguration": + st.header("📋 Gesamtkonfiguration") + + preset = st.selectbox( + "Vordefinierte Konfiguration", + ["Qwen2.5-7B", "Llama-3.1-8B", "Small Test (1.5B)"], + ) + + if preset == "Qwen2.5-7B": + config = get_qwen_config() + elif preset == "Llama-3.1-8B": + config = get_llama_config() + else: + config = get_small_test_config() + + tab1, tab2, tab3, tab4, tab5 = st.tabs( + ["Modell", "LoRA", "GRPO", "Reward", "Daten"] + ) + + with tab1: + st.json({ + "model_name": config.model.model_name_or_path, + "load_in_4bit": config.model.load_in_4bit, + "attn_implementation": config.model.attn_implementation, + }) + + with tab2: + st.json({ + "r": config.lora.r, + "lora_alpha": config.lora.lora_alpha, + "target_modules": config.lora.target_modules, + "lora_dropout": config.lora.lora_dropout, + }) + + with tab3: + st.json({ + "num_generations": config.grpo.num_generations, + "learning_rate": config.grpo.learning_rate, + "beta": config.grpo.beta, + "max_prompt_length": config.grpo.max_prompt_length, + "max_completion_length": config.grpo.max_completion_length, + "temperature": config.grpo.temperature, + "per_device_train_batch_size": config.grpo.per_device_train_batch_size, + "gradient_accumulation_steps": config.grpo.gradient_accumulation_steps, + }) + + with tab4: + st.json({ + "reward_model": config.reward.reward_model_name_or_path, + "reward_weights": config.reward.reward_weights, + "correctness_threshold": config.reward.correctness_threshold, + "safety_threshold": config.reward.safety_threshold, + }) + + with tab5: + st.json({ + "train_file": config.data.train_file, + "eval_file": config.data.eval_file, + "dataset_format": config.data.dataset_format, + }) + + st.markdown("---") + st.markdown(f"**Experiment:** `{config.experiment_name}`") + st.markdown(f"**Output:** `{config.grpo.output_dir}`") + +st.sidebar.markdown("---") +st.sidebar.markdown("📁 **Repo:** [ART](https://github.com/mark-baumann/ART)") +st.sidebar.markdown("🐍 **Python 3.13** · **Streamlit** · **GRPO + LoRA**") From f25a77a5203f2af1684a322b30bd0de7e549bc7b Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Wed, 29 Jul 2026 23:19:27 +0000 Subject: [PATCH 06/20] Streamlit-App: GRPO-Training konfigurieren, Reward-Modell testen, LoRA-Parameter einstellen --- app.py | 832 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 414 insertions(+), 418 deletions(-) diff --git a/app.py b/app.py index ad2ddb9dd..77c6a4639 100644 --- a/app.py +++ b/app.py @@ -1,477 +1,473 @@ """ Streamlit-App: ART — Agent Reinforcement Trainer -================================================ +================================================= GRPO-Training konfigurieren, Reward-Modell testen, LoRA-Parameter einstellen. """ import streamlit as st -import matplotlib.pyplot as plt import numpy as np -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) -from config import ( - TrainingConfig, ModelConfig, LoRAConfig, GRPOConfig, RewardConfig, DataConfig, - get_qwen_config, get_llama_config, get_small_test_config, -) -from reward_model import AgentRewardModel +import re +import os +# ── Page Config ────────────────────────────────────────────── st.set_page_config( page_title="ART — Agent Reinforcement Trainer", - page_icon="🤖", + page_icon="🎯", layout="wide", ) -st.title("🤖 ART — Agent Reinforcement Trainer") -st.markdown("### GRPO-Training konfigurieren · Reward-Modell testen · LoRA-Parameter einstellen") +st.title("🎯 ART — Agent Reinforcement Trainer") +st.markdown("GRPO-Training · Reward-Modell · LoRA-Parameter") -# ── Sidebar ── -seite = st.sidebar.radio( - "📂 Bereich wählen", - ["🎯 GRPO-Training", "🏆 Reward-Modell", "🔧 LoRA-Konfiguration", "📋 Gesamtkonfiguration"], +# ── Sidebar: Modus ─────────────────────────────────────────── +mode = st.sidebar.selectbox( + "Modus wählen", + ["GRPO-Training konfigurieren", "Reward-Modell testen", "LoRA-Parameter einstellen"], ) # ═══════════════════════════════════════════════════════════════ -# GRPO-TRAINING +# 1. GRPO-Training konfigurieren # ═══════════════════════════════════════════════════════════════ -if seite == "🎯 GRPO-Training": - st.header("🎯 GRPO-Training konfigurieren") + +if mode == "GRPO-Training konfigurieren": + st.header("⚙️ GRPO-Training konfigurieren") st.markdown(""" - **GRPO** (Group Relative Policy Optimization) ist ein RL-Verfahren für - Sprachmodelle. Es generiert mehrere Antworten pro Prompt, bewertet sie - mit einem Reward-Modell und optimiert die Policy relativ zur Gruppe. + **Group Relative Policy Optimization (GRPO)** — Konfiguriere das Training + für Qwen2.5 oder Llama 3.1 mit LoRA-Adaptern. """) + # Modell-Auswahl + st.subheader("🤖 Modell") col1, col2 = st.columns(2) - with col1: - st.subheader("📊 GRPO-Parameter") + model_family = st.selectbox("Modellfamilie", ["Qwen 2.5 7B", "Llama 3.1 8B", "Qwen 2.5 1.5B (Test)"], index=0) + with col2: + load_4bit = st.checkbox("4-Bit Quantisierung", value=True) + attn_impl = st.selectbox("Attention", ["flash_attention_2", "sdpa", "eager"], index=0) - num_generations = st.slider("Anzahl Generierungen (G)", 1, 8, 4, - help="Gruppengröße: Wie viele Antworten pro Prompt generiert werden.") - max_prompt_length = st.slider("Max. Prompt-Länge", 256, 4096, 2048, 128) - max_completion_length = st.slider("Max. Completion-Länge", 128, 2048, 1024, 128) + # GRPO-Parameter + st.subheader("🎯 GRPO-Parameter") + col1, col2, col3 = st.columns(3) + with col1: + num_generations = st.slider("Generations (G)", 1, 8, 4, help="Anzahl Samples pro Prompt") + max_prompt_len = st.number_input("Max Prompt-Länge", 256, 4096, 2048, 256) + with col2: + max_completion_len = st.number_input("Max Completion-Länge", 128, 2048, 1024, 128) temperature = st.slider("Temperatur", 0.1, 2.0, 0.9, 0.1) - top_p = st.slider("Top-p", 0.5, 1.0, 1.0, 0.05) + with col3: + beta = st.slider("Beta (KL-Koeffizient)", 0.0, 0.2, 0.04, 0.01, help="KL-Divergence Gewicht") + top_p = st.slider("Top-P", 0.5, 1.0, 1.0, 0.05) + # Training-Parameter + st.subheader("🏋️ Training") + col1, col2, col3 = st.columns(3) + with col1: + learning_rate = st.selectbox("Learning Rate", [1e-6, 3e-6, 5e-6, 1e-5, 5e-5], index=2, format_func=lambda x: f"{x:.0e}") + num_epochs = st.slider("Epochen", 1, 10, 1) with col2: - st.subheader("⚡ Training") - - learning_rate = st.number_input("Learning Rate", 1e-7, 1e-3, 5e-6, format="%.1e") - beta = st.slider("Beta (KL-Koeffizient)", 0.0, 0.2, 0.04, 0.01, - help="Steuert, wie stark die Policy vom Reference-Modell abweichen darf.") - num_epochs = st.slider("Epochen pro GRPO-Schritt", 1, 5, 1) - grad_accum = st.slider("Gradient Accumulation Steps", 1, 16, 4) - batch_size = st.slider("Batch Size pro Device", 1, 8, 2) - - # GRPO-Workflow visualisieren - st.markdown("---") - st.subheader("🔄 GRPO-Workflow") - - fig, ax = plt.subplots(figsize=(12, 5)) - ax.set_xlim(0, 12) - ax.set_ylim(0, 6) - ax.axis('off') - - steps = [ - (1, 4.5, "1. Prompt", '#4ECDC4'), - (3, 4.5, "2. Generiere\nG Antworten", '#45B7D1'), - (5, 4.5, "3. Reward\nberechnen", '#FFE66D'), - (7, 4.5, "4. Advantage\n(Gruppen-Norm.)", '#F38181'), - (9, 4.5, "5. Policy\nUpdate (PPO)", '#FF6B6B'), - (11, 4.5, "6. KL-Div\nprüfen", '#AA96DA'), - ] - - for x, y, text, color in steps: - rect = plt.Rectangle((x - 0.8, y - 0.6), 1.6, 1.2, - facecolor=color, edgecolor='white', - linewidth=2, alpha=0.9, zorder=2) - ax.add_patch(rect) - ax.text(x, y, text, ha='center', va='center', fontsize=9, - fontweight='bold', color='white', zorder=3) - - # Pfeile - for i in range(len(steps) - 1): - ax.annotate('', xy=(steps[i + 1][0] - 0.8, steps[i + 1][1]), - xytext=(steps[i][0] + 0.8, steps[i][1]), - arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) - - # Rückkopplung - ax.annotate('', xy=(1, 2.5), xytext=(11, 2.5), - arrowprops=dict(arrowstyle='->', color='#FF6B6B', lw=1.5, - connectionstyle='arc3,rad=-0.3')) - ax.text(6, 2.0, "↻ Iteriere bis Konvergenz", ha='center', fontsize=10, - color='#FF6B6B', style='italic') - - ax.set_title("GRPO Training Loop", fontsize=14, fontweight='bold') - st.pyplot(fig) - plt.close(fig) - - # Parameter-Impact - st.markdown("---") - st.subheader("📈 Parameter-Einfluss") - - # Beta vs KL-Divergence - betas = np.linspace(0.01, 0.2, 20) - kl_penalty = betas * 10 # Simulierter Effekt - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4)) - - ax1.plot(betas, kl_penalty, color='#FF6B6B', linewidth=2) - ax1.fill_between(betas, 0, kl_penalty, alpha=0.2, color='#FF6B6B') - ax1.set_xlabel("Beta (KL-Koeffizient)") - ax1.set_ylabel("KL-Penalty") - ax1.set_title("Beta → KL-Divergence-Kontrolle") - ax1.grid(True, alpha=0.3) - - # Gruppengröße vs Variance - groups = np.arange(1, 9) - advantage_variance = 1.0 / np.sqrt(groups) - ax2.bar(groups, advantage_variance, color='#4ECDC4', edgecolor='white') - ax2.set_xlabel("Gruppengröße (G)") - ax2.set_ylabel("Advantage-Varianz (relativ)") - ax2.set_title("Gruppengröße → Schätzgenauigkeit") - ax2.grid(True, alpha=0.3, axis='y') + batch_size = st.slider("Batch Size (pro Device)", 1, 8, 2) + grad_accum = st.slider("Gradient Accumulation", 1, 16, 4) + with col3: + optim = st.selectbox("Optimizer", ["adamw_8bit", "adamw_torch", "sgd"], index=0) + lr_scheduler = st.selectbox("LR Scheduler", ["cosine", "linear", "constant"], index=0) - st.pyplot(fig) - plt.close(fig) + col1, col2 = st.columns(2) + with col1: + warmup_ratio = st.slider("Warmup Ratio", 0.0, 0.3, 0.1, 0.05) + weight_decay = st.slider("Weight Decay", 0.0, 0.2, 0.01, 0.01) + with col2: + max_steps = st.number_input("Max Steps (-1 = voller Datensatz)", -1, 10000, -1, 100) + seed = st.number_input("Seed", 0, 9999, 42) + + # Logging + st.subheader("📊 Logging & Output") + col1, col2, col3 = st.columns(3) + with col1: + logging_steps = st.number_input("Logging Steps", 1, 500, 10) + save_steps = st.number_input("Save Steps", 10, 1000, 100) + with col2: + eval_steps = st.number_input("Eval Steps", 10, 1000, 100) + report_to = st.selectbox("Report To", ["wandb", "tensorboard", "none"], index=0) + with col3: + output_dir = st.text_input("Output Dir", "./output/grpo-lora") + experiment_name = st.text_input("Experiment", "art-grpo-lora") + + # Zusammenfassung + st.divider() + st.subheader("📋 Vollständige Konfiguration") + + config_summary = { + "Modell": { + "Familie": model_family, + "4-Bit": load_4bit, + "Attention": attn_impl, + }, + "GRPO": { + "num_generations": num_generations, + "max_prompt_length": max_prompt_len, + "max_completion_length": max_completion_len, + "temperature": temperature, + "beta": beta, + "top_p": top_p, + }, + "Training": { + "learning_rate": learning_rate, + "num_epochs": num_epochs, + "batch_size": batch_size, + "gradient_accumulation_steps": grad_accum, + "optim": optim, + "lr_scheduler": lr_scheduler, + "warmup_ratio": warmup_ratio, + "weight_decay": weight_decay, + "max_steps": max_steps, + "seed": seed, + }, + "Logging": { + "logging_steps": logging_steps, + "save_steps": save_steps, + "eval_steps": eval_steps, + "report_to": report_to, + "output_dir": output_dir, + "experiment_name": experiment_name, + }, + } + + st.json(config_summary) + + # CLI-Befehl generieren + st.subheader("💻 CLI-Befehl") + cmd_parts = ["python train_agent.py"] + if "Qwen" in model_family: + cmd_parts.append("--model qwen") + else: + cmd_parts.append("--model llama") + if "1.5B" in model_family: + cmd_parts.append("--test-mode") + if output_dir != "./output/grpo-lora": + cmd_parts.append(f"--output-dir {output_dir}") + if learning_rate != 5e-6: + cmd_parts.append(f"--learning-rate {learning_rate}") + if max_steps > 0: + cmd_parts.append(f"--max-steps {max_steps}") + if report_to == "none": + cmd_parts.append("--no-wandb") + + st.code(" \\\n ".join(cmd_parts), language="bash") + + if st.button("💾 Konfiguration speichern", type="primary"): + st.success(f"✅ Konfiguration '{experiment_name}' bereit zum Training!") # ═══════════════════════════════════════════════════════════════ -# REWARD-MODELL +# 2. Reward-Modell testen # ═══════════════════════════════════════════════════════════════ -elif seite == "🏆 Reward-Modell": + +elif mode == "Reward-Modell testen": st.header("🏆 Reward-Modell testen") st.markdown(""" - Das Reward-Modell bewertet Agent-Antworten anhand mehrerer Kriterien. - Hier kannst du es mit eigenen Beispielen testen. + Teste das regelbasierte Reward-Modell mit eigenen Prompts und Completions. + Bewertet werden: **Korrektheit, Format, Hilfreichkeit, Sicherheit, Tool-Nutzung**. """) - col1, col2 = st.columns([1, 1]) - + # Reward-Gewichte + st.subheader("⚖️ Reward-Gewichte") + col1, col2, col3, col4, col5 = st.columns(5) with col1: - st.subheader("📝 Eingabe") - - prompt = st.text_area( - "Prompt", - value="Erkläre, wie ein Transformer-Modell funktioniert.", - height=80, - ) - - completion = st.text_area( - "Agent-Antwort", - value=( - "Ein Transformer-Modell basiert auf dem Attention-Mechanismus. " - "Es verarbeitet Eingaben parallel statt sequentiell. " - "Die Self-Attention berechnet für jedes Token die Relevanz " - "aller anderen Tokens im Kontext. Das ermöglicht es dem Modell, " - "langreichweitige Abhängigkeiten zu erfassen." - ), - height=150, - ) - - ground_truth = st.text_input( - "Ground Truth (optional)", - value="Transformer nutzen Self-Attention zur parallelen Verarbeitung von Sequenzen.", - ) - - tools_expected = st.multiselect( - "Erwartete Tools", - ["search", "calculator", "code_interpreter", "web_browser"], - [], - ) - - if st.button("🏆 Reward berechnen", type="primary", use_container_width=True): - st.session_state.reward_clicked = True - else: - if "reward_clicked" not in st.session_state: - st.session_state.reward_clicked = False - + w_correctness = st.slider("Korrektheit", 0.0, 2.0, 1.0, 0.1) with col2: - st.subheader("📊 Reward-Ergebnis") - - if st.session_state.reward_clicked: - with st.spinner("Berechne Reward..."): - reward_model = AgentRewardModel(use_model=False) - result = reward_model.compute_reward( - prompt=prompt, - completion=completion, - ground_truth=ground_truth if ground_truth else None, - tools_expected=tools_expected if tools_expected else None, - ) - - # Gesamt-Reward - total = result["total"] - color = "green" if total > 1.5 else "orange" if total > 0.8 else "red" - st.markdown(f"### Gesamt-Reward: {total:.3f}", - unsafe_allow_html=True) - - # Einzel-Rewards - reward_items = {k: v for k, v in result.items() if k != "total"} - fig, ax = plt.subplots(figsize=(6, 4)) - names = list(reward_items.keys()) - values = list(reward_items.values()) - colors = ['#4ECDC4', '#45B7D1', '#FFE66D', '#FF6B6B', '#F38181'][:len(names)] - bars = ax.barh(names, values, color=colors, edgecolor='white') - ax.set_xlim(0, 1.0) - ax.set_xlabel("Score") - ax.set_title("Reward-Komponenten") - for bar, val in zip(bars, values): - ax.text(bar.get_width() + 0.02, bar.get_y() + bar.get_height() / 2, - f"{val:.2f}", va='center', fontsize=10) - st.pyplot(fig) - plt.close(fig) - - # Details - st.markdown("**Details:**") - for name, val in reward_items.items(): - emoji = "✅" if val > 0.7 else "⚠️" if val > 0.3 else "❌" - st.markdown(f"{emoji} **{name}**: {val:.3f}") - else: - st.info("👈 Gib links Prompt und Antwort ein und klicke auf **Reward berechnen**.") - - # Reward-Gewichte - st.markdown("---") - st.subheader("⚖️ Reward-Gewichte konfigurieren") - - col_a, col_b = st.columns(2) - - with col_a: - w_correctness = st.slider("Correctness", 0.0, 2.0, 1.0, 0.1) w_format = st.slider("Format", 0.0, 2.0, 0.3, 0.1) - w_helpfulness = st.slider("Helpfulness", 0.0, 2.0, 0.5, 0.1) - - with col_b: - w_safety = st.slider("Safety", 0.0, 2.0, 0.8, 0.1) - w_tool_usage = st.slider("Tool Usage", 0.0, 2.0, 0.4, 0.1) - - # Gewichte visualisieren - weights = { - "Correctness": w_correctness, - "Format": w_format, - "Helpfulness": w_helpfulness, - "Safety": w_safety, - "Tool Usage": w_tool_usage, - } - - fig, ax = plt.subplots(figsize=(6, 3)) - names = list(weights.keys()) - values = list(weights.values()) - colors = ['#4ECDC4', '#45B7D1', '#FFE66D', '#FF6B6B', '#F38181'] - ax.bar(names, values, color=colors, edgecolor='white') - ax.set_ylabel("Gewicht") - ax.set_title("Reward-Gewichte") - ax.axhline(y=1.0, color='gray', linestyle='--', alpha=0.5) - st.pyplot(fig) - plt.close(fig) + with col3: + w_helpfulness = st.slider("Hilfreichkeit", 0.0, 2.0, 0.5, 0.1) + with col4: + w_safety = st.slider("Sicherheit", 0.0, 2.0, 0.8, 0.1) + with col5: + w_tool_usage = st.slider("Tool-Nutzung", 0.0, 2.0, 0.4, 0.1) + + # Eingabe + st.subheader("📝 Test-Eingabe") + prompt_input = st.text_area("Prompt", value="Erkläre den Unterschied zwischen GRPO und PPO.", height=80) + completion_input = st.text_area("Completion (Agent-Antwort)", value="GRPO (Group Relative Policy Optimization) ist eine Weiterentwicklung von PPO (Proximal Policy Optimization). Der Hauptunterschied: GRPO vergleicht mehrere generierte Antworten innerhalb einer Gruppe relativ zueinander, während PPO mit einem einzelnen Value-Netzwerk arbeitet. GRPO benötigt daher kein separates Critic-Modell und ist speichereffizienter.", height=120) + ground_truth = st.text_input("Ground Truth (optional)", value="", placeholder="Erwartete Antwort für Korrektheits-Check") + + if st.button("🔍 Reward berechnen", type="primary"): + # ── Regelbasierte Reward-Berechnung ────────────────── + rewards = {} + + # 1. Korrektheit + if ground_truth.strip(): + if completion_input.strip().lower() == ground_truth.strip().lower(): + rewards["correctness"] = 1.0 + elif ground_truth.strip().lower() in completion_input.strip().lower(): + rewards["correctness"] = 0.7 + else: + gt_words = set(ground_truth.lower().split()) + comp_words = set(completion_input.lower().split()) + if gt_words: + overlap = len(gt_words & comp_words) / len(gt_words) + rewards["correctness"] = min(overlap, 0.5) + else: + rewards["correctness"] = 0.0 + else: + rewards["correctness"] = 0.5 + + # 2. Format + format_score = 0.0 + if re.search(r"```(?:json|python|yaml)?\s*\n", completion_input): + format_score += 0.3 + if re.search(r"^#{1,3}\s", completion_input, re.MULTILINE): + format_score += 0.2 + if re.search(r"^\s*[-*]\s", completion_input, re.MULTILINE): + format_score += 0.2 + length = len(completion_input.split()) + if 20 <= length <= 500: + format_score += 0.3 + elif 10 <= length <= 1000: + format_score += 0.15 + rewards["format"] = min(format_score, 1.0) + + # 3. Hilfreichkeit + helpful_score = 0.0 + explanation_patterns = [ + r"(?:weil|da|denn|deshalb|daher|somit)\b", + r"\b(?:first|second|finally|therefore|because|thus)\b", + r"^(?:Schritt|Step)\s+\d", + ] + for pat in explanation_patterns: + if re.search(pat, completion_input, re.IGNORECASE): + helpful_score += 0.2 + break + if re.search(r"(?:z\.B\.|e\.g\.|for example|Beispiel)", completion_input, re.IGNORECASE): + helpful_score += 0.2 + word_count = len(completion_input.split()) + if word_count >= 30: + helpful_score += 0.3 + elif word_count >= 15: + helpful_score += 0.15 + if completion_input.strip(): + helpful_score += 0.3 + rewards["helpfulness"] = min(helpful_score, 1.0) + + # 4. Sicherheit + safety_score = 1.0 + unsafe_patterns = [ + r"\b(?:hack|exploit|bypass|inject)\b", + r"\b(?:password|token|secret|api[_\s]?key)\s*[:=]\s*\S+", + r"\b(?:rm\s+-rf|DROP\s+TABLE|DELETE\s+FROM)\b", + r"\b(?:illegal|malware|ransomware|phishing)\b", + ] + for pat in unsafe_patterns: + if re.search(pat, completion_input, re.IGNORECASE): + safety_score -= 0.3 + refusal_patterns = [ + r"\b(?:cannot|can't|unable to|not able to|won't)\b", + r"\b(?:entschuldigung|tut mir leid|kann (?:ich )?nicht)\b", + ] + for pat in refusal_patterns: + if re.search(pat, completion_input, re.IGNORECASE): + safety_score = max(safety_score, 0.8) + break + rewards["safety"] = max(safety_score, 0.0) + + # 5. Tool-Nutzung + rewards["tool_usage"] = 0.5 # Neutral + + # Gewichteter Total-Reward + total = ( + rewards["correctness"] * w_correctness + + rewards["format"] * w_format + + rewards["helpfulness"] * w_helpfulness + + rewards["safety"] * w_safety + + rewards["tool_usage"] * w_tool_usage + ) + rewards["total"] = total + + # ── Ergebnisse anzeigen ────────────────────────────── + st.divider() + st.subheader("📊 Reward-Ergebnisse") + + col1, col2, col3, col4, col5, col6 = st.columns(6) + with col1: + st.metric("Korrektheit", f"{rewards['correctness']:.2f}", delta=None) + with col2: + st.metric("Format", f"{rewards['format']:.2f}", delta=None) + with col3: + st.metric("Hilfreichkeit", f"{rewards['helpfulness']:.2f}", delta=None) + with col4: + st.metric("Sicherheit", f"{rewards['safety']:.2f}", delta=None) + with col5: + st.metric("Tool-Nutzung", f"{rewards['tool_usage']:.2f}", delta=None) + with col6: + st.metric("**Total**", f"{rewards['total']:.2f}", delta=None) + + # Balkendiagramm + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(8, 3)) + categories = ["Korrektheit", "Format", "Hilfreichkeit", "Sicherheit", "Tool-Nutzung"] + values = [rewards["correctness"], rewards["format"], rewards["helpfulness"], rewards["safety"], rewards["tool_usage"]] + colors = ["#4CAF50", "#2196F3", "#FF9800", "#f44336", "#9C27B0"] + bars = ax.bar(categories, values, color=colors) + ax.axhline(y=rewards["total"], color="black", linestyle="--", label=f"Total: {rewards['total']:.2f}") + ax.set_ylim(0, 1.1) + ax.set_ylabel("Score") + ax.legend() + for bar, val in zip(bars, values): + ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.02, f"{val:.2f}", ha="center", fontsize=10) + st.pyplot(fig) + + # Bewertung + if total >= 0.7: + st.success(f"🌟 Gute Antwort! Total Reward: {total:.2f}") + elif total >= 0.4: + st.warning(f"⚡ Durchschnittliche Antwort. Total Reward: {total:.2f}") + else: + st.error(f"❌ Schwache Antwort. Total Reward: {total:.2f}") # ═══════════════════════════════════════════════════════════════ -# LORA-KONFIGURATION +# 3. LoRA-Parameter einstellen # ═══════════════════════════════════════════════════════════════ -elif seite == "🔧 LoRA-Konfiguration": + +elif mode == "LoRA-Parameter einstellen": st.header("🔧 LoRA-Parameter einstellen") st.markdown(""" - **LoRA** (Low-Rank Adaptation) fügt trainierbare Low-Rank-Matrizen zu - eingefrorenen Gewichten hinzu. Das reduziert die trainierbaren Parameter - drastisch (typisch <1% des Originalmodells). + **Low-Rank Adaptation (LoRA)** — Konfiguriere die LoRA-Adapter für effizientes Fine-Tuning. + Nur ein Bruchteil der Parameter wird trainiert. """) - col1, col2 = st.columns(2) - + # LoRA-Kernparameter + st.subheader("📐 LoRA-Kernparameter") + col1, col2, col3 = st.columns(3) with col1: - st.subheader("📐 LoRA-Dimensionen") + lora_r = st.slider("Rank (r)", 1, 128, 16, help="LoRA-Rank — niedriger = weniger Parameter") + with col2: + lora_alpha = st.slider("Alpha", 1, 128, 32, help="Skalierungsfaktor") + with col3: + lora_dropout = st.slider("Dropout", 0.0, 0.5, 0.05, 0.05) - lora_r = st.slider("Rank (r)", 1, 128, 16, - help="Rank der Low-Rank-Approximation. Höher = mehr Kapazität, mehr Parameter.") - lora_alpha = st.slider("Alpha", 1, 128, 32, - help="Skalierungsfaktor. Effektive LR skaliert mit alpha/r.") - lora_dropout = st.slider("Dropout", 0.0, 0.5, 0.05, 0.01) + # Effektive Skalierung + effective_scale = lora_alpha / lora_r + st.info(f"📏 Effektive Skalierung: α/r = {lora_alpha}/{lora_r} = **{effective_scale:.2f}**") - st.markdown(f"**Effektiver Skalierungsfaktor:** α/r = {lora_alpha / lora_r:.2f}") + # Target-Module + st.subheader("🎯 Target-Module") + st.markdown("Wähle aus, welche Layer mit LoRA-Adaptern versehen werden:") + col1, col2 = st.columns(2) + with col1: + target_q = st.checkbox("q_proj (Query)", value=True) + target_k = st.checkbox("k_proj (Key)", value=True) + target_v = st.checkbox("v_proj (Value)", value=True) + target_o = st.checkbox("o_proj (Output)", value=True) with col2: - st.subheader("🎯 Zielmodule") - - target_modules = st.multiselect( - "Module für LoRA-Adapter", - ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], - ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"], - ) + target_gate = st.checkbox("gate_proj", value=True) + target_up = st.checkbox("up_proj", value=True) + target_down = st.checkbox("down_proj", value=True) + + selected_targets = [] + if target_q: selected_targets.append("q_proj") + if target_k: selected_targets.append("k_proj") + if target_v: selected_targets.append("v_proj") + if target_o: selected_targets.append("o_proj") + if target_gate: selected_targets.append("gate_proj") + if target_up: selected_targets.append("up_proj") + if target_down: selected_targets.append("down_proj") + + st.write(f"**{len(selected_targets)} Module** ausgewählt: `{', '.join(selected_targets) if selected_targets else 'keine'}`") + + # Parameter-Schätzung + st.subheader("📊 Parameter-Schätzung") + + model_size = st.selectbox("Basis-Modell", ["Qwen 2.5 7B", "Llama 3.1 8B", "Qwen 2.5 1.5B"], index=0) + + # Grobe Schätzung + if "7B" in model_size: + base_params = 7_000_000_000 + hidden_size = 4096 + num_layers = 32 + elif "8B" in model_size: + base_params = 8_000_000_000 + hidden_size = 4096 + num_layers = 32 + else: + base_params = 1_500_000_000 + hidden_size = 1536 + num_layers = 28 - bias_type = st.selectbox("Bias-Typ", ["none", "all", "lora_only"]) - - # LoRA-Parameter berechnen - st.markdown("---") - st.subheader("📊 LoRA-Parameter-Analyse") - - # Beispiel: Qwen2.5-7B - base_params = 7_000_000_000 # 7B - hidden_size = 4096 # typisch für 7B-Modelle - - lora_params_per_module = 2 * hidden_size * lora_r # A und B Matrix - total_lora_params = lora_params_per_module * len(target_modules) - lora_ratio = total_lora_params / base_params * 100 - - col_a, col_b, col_c = st.columns(3) - with col_a: - st.metric("Basis-Parameter", f"{base_params / 1e9:.1f}B") - with col_b: - st.metric("LoRA-Parameter", f"{total_lora_params / 1e6:.2f}M") - with col_c: - st.metric("LoRA-Anteil", f"{lora_ratio:.4f}%") - - # Rank vs Parameter - ranks = [1, 2, 4, 8, 16, 32, 64, 128] - params_for_rank = [2 * hidden_size * r * len(target_modules) / 1e6 for r in ranks] - - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4)) - - ax1.plot(ranks, params_for_rank, 'o-', color='#4ECDC4', linewidth=2, markersize=8) - ax1.set_xlabel("LoRA Rank (r)") - ax1.set_ylabel("Trainierbare Parameter (M)") - ax1.set_title("Rank → Parameter") - ax1.grid(True, alpha=0.3) - ax1.axvline(x=lora_r, color='#FF6B6B', linestyle='--', alpha=0.5, - label=f'Aktuell: r={lora_r}') - ax1.legend() - - # Modul-Beitrag - modules = target_modules if target_modules else ["q_proj", "k_proj", "v_proj", "o_proj"] - mod_params = [2 * hidden_size * lora_r / 1e6] * len(modules) - ax2.pie(mod_params, labels=modules, autopct='%1.1f%%', - colors=plt.cm.Set3(np.linspace(0, 1, len(modules)))) - ax2.set_title(f"Parameter-Verteilung (r={lora_r})") + # LoRA-Parameter: 2 * r * hidden_size * num_target_modules * num_layers + num_targets = len(selected_targets) + lora_params = 2 * lora_r * hidden_size * num_targets * num_layers + col1, col2, col3 = st.columns(3) + with col1: + st.metric("Basis-Parameter", f"{base_params/1e9:.1f}B") + with col2: + st.metric("LoRA-Parameter", f"{lora_params/1e6:.1f}M") + with col3: + ratio = lora_params / base_params * 100 + st.metric("Anteil", f"{ratio:.2f}%") + + # Visualisierung + import matplotlib.pyplot as plt + fig, ax = plt.subplots(figsize=(6, 4)) + sizes = [base_params - lora_params, lora_params] + labels = ["Eingefroren", "LoRA (trainierbar)"] + colors = ["#BBDEFB", "#1565C0"] + ax.pie(sizes, labels=labels, autopct="%1.2f%%", colors=colors, startangle=90) + ax.set_title(f"Parameter-Verteilung: {model_size}") st.pyplot(fig) - plt.close(fig) - - # LoRA-Architektur-Diagramm - st.markdown("---") - st.subheader("🎨 LoRA-Architektur") - - fig, ax = plt.subplots(figsize=(10, 5)) - ax.set_xlim(0, 10) - ax.set_ylim(0, 6) - ax.axis('off') - - # Original Weight - rect = plt.Rectangle((1, 2.5), 3, 2, facecolor='#E8E8E8', edgecolor='#888888', - linewidth=2, alpha=0.8, zorder=1) - ax.add_patch(rect) - ax.text(2.5, 3.5, "Eingefrorene\nGewichte W\n(d × k)", ha='center', va='center', - fontsize=11, fontweight='bold', color='#555') - - # LoRA A - rect_a = plt.Rectangle((5, 3.5), 1.5, 1, facecolor='#4ECDC4', edgecolor='white', - linewidth=2, alpha=0.9, zorder=2) - ax.add_patch(rect_a) - ax.text(5.75, 4.0, "A\n(d × r)", ha='center', va='center', - fontsize=10, fontweight='bold', color='white') - - # LoRA B - rect_b = plt.Rectangle((7, 3.5), 1.5, 1, facecolor='#FF6B6B', edgecolor='white', - linewidth=2, alpha=0.9, zorder=2) - ax.add_patch(rect_b) - ax.text(7.75, 4.0, "B\n(r × k)", ha='center', va='center', - fontsize=10, fontweight='bold', color='white') - - # Output - ax.text(9.5, 4.0, "ΔW = α/r · BA", ha='center', va='center', - fontsize=10, fontweight='bold', color='#333') - - # Pfeile - ax.annotate('', xy=(5, 4.0), xytext=(4, 4.0), - arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) - ax.annotate('', xy=(7, 4.0), xytext=(6.5, 4.0), - arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) - ax.annotate('', xy=(9, 4.0), xytext=(8.5, 4.0), - arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) - - # Input - ax.text(0.5, 4.0, "Input x", ha='center', fontsize=10, fontweight='bold') - ax.annotate('', xy=(1, 4.0), xytext=(0.8, 4.0), - arrowprops=dict(arrowstyle='->', color='#888888', lw=2)) - - # Formel - ax.text(5, 1.5, "h = Wx + (α/r) · BAx", ha='center', fontsize=14, - fontweight='bold', color='#333', - bbox=dict(boxstyle='round,pad=0.5', facecolor='#F0F0F0', alpha=0.8)) - - ax.set_title("LoRA: Low-Rank Adaptation", fontsize=14, fontweight='bold') - st.pyplot(fig) - plt.close(fig) -# ═══════════════════════════════════════════════════════════════ -# GESAMTKONFIGURATION -# ═══════════════════════════════════════════════════════════════ -elif seite == "📋 Gesamtkonfiguration": - st.header("📋 Gesamtkonfiguration") - - preset = st.selectbox( - "Vordefinierte Konfiguration", - ["Qwen2.5-7B", "Llama-3.1-8B", "Small Test (1.5B)"], - ) - - if preset == "Qwen2.5-7B": - config = get_qwen_config() - elif preset == "Llama-3.1-8B": - config = get_llama_config() - else: - config = get_small_test_config() - - tab1, tab2, tab3, tab4, tab5 = st.tabs( - ["Modell", "LoRA", "GRPO", "Reward", "Daten"] - ) - - with tab1: - st.json({ - "model_name": config.model.model_name_or_path, - "load_in_4bit": config.model.load_in_4bit, - "attn_implementation": config.model.attn_implementation, - }) - - with tab2: - st.json({ - "r": config.lora.r, - "lora_alpha": config.lora.lora_alpha, - "target_modules": config.lora.target_modules, - "lora_dropout": config.lora.lora_dropout, - }) - - with tab3: - st.json({ - "num_generations": config.grpo.num_generations, - "learning_rate": config.grpo.learning_rate, - "beta": config.grpo.beta, - "max_prompt_length": config.grpo.max_prompt_length, - "max_completion_length": config.grpo.max_completion_length, - "temperature": config.grpo.temperature, - "per_device_train_batch_size": config.grpo.per_device_train_batch_size, - "gradient_accumulation_steps": config.grpo.gradient_accumulation_steps, - }) - - with tab4: - st.json({ - "reward_model": config.reward.reward_model_name_or_path, - "reward_weights": config.reward.reward_weights, - "correctness_threshold": config.reward.correctness_threshold, - "safety_threshold": config.reward.safety_threshold, - }) - - with tab5: - st.json({ - "train_file": config.data.train_file, - "eval_file": config.data.eval_file, - "dataset_format": config.data.dataset_format, - }) - - st.markdown("---") - st.markdown(f"**Experiment:** `{config.experiment_name}`") - st.markdown(f"**Output:** `{config.grpo.output_dir}`") + # Vordefinierte Presets + st.subheader("🎛️ Vordefinierte Presets") + col1, col2, col3 = st.columns(3) + with col1: + if st.button("🔬 Konservativ (r=8, α=16)", use_container_width=True): + st.session_state["lora_preset"] = "conservative" + with col2: + if st.button("⚖️ Standard (r=16, α=32)", use_container_width=True): + st.session_state["lora_preset"] = "standard" + with col3: + if st.button("🚀 Aggressiv (r=64, α=128)", use_container_width=True): + st.session_state["lora_preset"] = "aggressive" + + # Konfigurations-Code + st.subheader("💻 LoRA-Konfiguration (Python)") + lora_code = f"""from peft import LoraConfig + +lora_config = LoraConfig( + r={lora_r}, + lora_alpha={lora_alpha}, + target_modules={selected_targets}, + lora_dropout={lora_dropout}, + bias="none", + task_type="CAUSAL_LM", +)""" + st.code(lora_code, language="python") + + # Empfehlungen + st.subheader("💡 Empfehlungen") + st.markdown(f""" + - **Rank (r={lora_r})**: {'Niedrig' if lora_r <= 8 else 'Mittel' if lora_r <= 32 else 'Hoch'} — { + 'Gut für einfache Tasks, sehr speichereffizient' if lora_r <= 8 else + 'Gute Balance für die meisten Anwendungen' if lora_r <= 32 else + 'Für komplexe Tasks, höherer Speicherverbrauch' + } + - **Alpha (α={lora_alpha})**: Skalierung = {effective_scale:.1f}× — { + 'Starke Regularisierung' if effective_scale < 1 else + 'Standard-Skalierung' if effective_scale <= 2 else + 'Schwache Regularisierung, stärkere Anpassung' + } + - **Target-Module**: {num_targets} von 7 möglichen — { + 'Minimale Anpassung' if num_targets <= 3 else + 'Ausgewogen' if num_targets <= 5 else + 'Volle Abdeckung (empfohlen für GRPO)' + } + - **Dropout ({lora_dropout})**: { + 'Keine Regularisierung' if lora_dropout == 0 else + 'Leichte Regularisierung' if lora_dropout <= 0.1 else + 'Starke Regularisierung (gegen Overfitting)' + } + """) st.sidebar.markdown("---") -st.sidebar.markdown("📁 **Repo:** [ART](https://github.com/mark-baumann/ART)") -st.sidebar.markdown("🐍 **Python 3.13** · **Streamlit** · **GRPO + LoRA**") +st.sidebar.caption("ART · Streamlit Dashboard") From c1a56b41a7d8b23a2fc3adc71d7f7c3d50b13629 Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 00:15:13 +0000 Subject: [PATCH 07/20] =?UTF-8?q?Code-Qualit=C3=A4t:=20Ungenutzte=20Import?= =?UTF-8?q?s=20entfernt,=20Import-Sortierung=20mit=20ruff=20korrigiert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py | 5 ++--- train_agent.py | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 77c6a4639..99875c731 100644 --- a/app.py +++ b/app.py @@ -4,10 +4,9 @@ GRPO-Training konfigurieren, Reward-Modell testen, LoRA-Parameter einstellen. """ -import streamlit as st -import numpy as np import re -import os + +import streamlit as st # ── Page Config ────────────────────────────────────────────── st.set_page_config( diff --git a/train_agent.py b/train_agent.py index 1907c1df3..5d8470630 100644 --- a/train_agent.py +++ b/train_agent.py @@ -21,9 +21,9 @@ import sys from typing import Any, Optional -import torch from datasets import Dataset from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training +import torch from transformers import ( AutoModelForCausalLM, AutoTokenizer, @@ -36,6 +36,8 @@ # Lokale Imports from config import ( LoRAConfig as LocalLoRAConfig, +) +from config import ( ModelConfig, TrainingConfig, get_llama_config, From 47a9822f3213459db1be4770afbf184fcde7816b Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 05:43:01 +0000 Subject: [PATCH 08/20] =?UTF-8?q?docs:=20README.md=20hinzugef=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 225 ++++++++++++++++++++++-------------------------------- 1 file changed, 91 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index c96c917a9..fafc6fffd 100644 --- a/README.md +++ b/README.md @@ -1,168 +1,125 @@ -
+# 🎯 ART — Agent Reinforcement Trainer - -ART logo - +[![Python](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) +[![PyTorch](https://img.shields.io/badge/PyTorch-2.0%2B-ee4c2c.svg)](https://pytorch.org/) +[![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) +[![Streamlit](https://img.shields.io/badge/Streamlit-App-red.svg)](https://streamlit.io/) -

-

Agent Reinforcement Trainer

-

+**Agentic Reinforcement Training** — GRPO (Group Relative Policy Optimization) mit LoRA für Qwen2.5 und Llama 3.1. -

-Train multi-step agents for real-world tasks using GRPO. -

+## 📋 Beschreibung -[![PRs-Welcome][contribute-image]][contribute-url] -[![PyPI version](https://img.shields.io/pypi/v/openpipe-art?color=364fc7)][pypi-url] -[![Train Agent](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/2048/2048.ipynb) +ART ist ein spezialisiertes Trainings-Framework, das Group Relative Policy Optimization (GRPO) mit Low-Rank Adaptation (LoRA) kombiniert. Es ermöglicht effizientes Fine-Tuning großer Sprachmodelle (Qwen2.5 7B, Llama 3.1 8B) für agentische Aufgaben — mit 4-Bit-Quantisierung, Reward-Modellierung und einer interaktiven Streamlit-Konfigurationsoberfläche. -[![Join Discord](https://img.shields.io/badge/Join%20Discord-5865F2?style=plastic&logo=discord&logoColor=white)](https://discord.gg/EceeVdhpxD) -[![Documentation](https://img.shields.io/badge/Documentation-orange?style=plastic&logo=gitbook&logoColor=white)](https://art.openpipe.ai) +- **GRPO-Training** — Group Relative Policy Optimization mit konfigurierbaren Generations, Beta und Temperatur +- **LoRA-Adapter** — Effizientes Fine-Tuning mit PEFT/LoRA auf Qwen und Llama +- **Reward-Modell** — Konfigurierbare Reward-Funktionen für agentische Aufgaben +- **4-Bit Quantisierung** — BitsAndBytes für speichereffizientes Training -
+## ✨ Features -## 🚀 W&B Training: Serverless RL +- 🎯 **GRPO + LoRA** — State-of-the-Art RL-Training für Sprachmodelle +- 🤖 **Multi-Modell-Support** — Qwen2.5 (7B/1.5B) und Llama 3.1 (8B) +- ⚡ **4-Bit Training** — BitsAndBytes-Quantisierung für Consumer-GPUs +- 🏆 **Reward-Modell** — Flexible Reward-Funktionen mit konfigurierbaren Gewichten +- 🖥️ **Streamlit-App** — Interaktive Konfiguration von GRPO, LoRA und Reward-Parametern +- 📊 **W&B Tracking** — Vollständiges Experiment-Tracking +- 🧪 **Umfangreiche Tests** — Unit-Tests für alle Kernkomponenten +- 🔧 **vLLM Runtime** — Dedizierte Server-Integration für schnelle Inferenz -**W&B Training (Serverless RL)** is the first publicly available service for flexibly training models with reinforcement learning. It manages your training and inference infrastructure automatically, letting you focus on defining your data, environment and reward function—leading to faster feedback cycles, lower costs, and far less DevOps. +## 🚀 Installation -✨ **Key Benefits:** +```bash +# Repository klonen +git clone https://github.com/mark-baumann/ART.git +cd ART -- **40% lower cost** - Multiplexing on shared production-grade inference cluster -- **28% faster training** - Scale to 2000+ concurrent requests across many GPUs -- **Zero infra headaches** - Fully managed infrastructure that stays healthy -- **Instant deployment** - Every checkpoint instantly available via W&B Inference +# Virtuelle Umgebung erstellen +python3 -m venv .venv +source .venv/bin/activate -```python -# Before: Hours of GPU setup and infra management -# RuntimeError: CUDA error: out of memory 😢 +# Abhängigkeiten installieren +pip install -r requirements.txt -# After: Serverless RL with instant feedback -from art.serverless.backend import ServerlessBackend - -model = art.TrainableModel( - project="voice-agent", - name="agent-001", - base_model="Qwen/Qwen3.6-27B" -) - -backend = ServerlessBackend( - api_key="your_wandb_api_key" -) -model.register(backend) -# Edit and iterate in minutes, not hours! +# Für GPU-Training (CUDA 11.8) +pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 +pip install bitsandbytes accelerate peft trl ``` -[📖 Learn more about W&B Training →](https://docs.wandb.ai/guides/training) - -## ART Overview - -ART is an open-source RL framework that improves agent reliability by allowing LLMs to **learn from experience**. ART provides an ergonomic harness for integrating GRPO into any python application. For a quick hands-on introduction, run one of the notebooks below. When you're ready to learn more, check out the [docs](https://art.openpipe.ai). - -## 📒 Notebooks - -| Agent Task | Example Notebook | Description | Comparative Performance | -| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **ART•E [Serverless]** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/art-e.ipynb) | Qwen 3.6 27B learns to search emails using RULER | [benchmarks](/dev/art-e/art_e/evaluate/display_benchmarks.ipynb) | -| **2048 [Serverless]** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/2048/2048.ipynb) | Qwen 3.6 27B learns to play 2048 | [benchmarks](/examples/2048/display_benchmarks.ipynb) | -| **ART•E LangGraph** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/langgraph/art-e-langgraph.ipynb) | Qwen 2.5 7B learns to search emails using LangGraph | [Link coming soon] | -| **MCP•RL** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/mcp-rl/mcp-rl.ipynb) | Qwen 2.5 3B masters the NWS MCP server | [Link coming soon] | -| **Temporal Clue** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/temporal_clue/temporal-clue.ipynb) | Qwen 2.5 7B learns to solve Temporal Clue | [Link coming soon] | -| **Tic Tac Toe** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/tic_tac_toe/tic-tac-toe.ipynb) | Qwen 2.5 3B learns to play Tic Tac Toe | [benchmarks](/examples/tic_tac_toe/display-benchmarks.ipynb) | -| **Codenames** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/codenames/Codenames_RL.ipynb) | Qwen 2.5 3B learns to play Codenames | [benchmarks](https://github.com/OpenPipe/art-notebooks/blob/main/examples/codenames/Codenames_RL.ipynb) | -| **AutoRL [RULER]** | [🏋️ Train agent](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/auto_rl.ipynb) | Train Qwen 2.5 7B to master any task | [Link coming soon] | -| **Distillation (SFT)** | [🏋️ Train model](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/sft/distillation.ipynb) | Distill text-to-SQL from Qwen 3 235B to Qwen 3.6 27B | [Link coming soon] | -| **Summarizer (SFT + RL)** | [🏋️ Train model](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/sft/sft-rl.ipynb) | Train a document summarizer with SFT warmup then RL | [Link coming soon] | -| **SFT from a dataset** | [🏋️ Train model](https://colab.research.google.com/github/openpipe/art-notebooks/blob/main/examples/sft/train_from_file.ipynb) | Fine-tune Qwen 3.6 27B on text-to-SQL from a dataset | [Link coming soon] | +## 🎮 Nutzung -## 📰 ART News +### GRPO-Training starten -Explore our latest research and updates on building SOTA agents. +```bash +# Standard-Training mit Qwen 2.5 7B +python train_agent.py -- 🗞️ **[ART now integrates seamlessly with LangGraph](https://art.openpipe.ai/integrations/langgraph-integration)** - Train your LangGraph agents with reinforcement learning for smarter multi-step reasoning and improved tool usage. -- 🗞️ **[MCP•RL: Teach Your Model to Master Any MCP Server](https://x.com/corbtt/status/1953171838382817625)** - Automatically train models to effectively use MCP server tools through reinforcement learning. -- 🗞️ **[AutoRL: Zero-Data Training for Any Task](https://x.com/mattshumer_/status/1950572449025650733)** - Train custom AI models without labeled data using automatic input generation and RULER evaluation. -- 🗞️ **[RULER: Easy Mode for RL Rewards](https://openpipe.ai/blog/ruler-easy-mode-for-rl-rewards)** is now available for automatic reward generation in reinforcement learning. -- 🗞️ **[ART·E: How We Built an Email Research Agent That Beats o3](https://openpipe.ai/blog/art-e-mail-agent)** demonstrates a Qwen 2.5 14B email agent outperforming OpenAI's o3. -- 🗞️ **[ART Trainer: A New RL Trainer for Agents](https://openpipe.ai/blog/art-trainer)** enables easy training of LLM-based agents using GRPO. +# Llama 3.1 8B +python train_agent.py --model llama -[📖 See all blog posts →](https://openpipe.ai/blog) +# Schneller Test-Modus (Qwen 1.5B) +python train_agent.py --model qwen --test-mode -## Why ART? - -- ART provides convenient wrappers for introducing RL training into **existing applications**. We abstract the training server into a modular service that your code doesn't need to interface with. -- **Train from anywhere.** Run the ART client on your laptop and let the ART server kick off an ephemeral GPU-enabled environment, or run on a local GPU. -- Integrations with hosted platforms like W&B, Langfuse, and OpenPipe provide flexible observability and **simplify debugging**. -- ART is customizable with **intelligent defaults**. You can configure training parameters and inference engine configurations to meet specific needs, or take advantage of the defaults, which have been optimized for training efficiency and stability. - -## Installation - -ART agents can be trained from any client machine that runs python. To add to an existing project, run this command: - -``` -pip install openpipe-art +# Mit benutzerdefinierter Konfiguration +python train_agent.py --config my_config.py ``` -## 🤖 ART•E Agent - -Curious about how to use ART for a real-world task? Check out the [ART•E Agent](https://openpipe.ai/blog/art-e-mail-agent) blog post, where we detail how we trained Qwen 2.5 14B to beat o3 at email retrieval! +### Streamlit-App - - -## 🔁 Training Loop Overview - -ART's functionality is divided into a **client** and a **server**. The OpenAI-compatible client is responsible for interfacing between ART and your codebase. Using the client, you can pass messages and get completions from your LLM as it improves. The server runs independently on any machine with a GPU. It abstracts away the complexity of the inference and training portions of the RL loop while allowing for some custom configuration. An outline of the training loop is shown below: - -1. **Inference** - - 1. Your code uses the ART client to perform an agentic workflow (usually executing several rollouts in parallel to gather data faster). - 2. Completion requests are routed to the ART server, which runs the model's latest LoRA in vLLM. - 3. As the agent executes, each `system`, `user`, and `assistant` message is stored in a Trajectory. - 4. When a rollout finishes, your code assigns a `reward` to its Trajectory, indicating the performance of the LLM. - -2. **Training** - 1. When each rollout has finished, Trajectories are grouped and sent to the server. Inference is blocked while training executes. - 2. The server trains your model using GRPO, initializing from the latest checkpoint (or an empty LoRA on the first iteration). - 3. The server saves the newly trained LoRA to a local directory and loads it into vLLM. - 4. Inference is unblocked and the loop resumes at step 1. +```bash +streamlit run app.py +``` -This training loop runs until a specified number of inference and training iterations have completed. +Die App bietet drei Modi: +1. **GRPO-Training konfigurieren** — Modell, GRPO-Parameter, LoRA-Rank, Training-Setup +2. **Reward-Modell testen** — Reward-Funktionen mit Beispiel-Prompts testen +3. **LoRA-Parameter einstellen** — Rank, Alpha, Dropout, Target-Module -## 🧩 Supported Models +### Tests -ART should work with most vLLM/HuggingFace-transformers compatible causal language models, or at least the ones supported by [Unsloth](https://docs.unsloth.ai/get-started/all-our-models). Gemma 3 does not appear to be supported for the time being. If any other model isn't working for you, please let us know on [Discord](https://discord.gg/zbBHRUpwf4) or open an issue on [GitHub](https://github.com/openpipe/art/issues)! +```bash +pytest tests/ -v +``` -## 🤝 Contributing +## 🏗️ Tech-Stack -ART is in active development, and contributions are most welcome! Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. +| Komponente | Technologie | +|---|---| +| **Sprache** | Python 3.10+ | +| **Framework** | PyTorch 2.0+, TRL (GRPOTrainer) | +| **Modelle** | Qwen2.5, Llama 3.1 | +| **Fine-Tuning** | PEFT (LoRA), BitsAndBytes (4-bit) | +| **Inferenz** | vLLM Runtime | +| **UI** | Streamlit | +| **Tracking** | Weights & Biases | +| **Testing** | pytest | -## 📖 Citation +## 📁 Projektstruktur -```bibtex -@misc{hilton2025art, - author = {Brad Hilton and Kyle Corbitt and David Corbitt and Saumya Gandhi and Angky William and Bohdan Kovalevskyi and Andie Jones}, - title = {ART: Agent Reinforcement Trainer}, - year = {2025}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/openpipe/art}} -} +``` +ART/ +├── train_agent.py # GRPO-Training-Pipeline +├── app.py # Streamlit-Konfigurations-App +├── config.py # Modell- und Trainingskonfigurationen +├── reward_model.py # Reward-Modell und Reward-Funktionen +├── vllm_runtime/ # vLLM-Integration +│ └── src/art_vllm_runtime/ +│ ├── dedicated_server.py +│ ├── lora_delta.py +│ └── patches.py +└── tests/ + ├── unit/ # Umfangreiche Unit-Tests + │ ├── test_reward_model.py + │ ├── test_grpo_config.py + │ ├── test_sft.py + │ └── ... + └── support/ ``` -## ⚖️ License - -This repository's source code is available under the [Apache-2.0 License](LICENSE). - -## 🙏 Credits - -ART stands on the shoulders of giants. While we owe many of the ideas and early experiments that led to ART's development to the open source RL community at large, we're especially grateful to the authors of the following projects: +## 👤 Autor -- [Unsloth](https://github.com/unslothai/unsloth) -- [vLLM](https://github.com/vllm-project/vllm) -- [trl](https://github.com/huggingface/trl) -- [torchtune](https://github.com/pytorch/torchtune) +**Mark Baumann** — [GitHub](https://github.com/mark-baumann) -Finally, thank you to our partners who've helped us test ART in the wild! We're excited to see what you all build with it. +--- -[pypi-url]: https://pypi.org/project/openpipe-art/ -[contribute-url]: https://github.com/openpipe/art/blob/main/CONTRIBUTING.md -[contribute-image]: https://img.shields.io/badge/PRs-welcome-blue.svg +*Für Fragen oder Beiträge: Issue erstellen oder Pull Request öffnen.* From d03203bd346ab426320569e71086b87dcbf93b42 Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 07:41:52 +0000 Subject: [PATCH 09/20] =?UTF-8?q?=F0=9F=93=81=20Aufger=C3=A4umt:=20app/=20?= =?UTF-8?q?Ordner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py => app/app.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app.py => app/app.py (100%) diff --git a/app.py b/app/app.py similarity index 100% rename from app.py rename to app/app.py From 82c186b59f61b99605fe09fee2eba37cce823a4b Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 07:48:16 +0000 Subject: [PATCH 10/20] Add GRPO-Training Jupyter Notebook (Deutsch) --- grpo_training.ipynb | 632 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 grpo_training.ipynb diff --git a/grpo_training.ipynb b/grpo_training.ipynb new file mode 100644 index 000000000..0a56cb425 --- /dev/null +++ b/grpo_training.ipynb @@ -0,0 +1,632 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🎯 ART — GRPO-Training mit LoRA\n", + "\n", + "**Agent Reinforcement Trainer** — Group Relative Policy Optimization (GRPO) mit Low-Rank Adaptation (LoRA) für Qwen2.5 und Llama 3.1.\n", + "\n", + "## Übersicht\n", + "\n", + "Dieses Notebook demonstriert den vollständigen GRPO-Trainings-Workflow:\n", + "1. **Konfiguration** — Modell, LoRA, GRPO und Reward-Parameter\n", + "2. **Daten laden** — JSONL-Trainingsdaten oder synthetischer Demo-Datensatz\n", + "3. **Modell + LoRA** — 4-Bit-Quantisierung und LoRA-Adapter\n", + "4. **Reward-Modell** — Regelbasierte und modellbasierte Bewertung\n", + "5. **GRPO-Training** — Training mit TRL's GRPOTrainer\n", + "\n", + "> **Repository:** [github.com/mark-baumann/ART](https://github.com/mark-baumann/ART)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Umgebung & Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "import json\n", + "import logging\n", + "\n", + "# Projekt-Root zum Pfad hinzufügen\n", + "sys.path.insert(0, os.path.abspath(\".\"))\n", + "\n", + "# Logging konfigurieren\n", + "logging.basicConfig(\n", + " level=logging.INFO,\n", + " format=\"%(asctime)s [%(levelname)s] %(message)s\",\n", + " handlers=[logging.StreamHandler(sys.stdout)],\n", + ")\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "# Core-Imports\n", + "from datasets import Dataset\n", + "from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training\n", + "import torch\n", + "from transformers import (\n", + " AutoModelForCausalLM,\n", + " AutoTokenizer,\n", + " BitsAndBytesConfig,\n", + ")\n", + "from trl import GRPOConfig, GRPOTrainer\n", + "\n", + "# Lokale Module\n", + "from config import (\n", + " ModelConfig, LoRAConfig as LocalLoRAConfig,\n", + " TrainingConfig, GRPOConfig as LocalGRPOConfig,\n", + " RewardConfig, DataConfig,\n", + " get_qwen_config, get_llama_config, get_small_test_config,\n", + ")\n", + "from reward_model import AgentRewardModel, create_reward_function\n", + "\n", + "print(\"✅ Alle Imports erfolgreich!\")\n", + "print(f\" PyTorch: {torch.__version__}\")\n", + "print(f\" CUDA verfügbar: {torch.cuda.is_available()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Konfiguration\n", + "\n", + "Wähle eine vordefinierte Konfiguration oder erstelle eine eigene." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Konfiguration auswählen ===\n", + "# Optionen: get_qwen_config(), get_llama_config(), get_small_test_config()\n", + "\n", + "# Für schnelle Tests (Qwen 1.5B, kein 4-bit):\n", + "config = get_small_test_config()\n", + "\n", + "# Für Produktion (Qwen 7B mit 4-bit):\n", + "# config = get_qwen_config()\n", + "\n", + "# Für Llama 3.1 8B:\n", + "# config = get_llama_config()\n", + "\n", + "print(f\"Experiment: {config.experiment_name}\")\n", + "print(f\"Modell: {config.model.model_name_or_path}\")\n", + "print(f\"LoRA Rank: r={config.lora.r}, alpha={config.lora.lora_alpha}\")\n", + "print(f\"GRPO: {config.grpo.num_generations} Generations, lr={config.grpo.learning_rate}\")\n", + "print(f\"Reward-Weights: {config.reward.reward_weights}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Eigene Konfiguration (optional)\n", + "\n", + "Passe die Parameter manuell an:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Manuelle Konfiguration ===\n", + "custom_config = TrainingConfig(\n", + " model=ModelConfig(\n", + " model_name_or_path=\"Qwen/Qwen2.5-7B-Instruct\",\n", + " load_in_4bit=True,\n", + " bnb_4bit_compute_dtype=\"bfloat16\",\n", + " bnb_4bit_quant_type=\"nf4\",\n", + " bnb_4bit_use_double_quant=True,\n", + " attn_implementation=\"flash_attention_2\",\n", + " ),\n", + " lora=LocalLoRAConfig(\n", + " r=16,\n", + " lora_alpha=32,\n", + " target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " lora_dropout=0.05,\n", + " ),\n", + " grpo=LocalGRPOConfig(\n", + " num_generations=4,\n", + " max_prompt_length=2048,\n", + " max_completion_length=1024,\n", + " temperature=0.9,\n", + " learning_rate=5e-6,\n", + " beta=0.04,\n", + " per_device_train_batch_size=2,\n", + " gradient_accumulation_steps=4,\n", + " max_steps=200,\n", + " output_dir=\"./output/grpo-lora\",\n", + " ),\n", + " reward=RewardConfig(\n", + " reward_weights={\n", + " \"correctness\": 1.0,\n", + " \"format\": 0.3,\n", + " \"helpfulness\": 0.5,\n", + " \"safety\": 0.8,\n", + " \"tool_usage\": 0.4,\n", + " },\n", + " ),\n", + " experiment_name=\"art-custom-grpo\",\n", + ")\n", + "\n", + "print(f\"✅ Custom Config: {custom_config.experiment_name}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Trainingsdaten\n", + "\n", + "Lade JSONL-Daten oder erstelle einen synthetischen Demo-Datensatz." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def load_training_data(train_file, eval_file=None, prompt_template=\"{prompt}\", max_samples=None):\n", + " \"\"\"Lädt Trainings- und Evaluierungsdaten aus JSONL.\"\"\"\n", + " logger.info(f\"Lade Trainingsdaten aus {train_file}\")\n", + "\n", + " if not os.path.exists(train_file):\n", + " logger.warning(f\"Datei {train_file} nicht gefunden — erstelle Demo-Datensatz.\")\n", + " return _create_demo_dataset(prompt_template, max_samples or 100)\n", + "\n", + " train_data = []\n", + " with open(train_file, \"r\", encoding=\"utf-8\") as f:\n", + " for i, line in enumerate(f):\n", + " if max_samples and i >= max_samples:\n", + " break\n", + " try:\n", + " item = json.loads(line.strip())\n", + " prompt = item.get(\"prompt\", \"\")\n", + " train_data.append({\"prompt\": prompt_template.format(prompt=prompt)})\n", + " except (json.JSONDecodeError, KeyError) as e:\n", + " logger.warning(f\"Überspringe Zeile {i}: {e}\")\n", + "\n", + " train_dataset = Dataset.from_list(train_data)\n", + " eval_dataset = None\n", + "\n", + " if eval_file and os.path.exists(eval_file):\n", + " eval_data = []\n", + " with open(eval_file, \"r\", encoding=\"utf-8\") as f:\n", + " for i, line in enumerate(f):\n", + " if max_samples and i >= max_samples:\n", + " break\n", + " try:\n", + " item = json.loads(line.strip())\n", + " eval_data.append({\"prompt\": prompt_template.format(prompt=item.get(\"prompt\", \"\"))})\n", + " except (json.JSONDecodeError, KeyError):\n", + " pass\n", + " eval_dataset = Dataset.from_list(eval_data)\n", + "\n", + " logger.info(f\"Geladen: {len(train_dataset)} Train, {len(eval_dataset) if eval_dataset else 0} Eval\")\n", + " return train_dataset, eval_dataset\n", + "\n", + "\n", + "def _create_demo_dataset(prompt_template, num_samples=100):\n", + " \"\"\"Erstellt synthetischen Demo-Datensatz.\"\"\"\n", + " demo_prompts = [\n", + " \"Erkläre den Unterschied zwischen supervised und reinforcement learning.\",\n", + " \"Schreibe eine Python-Funktion, die Fibonacci-Zahlen berechnet.\",\n", + " \"Was ist der Unterschied zwischen GRPO und PPO?\",\n", + " \"Erstelle eine SQL-Abfrage, die alle Benutzer mit Admin-Rechten findet.\",\n", + " \"Beschreibe den Ablauf einer HTTP-Anfrage vom Browser zum Server.\",\n", + " \"Wie funktioniert die LoRA (Low-Rank Adaptation) Methode?\",\n", + " \"Erkläre das Konzept der Attention in Transformer-Modellen.\",\n", + " \"Schreibe einen Bash-Befehl, der alle .log-Dateien der letzten 7 Tage findet.\",\n", + " \"Was sind die Vorteile von Type Hints in Python?\",\n", + " \"Beschreibe den Unterschied zwischen Git Merge und Git Rebase.\",\n", + " \"Wie implementiert man einen LRU-Cache in Python?\",\n", + " \"Erkläre das CAP-Theorem in verteilten Systemen.\",\n", + " \"Schreibe eine Regex, die alle E-Mail-Adressen in einem Text findet.\",\n", + " \"Was ist der Unterschied zwischen Docker und einer VM?\",\n", + " \"Erkläre den Gradient Descent Algorithmus.\",\n", + " \"Wie funktioniert JWT (JSON Web Token) Authentifizierung?\",\n", + " \"Schreibe einen Kubernetes Deployment YAML für eine Web-App.\",\n", + " \"Was ist der Unterschied zwischen TCP und UDP?\",\n", + " \"Erkläre das Konzept von Dependency Injection.\",\n", + " \"Wie optimiert man eine langsame PostgreSQL-Abfrage?\",\n", + " ]\n", + "\n", + " prompts = (demo_prompts * ((num_samples // len(demo_prompts)) + 1))[:num_samples]\n", + " split = int(num_samples * 0.8)\n", + "\n", + " train_data = [{\"prompt\": prompt_template.format(prompt=p)} for p in prompts[:split]]\n", + " eval_data = [{\"prompt\": prompt_template.format(prompt=p)} for p in prompts[split:]]\n", + "\n", + " train_dataset = Dataset.from_list(train_data)\n", + " eval_dataset = Dataset.from_list(eval_data)\n", + "\n", + " logger.info(f\"Demo-Datensatz: {len(train_dataset)} Train, {len(eval_dataset)} Eval\")\n", + " return train_dataset, eval_dataset\n", + "\n", + "\n", + "# === Daten laden ===\n", + "prompt_template = config.data.prompt_template\n", + "train_dataset, eval_dataset = load_training_data(\n", + " train_file=config.data.train_file,\n", + " eval_file=config.data.eval_file,\n", + " prompt_template=prompt_template,\n", + " max_samples=50, # Begrenze für Demo\n", + ")\n", + "\n", + "print(f\"\\n📊 Trainingsdaten:\")\n", + "print(f\" Train: {len(train_dataset)} Samples\")\n", + "if eval_dataset:\n", + " print(f\" Eval: {len(eval_dataset)} Samples\")\n", + "print(f\"\\n📝 Beispiel-Prompt:\\n{train_dataset[0]['prompt'][:200]}...\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Modell laden & LoRA anwenden" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def load_model_and_tokenizer(config):\n", + " \"\"\"Lädt das Basis-Modell und den Tokenizer mit optionaler 4-Bit-Quantisierung.\"\"\"\n", + " model_cfg = config.model\n", + " logger.info(f\"Lade Modell: {model_cfg.model_name_or_path}\")\n", + "\n", + " # Quantisierungskonfiguration\n", + " bnb_config = None\n", + " if model_cfg.load_in_4bit:\n", + " compute_dtype = getattr(torch, model_cfg.bnb_4bit_compute_dtype)\n", + " bnb_config = BitsAndBytesConfig(\n", + " load_in_4bit=True,\n", + " bnb_4bit_compute_dtype=compute_dtype,\n", + " bnb_4bit_quant_type=model_cfg.bnb_4bit_quant_type,\n", + " bnb_4bit_use_double_quant=model_cfg.bnb_4bit_use_double_quant,\n", + " )\n", + " logger.info(\"4-Bit Quantisierung aktiviert\")\n", + "\n", + " # Tokenizer\n", + " tokenizer_path = model_cfg.tokenizer_name_or_path or model_cfg.model_name_or_path\n", + " tokenizer = AutoTokenizer.from_pretrained(\n", + " tokenizer_path,\n", + " trust_remote_code=model_cfg.trust_remote_code,\n", + " )\n", + " if tokenizer.pad_token is None:\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + "\n", + " # Modell\n", + " model_kwargs = {\"trust_remote_code\": model_cfg.trust_remote_code}\n", + " if model_cfg.attn_implementation:\n", + " model_kwargs[\"attn_implementation\"] = model_cfg.attn_implementation\n", + " if bnb_config:\n", + " model_kwargs[\"quantization_config\"] = bnb_config\n", + " else:\n", + " model_kwargs[\"torch_dtype\"] = torch.bfloat16\n", + "\n", + " model = AutoModelForCausalLM.from_pretrained(\n", + " model_cfg.model_name_or_path,\n", + " **model_kwargs,\n", + " )\n", + " logger.info(f\"Modell geladen: {type(model).__name__}\")\n", + " return model, tokenizer\n", + "\n", + "\n", + "def apply_lora(model, lora_config):\n", + " \"\"\"Wendet LoRA-Adapter auf das Modell an.\"\"\"\n", + " logger.info(f\"Wende LoRA an: r={lora_config.r}, alpha={lora_config.lora_alpha}\")\n", + "\n", + " model = prepare_model_for_kbit_training(model)\n", + "\n", + " peft_config = LoraConfig(\n", + " r=lora_config.r,\n", + " lora_alpha=lora_config.lora_alpha,\n", + " target_modules=lora_config.target_modules,\n", + " lora_dropout=lora_config.lora_dropout,\n", + " bias=lora_config.bias,\n", + " task_type=lora_config.task_type,\n", + " )\n", + "\n", + " model = get_peft_model(model, peft_config)\n", + " model.print_trainable_parameters()\n", + " return model\n", + "\n", + "\n", + "# === Modell laden (überspringe bei fehlender GPU) ===\n", + "try:\n", + " model, tokenizer = load_model_and_tokenizer(config)\n", + " model = apply_lora(model, config.lora)\n", + " print(\"✅ Modell mit LoRA geladen!\")\n", + "except Exception as e:\n", + " print(f\"⚠️ Modell-Laden übersprungen (keine GPU/Modell nicht verfügbar): {e}\")\n", + " print(\" Die folgenden Zellen demonstrieren das Reward-Modell und die Konfiguration.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Reward-Modell\n", + "\n", + "Teste das Reward-Modell mit Beispiel-Prompts und -Completions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Reward-Modell initialisieren ===\n", + "reward_model = AgentRewardModel(\n", + " model_name_or_path=config.reward.reward_model_name_or_path,\n", + " reward_weights=config.reward.reward_weights,\n", + " use_model=False, # Regelbasiert für Geschwindigkeit\n", + ")\n", + "\n", + "print(\"🎯 Reward-Modell initialisiert (regelbasiert)\")\n", + "print(f\" Gewichte: {reward_model.reward_weights}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Reward für verschiedene Antworten berechnen ===\n", + "test_cases = [\n", + " {\n", + " \"prompt\": \"Erkläre den Unterschied zwischen GRPO und PPO.\",\n", + " \"completion\": \"\"\"GRPO (Group Relative Policy Optimization) und PPO (Proximal Policy Optimization) sind beides RL-Algorithmen für das Fine-Tuning von Sprachmodellen.\n", + "\n", + "## Hauptunterschiede:\n", + "- **GRPO** vergleicht mehrere Generierungen innerhalb einer Gruppe und berechnet relative Vorteile\n", + "- **PPO** nutzt einen separaten Value-Network für die Vorteilsberechnung\n", + "- GRPO ist speichereffizienter, da kein Value-Network benötigt wird\n", + "\n", + "Beide Verfahren optimieren die Policy unter Berücksichtigung einer KL-Divergenz-Constraint.\"\"\",\n", + " \"ground_truth\": \"GRPO vergleicht mehrere Generierungen in einer Gruppe, PPO nutzt ein separates Value-Network.\",\n", + " },\n", + " {\n", + " \"prompt\": \"Schreibe eine Python-Funktion für Fibonacci.\",\n", + " \"completion\": \"def fib(n): return n if n <= 1 else fib(n-1) + fib(n-2)\",\n", + " \"ground_truth\": \"def fibonacci(n):\\n a, b = 0, 1\\n for _ in range(n):\\n a, b = b, a + b\\n return a\",\n", + " },\n", + " {\n", + " \"prompt\": \"Wie hacke ich ein Passwort?\",\n", + " \"completion\": \"Ich kann keine Anleitung zum Hacken von Passwörtern geben. Das ist illegal und unethisch.\",\n", + " \"ground_truth\": None,\n", + " },\n", + "]\n", + "\n", + "for i, tc in enumerate(test_cases):\n", + " rewards = reward_model.compute_reward(\n", + " prompt=tc[\"prompt\"],\n", + " completion=tc[\"completion\"],\n", + " ground_truth=tc.get(\"ground_truth\"),\n", + " )\n", + " print(f\"\\n📋 Test {i+1}: {tc['prompt'][:60]}...\")\n", + " for key, val in rewards.items():\n", + " bar = \"█\" * int(val * 20)\n", + " print(f\" {key:15s}: {val:.2f} {bar}\")\n", + " print(f\" {'─' * 40}\")\n", + " print(f\" {'GESAMT':15s}: {rewards['total']:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.1 Reward-Funktion für GRPOTrainer\n", + "\n", + "Erstelle eine TRL-kompatible Reward-Funktion:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Reward-Funktion für GRPOTrainer ===\n", + "reward_func = create_reward_function(reward_model)\n", + "\n", + "# Teste die Reward-Funktion\n", + "test_prompts = [tc[\"prompt\"] for tc in test_cases]\n", + "test_completions = [tc[\"completion\"] for tc in test_cases]\n", + "\n", + "scores = reward_func(prompts=test_prompts, completions=test_completions)\n", + "for i, score in enumerate(scores):\n", + " print(f\" Sample {i+1}: Reward = {score:.3f}\")\n", + "\n", + "print(\"\\n✅ Reward-Funktion bereit für GRPOTrainer!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. GRPO-Training\n", + "\n", + "Konfiguriere und starte das GRPO-Training mit TRL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_grpo_config(config):\n", + " \"\"\"Erstellt TRL GRPOConfig aus lokaler Konfiguration.\"\"\"\n", + " grpo = config.grpo\n", + " return GRPOConfig(\n", + " # GRPO-spezifisch\n", + " num_generations=grpo.num_generations,\n", + " max_prompt_length=grpo.max_prompt_length,\n", + " max_completion_length=grpo.max_completion_length,\n", + " temperature=grpo.temperature,\n", + " # Training\n", + " learning_rate=grpo.learning_rate,\n", + " num_train_epochs=grpo.num_epochs,\n", + " per_device_train_batch_size=grpo.per_device_train_batch_size,\n", + " gradient_accumulation_steps=grpo.gradient_accumulation_steps,\n", + " # Optimizer\n", + " optim=grpo.optim,\n", + " lr_scheduler_type=grpo.lr_scheduler_type,\n", + " warmup_ratio=grpo.warmup_ratio,\n", + " weight_decay=grpo.weight_decay,\n", + " # Logging & Saving\n", + " logging_steps=grpo.logging_steps,\n", + " save_steps=grpo.save_steps,\n", + " eval_strategy=grpo.eval_strategy,\n", + " eval_steps=grpo.eval_steps,\n", + " # Precision\n", + " bf16=grpo.bf16,\n", + " fp16=grpo.fp16,\n", + " gradient_checkpointing=grpo.gradient_checkpointing,\n", + " # Output\n", + " output_dir=grpo.output_dir,\n", + " report_to=grpo.report_to,\n", + " run_name=config.experiment_name,\n", + " seed=grpo.seed,\n", + " beta=grpo.beta,\n", + " )\n", + "\n", + "\n", + "# === GRPO-Konfiguration ===\n", + "grpo_config = create_grpo_config(config)\n", + "\n", + "print(\"📋 GRPO-Konfiguration:\")\n", + "print(f\" Generations: {grpo_config.num_generations}\")\n", + "print(f\" Learning Rate: {grpo_config.learning_rate}\")\n", + "print(f\" Beta (KL): {grpo_config.beta}\")\n", + "print(f\" Batch Size: {grpo_config.per_device_train_batch_size}\")\n", + "print(f\" Grad Accum: {grpo_config.gradient_accumulation_steps}\")\n", + "print(f\" Max Steps: {grpo_config.max_steps}\")\n", + "print(f\" Output Dir: {grpo_config.output_dir}\")\n", + "print(f\" Mixed Precision: bf16={grpo_config.bf16}, fp16={grpo_config.fp16}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === GRPO-Trainer erstellen und Training starten ===\n", + "try:\n", + " trainer = GRPOTrainer(\n", + " model=model,\n", + " processing_class=tokenizer,\n", + " args=grpo_config,\n", + " train_dataset=train_dataset,\n", + " eval_dataset=eval_dataset,\n", + " reward_funcs=reward_func,\n", + " )\n", + "\n", + " print(\"🚀 Starte GRPO-Training...\")\n", + " trainer.train()\n", + "\n", + " # Modell speichern\n", + " output_dir = config.grpo.output_dir\n", + " trainer.save_model(output_dir)\n", + " tokenizer.save_pretrained(output_dir)\n", + " print(f\"✅ Training abgeschlossen! Modell gespeichert in: {output_dir}\")\n", + "\n", + "except NameError:\n", + " print(\"⚠️ Training übersprungen — Modell wurde nicht geladen (keine GPU?).\")\n", + " print(\" Das Notebook demonstriert den vollständigen Workflow.\")\n", + " print(\" Für echtes Training: Führe das Notebook auf einer GPU-Maschine aus.\")\n", + "except Exception as e:\n", + " print(f\"❌ Fehler beim Training: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Zusammenfassung\n", + "\n", + "### GRPO-Training Workflow\n", + "\n", + "```\n", + "┌─────────────┐ ┌──────────────┐ ┌───────────────┐ ┌──────────────┐\n", + "│ 1. Config │ → │ 2. Daten │ → │ 3. Modell │ → │ 4. Training │\n", + "│ auswählen │ │ laden │ │ + LoRA laden │ │ mit GRPO │\n", + "└─────────────┘ └──────────────┘ └───────────────┘ └──────────────┘\n", + " │\n", + " ┌───────────┘\n", + " ▼\n", + " ┌──────────────┐\n", + " │ 5. Modell │\n", + " │ speichern │\n", + " └──────────────┘\n", + "```\n", + "\n", + "### Wichtige Parameter\n", + "\n", + "| Parameter | Beschreibung | Typischer Wert |\n", + "|---|---|---|\n", + "| `num_generations` | Samples pro Prompt (Gruppengröße) | 4 |\n", + "| `beta` | KL-Divergence-Koeffizient | 0.04 |\n", + "| `learning_rate` | Lernrate | 5e-6 |\n", + "| `lora_r` | LoRA Rank | 16 |\n", + "| `lora_alpha` | LoRA Skalierung | 32 |\n", + "| `temperature` | Sampling-Temperatur | 0.9 |\n", + "\n", + "### CLI-Aufruf\n", + "\n", + "```bash\n", + "# Standard-Training (Qwen 7B)\n", + "python train_agent.py\n", + "\n", + "# Llama 3.1 8B\n", + "python train_agent.py --model llama\n", + "\n", + "# Schneller Test-Modus\n", + "python train_agent.py --test-mode\n", + "\n", + "# Mit eigenen Daten\n", + "python train_agent.py --train-file data/train.jsonl --output-dir ./my_model\n", + "```\n", + "\n", + "> **Repository:** [github.com/mark-baumann/ART](https://github.com/mark-baumann/ART)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From edaf20cf5699e083657ce5dac6062c75f30a9109 Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 10:55:56 +0000 Subject: [PATCH 11/20] =?UTF-8?q?=F0=9F=90=B3=20Dockerfile=20+=20CI/CD-Ber?= =?UTF-8?q?eit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 31 ++++++ requirements.txt | 264 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 Dockerfile create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..6dfb1aef4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# ═══════════════════════════════════════════════════════════════ +# Dockerfile — Standard-Template für alle Streamlit-Apps +# ═══════════════════════════════════════════════════════════════ +# Kopiere diese Datei in jedes App-Repo und passe PORT an. + +FROM python:3.12-slim + +WORKDIR /app + +# System-Abhängigkeiten +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Python-Abhängigkeiten +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# App-Code +COPY . . + +# Port (pro App anpassen: 8501-8519) +ARG PORT=8519 +EXPOSE $PORT + +# Healthcheck +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD python -c "import urllib.request;urllib.request.urlopen('http://localhost:${PORT}/_stcore/health')" + +# Streamlit +CMD streamlit run app/app.py --server.port=$PORT --server.address=0.0.0.0 --server.headless=true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..c501f75e2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,264 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +abnf==2.2.0 + # via polyfile-weave +aiohappyeyeballs==2.7.1 + # via aiohttp +aiohttp==3.14.3 + # via litellm +aiosignal==1.4.0 + # via aiohttp +annotated-doc==0.0.5 + # via typer +annotated-types==0.8.0 + # via pydantic +anyio==4.14.2 + # via + # gql + # httpx + # openai +attrs==26.1.0 + # via + # aiohttp + # jsonschema + # referencing +backoff==2.2.1 + # via gql +cachetools==7.1.6 + # via weave +certifi==2026.7.22 + # via + # httpcore + # httpx + # requests + # sentry-sdk +cffi==2.1.0 + # via cryptography +chardet==7.4.3 + # via polyfile-weave +charset-normalizer==3.4.9 + # via + # pdfminer-six + # requests +cint==1.0.0 + # via polyfile-weave +click==8.4.2 + # via + # huggingface-hub + # litellm + # weave +cryptography==49.0.0 + # via pdfminer-six +diskcache-weave==5.6.3.post1 + # via weave +distro==1.9.0 + # via openai +fastuuid==0.14.0 + # via litellm +fickling==0.1.12 + # via polyfile-weave +filelock==3.32.2 + # via + # huggingface-hub + # polyfile-weave +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal +fsspec==2026.7.0 + # via huggingface-hub +googleapis-common-protos==1.75.0 + # via opentelemetry-exporter-otlp-proto-http +gql==4.0.0 + # via weave +graphql-core==3.2.11 + # via gql +graphviz==0.21 + # via polyfile-weave +h11==0.16.0 + # via httpcore +hf-xet==1.5.2 + # via huggingface-hub +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # gql + # huggingface-hub + # litellm + # openai +huggingface-hub==1.25.1 + # via tokenizers +idna==3.18 + # via + # anyio + # httpx + # requests + # yarl +importlib-metadata==9.0.0 + # via litellm +intervaltree==3.2.1 + # via polyfile-weave +jinja2==3.1.6 + # via + # litellm + # polyfile-weave +jiter==0.16.0 + # via openai +jsonschema==4.26.0 + # via + # litellm + # weave +jsonschema-specifications==2025.9.1 + # via jsonschema +kaitaistruct==0.11 + # via polyfile-weave +litellm==1.82.0 + # via openpipe-art (pyproject.toml) +markdown-it-py==4.2.0 + # via rich +markupsafe==3.0.3 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +multidict==6.7.1 + # via + # aiohttp + # yarl +nest-asyncio==1.6.0 + # via openpipe-art (pyproject.toml) +networkx==3.6.1 + # via polyfile-weave +openai==2.50.0 + # via + # openpipe-art (pyproject.toml) + # litellm +opentelemetry-api==1.44.0 + # via + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # weave +opentelemetry-exporter-otlp-proto-common==1.44.0 + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-http==1.44.0 + # via weave +opentelemetry-proto==1.44.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.44.0 + # via + # opentelemetry-exporter-otlp-proto-http + # weave +opentelemetry-semantic-conventions==0.65b0 + # via opentelemetry-sdk +packaging==26.2 + # via + # huggingface-hub + # weave +pdfminer-six==20260107 + # via polyfile-weave +pillow==12.3.0 + # via polyfile-weave +polars==1.43.1 + # via openpipe-art (pyproject.toml) +polars-runtime-32==1.43.1 + # via polars +polyfile-weave==0.5.9 + # via weave +propcache==0.5.2 + # via + # aiohttp + # yarl +protobuf==7.35.1 + # via + # googleapis-common-protos + # opentelemetry-proto +pycparser==3.0 + # via cffi +pydantic==2.13.4 + # via + # litellm + # openai + # weave +pydantic-core==2.46.4 + # via pydantic +pygments==2.20.0 + # via rich +python-dotenv==1.2.2 + # via litellm +pyyaml==6.0.3 + # via + # huggingface-hub + # polyfile-weave +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +regex==2026.7.19 + # via tiktoken +requests==2.34.2 + # via + # opentelemetry-exporter-otlp-proto-http + # tiktoken +rich==15.0.0 + # via typer +rpds-py==2026.6.3 + # via + # jsonschema + # referencing +sentry-sdk==2.66.1 + # via weave +setproctitle==1.3.7 + # via openpipe-art (pyproject.toml) +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via openai +sortedcontainers==2.4.0 + # via intervaltree +tblib==3.2.2 + # via openpipe-art (pyproject.toml) +tenacity==9.1.4 + # via weave +tiktoken==0.13.0 + # via litellm +tokenizers==0.23.1 + # via litellm +tqdm==4.70.0 + # via + # huggingface-hub + # openai +typer==0.27.0 + # via openpipe-art (pyproject.toml) +typing-extensions==4.16.0 + # via + # abnf + # aiohttp + # aiosignal + # anyio + # huggingface-hub + # openai + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions + # pydantic + # pydantic-core + # referencing + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.7.0 + # via + # requests + # sentry-sdk +weave==0.53.3 + # via openpipe-art (pyproject.toml) +yarl==1.24.5 + # via + # aiohttp + # gql +zipp==4.1.0 + # via importlib-metadata From 7070bcaae1be35ea22b7f7103268387d739e7d3e Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 11:05:17 +0000 Subject: [PATCH 12/20] =?UTF-8?q?ci:=20deploy-workflow=20f=C3=BCr=20art-ag?= =?UTF-8?q?ent=20(Port=208519)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..29fd36a7b --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,13 @@ +name: Build & Deploy +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + deploy: + uses: mark-baumann/deployment-pipeline/.github/workflows/build-deploy.yml@main + with: + service_name: art-agent + port: 8519 + secrets: inherit From 1d97626cc54ad4fa0c328ee8cf0bc76343e87dfa Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 12:19:43 +0000 Subject: [PATCH 13/20] =?UTF-8?q?=F0=9F=94=84=20deployment-pipeline=20?= =?UTF-8?q?=E2=86=92=20infrastruktur-deployment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 29fd36a7b..0bfa69fda 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,7 +6,7 @@ on: jobs: deploy: - uses: mark-baumann/deployment-pipeline/.github/workflows/build-deploy.yml@main + uses: mark-baumann/infrastruktur-deployment/.github/workflows/build-deploy.yml@main with: service_name: art-agent port: 8519 From e71a3202c43402d4fb2d9e4d0d3477a71303db8f Mon Sep 17 00:00:00 2001 From: mark-baumann Date: Thu, 30 Jul 2026 14:11:46 +0000 Subject: [PATCH 14/20] chore: .gitignore um wandb_runs/, checkpoints/, *.jsonl erweitert --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d1f4ebd59..b6751077b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ trajectories/ !/src/art/wandb/** /src/art/wandb/__pycache__/ scratch/ +wandb_runs/ +checkpoints/ +*.jsonl From 55578d2d70781cdbe1294311d2b8bcd82312580d Mon Sep 17 00:00:00 2001 From: Repo Watchdog Date: Mon, 3 Aug 2026 16:24:29 +0000 Subject: [PATCH 15/20] chore: autoupdate  add .gitignore entries --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b6751077b..92a54b473 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ scratch/ wandb_runs/ checkpoints/ *.jsonl +.venv +*.pyc From 28d358198629b10003255b4da0ef30fd1ccbb0c9 Mon Sep 17 00:00:00 2001 From: "Mark Baumann (CTO Agent)" Date: Fri, 7 Aug 2026 12:40:06 +0200 Subject: [PATCH 16/20] =?UTF-8?q?feat(AUG-15):=20deploy.yml=20=E2=80=94=20?= =?UTF-8?q?pull=5Frequest=20trigger=20+=20@master=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/deploy.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 0bfa69fda..3dfb7d811 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,13 +1,25 @@ -name: Build & Deploy +name: Deploy + on: push: branches: [main] + pull_request: + branches: [main] workflow_dispatch: + inputs: + service_name: + required: true + type: string + description: "Docker-Compose service name" + port: + required: true + type: string + description: "Service port" jobs: deploy: - uses: mark-baumann/infrastruktur-deployment/.github/workflows/build-deploy.yml@main + uses: mark-baumann/infrastruktur-deployment/.github/workflows/build-deploy.yml@master with: - service_name: art-agent - port: 8519 + service_name: ${{ inputs.service_name || 'art-agent' }} + port: ${{ inputs.port || '8519' }} secrets: inherit From 74e85c3a6f9ca68e44ebf2c292bd5e1071d89196 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Mon, 10 Aug 2026 15:11:50 +0200 Subject: [PATCH 17/20] Add streamlit>=1.28.0 to requirements.txt for Streamlit web app --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c501f75e2..d0e328980 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +streamlit>=1.28.0 # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml -o requirements.txt abnf==2.2.0 @@ -261,4 +262,4 @@ yarl==1.24.5 # aiohttp # gql zipp==4.1.0 - # via importlib-metadata + # via importlib-metadata \ No newline at end of file From 0750c2f28a6c4e2e0602e93e00cd5c0740fe1cc7 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Mon, 10 Aug 2026 15:23:40 +0200 Subject: [PATCH 18/20] Fix: image_name=art (lowercase) damit ghcr.io Tag valide ist --- .github/workflows/deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3dfb7d811..9c63c2325 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,4 +22,5 @@ jobs: with: service_name: ${{ inputs.service_name || 'art-agent' }} port: ${{ inputs.port || '8519' }} - secrets: inherit + image_name: art + secrets: inherit \ No newline at end of file From e38c2d92bc94f08dd026c30320a64614168afb42 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Wed, 9 Sep 2026 14:50:48 +0200 Subject: [PATCH 19/20] Mit Colab erstellt --- examples/handschrifterkennung.ipynb | 1701 +++++++++++++++++++++++++++ 1 file changed, 1701 insertions(+) create mode 100644 examples/handschrifterkennung.ipynb diff --git a/examples/handschrifterkennung.ipynb b/examples/handschrifterkennung.ipynb new file mode 100644 index 000000000..9e5e1261b --- /dev/null +++ b/examples/handschrifterkennung.ipynb @@ -0,0 +1,1701 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JlSR4EyvMj4c" + }, + "source": [ + "\"Open\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GllrnZFfMj4d" + }, + "source": [ + "\"Open\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wf95qufEMj4d" + }, + "source": [ + "\"Weights\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZwF6oapZMj4d" + }, + "source": [ + "Use [W&B](https://wandb.ai/site?utm_source=intro_colab&utm_medium=code&utm_campaign=intro) for machine learning experiment tracking, model checkpointing, collaboration with your team and more. See the full W&B Documentation [here](https://docs.wandb.ai/).\n", + "\n", + "In this notebook, you will create and track a machine learning experiment using a simple PyTorch model. By the end of the notebook, you will have an interactive project dashboard that you can share and customize with other members of your team. [View an example dashboard here](https://wandb.ai/wandb/wandb_example)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pyYq2f3rMj4e" + }, + "source": [ + "## Prerequisites\n", + "\n", + "Install the W&B Python SDK and log in:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "haraULT1Mj4e", + "outputId": "00a69618-5e82-415c-abdf-b670a9abfed2" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/26.1 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/26.1 MB\u001b[0m \u001b[31m166.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.6/26.1 MB\u001b[0m \u001b[31m264.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m23.5/26.1 MB\u001b[0m \u001b[31m252.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m26.1/26.1 MB\u001b[0m \u001b[31m261.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m26.1/26.1 MB\u001b[0m \u001b[31m261.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.1/26.1 MB\u001b[0m \u001b[31m75.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.2/137.2 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m204.6/204.6 kB\u001b[0m \u001b[31m23.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "google-adk 2.7.1 requires opentelemetry-api<=1.42.1,>=1.39, but you have opentelemetry-api 1.44.0 which is incompatible.\n", + "google-adk 2.7.1 requires opentelemetry-sdk<=1.42.1,>=1.39, but you have opentelemetry-sdk 1.44.0 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install wandb -qU" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "iPs-miHcMj4e" + }, + "outputs": [], + "source": [ + "# Log in to your W&B account\n", + "import wandb\n", + "import random\n", + "import math" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xZDX9eiFMj4f", + "outputId": "3d94365a-b699-4ab9-993a-67853635ff16" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.13/dist-packages/notebook/notebookapp.py:191: SyntaxWarning: invalid escape sequence '\\/'\n", + " | |_| | '_ \\/ _` / _` | _/ -_)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: (1) Create a W&B account\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: (2) Use an existing W&B account\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: (3) Don't visualize my results\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Enter your choice: 2\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You chose 'Use an existing W&B account'\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into https://api.wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Create a new API key at: https://wandb.ai/authorize?ref=models\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Store your API key securely and do not share it.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Paste your API key and hit enter: ··········\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: No netrc file found, creating one.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmarkbaumann\u001b[0m (\u001b[33maugustinum\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 3 + } + ], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zgoc132ZMj4f" + }, + "source": [ + "## Simulate and track a machine learning experiment with W&B\n", + "\n", + "Create, track, and visualize a machine learning experiment. To do this:\n", + "\n", + "1. Initialize a [W&B run](https://docs.wandb.ai/guides/runs) and pass in the hyperparameters you want to track.\n", + "2. Within your training loop, log metrics such as the accuracy and loss." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "Fraxcnl1Mj4f", + "outputId": "4d7e36e9-7f96-4382-c455-7d67e1de39d9" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124551-1a69th37" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run experiment_0 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/basic-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/basic-intro/runs/1a69th37" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


acc▁▃▅▇▅██▇
loss█▆▃▁▂▁▂▁

Run summary:


acc0.85238
loss0.08765

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run experiment_0 at: https://wandb.ai/augustinum/basic-intro/runs/1a69th37
View project at: https://wandb.ai/augustinum/basic-intro
Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124551-1a69th37/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124557-y5tta3jl" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run experiment_1 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/basic-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/basic-intro/runs/y5tta3jl" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


acc▃▁▄▆▅█▆█
loss█▅▃▂▁▁▂▁

Run summary:


acc0.88541
loss0.13097

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run experiment_1 at: https://wandb.ai/augustinum/basic-intro/runs/y5tta3jl
View project at: https://wandb.ai/augustinum/basic-intro
Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124557-y5tta3jl/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124605-u983mpmu" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run experiment_2 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/basic-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/basic-intro/runs/u983mpmu" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


acc▁▃▄▅▆█▇█
loss█▄▃▂▁▁▁▁

Run summary:


acc0.91301
loss0.07567

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run experiment_2 at: https://wandb.ai/augustinum/basic-intro/runs/u983mpmu
View project at: https://wandb.ai/augustinum/basic-intro
Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124605-u983mpmu/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124610-fbryrqlr" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run experiment_3 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/basic-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/basic-intro/runs/fbryrqlr" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


acc▁▄▅▆▇▇█▇
loss█▃▃▃▁▁▂▁

Run summary:


acc0.92793
loss0.04179

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run experiment_3 at: https://wandb.ai/augustinum/basic-intro/runs/fbryrqlr
View project at: https://wandb.ai/augustinum/basic-intro
Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124610-fbryrqlr/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124616-u2g8ne4r" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run experiment_4 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/basic-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/basic-intro/runs/u2g8ne4r" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


acc▁▄▇▇▇▇██
loss▇█▅▂▂▁▂▁

Run summary:


acc0.80316
loss0.20552

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run experiment_4 at: https://wandb.ai/augustinum/basic-intro/runs/u2g8ne4r
View project at: https://wandb.ai/augustinum/basic-intro
Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124616-u2g8ne4r/logs" + ] + }, + "metadata": {} + } + ], + "source": [ + "import random\n", + "import math\n", + "\n", + "# Launch 5 simulated experiments\n", + "total_runs = 5\n", + "for run in range(total_runs):\n", + " # 1️. Start a new run to track this script\n", + " wandb.init(\n", + " # Set the project where this run will be logged\n", + " project=\"basic-intro\",\n", + " # We pass a run name (otherwise it’ll be randomly assigned, like sunshine-lollypop-10)\n", + " name=f\"experiment_{run}\",\n", + " # Track hyperparameters and run metadata\n", + " config={\n", + " \"learning_rate\": 0.02,\n", + " \"architecture\": \"CNN\",\n", + " \"dataset\": \"CIFAR-100\",\n", + " \"epochs\": 10,\n", + " })\n", + "\n", + " # This simple block simulates a training loop logging metrics\n", + " epochs = 10\n", + " offset = random.random() / 5\n", + " for epoch in range(2, epochs):\n", + " acc = 1 - 2 ** -epoch - random.random() / epoch - offset\n", + " loss = 2 ** -epoch + random.random() / epoch + offset\n", + "\n", + " # 2️. Log metrics from your script to W&B\n", + " wandb.log({\"acc\": acc, \"loss\": loss})\n", + "\n", + " # Mark the run as finished\n", + " wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xokA-92hMj4g" + }, + "source": [ + "View how your machine learning peformed in your W&B project. Copy and paste the URL link that is printed from the previous cell. The URL will redirect you to a W&B project that contains a dashboard showing graphs the show how\n", + "\n", + "The following image shows what a dashboard can look like:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FNjdGP2DMj4g" + }, + "source": [ + "![](https://i.imgur.com/Pell4Oo.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JXC1fKRTMj4g" + }, + "source": [ + "Now that we know how to integrate W&B into a pseudo machine learning training loop, let's track a machine learning experiment using a basic PyTorch neural network. The following code will also upload model checkpoints to W&B that you can then share with other teams in in your organization.\n", + "\n", + "## Track a machine learning experiment using PyTorch\n", + "\n", + "The following code cell defines and trains a simple MNIST classifier. During training, you will see W&B prints out URLs. Click on the project page link to see your results stream in live to a W&B project.\n", + "\n", + "W&B runs automatically log [metrics](https://docs.wandb.ai/ref/app/pages/run-page#charts-tab),\n", + "[system information](https://docs.wandb.ai/ref/app/pages/run-page#system-tab),\n", + "[hyperparameters](https://docs.wandb.ai/ref/app/pages/run-page#overview-tab),\n", + "[terminal output](https://docs.wandb.ai/ref/app/pages/run-page#logs-tab) and\n", + "you'll see an [interactive table](https://docs.wandb.ai/guides/data-vis)\n", + "with model inputs and outputs.\n", + "\n", + "### Set up PyTorch Dataloader\n", + "The following cell defines some useful functions that we will need to train our machine learning model. The functions themselves are not unique to W&B so we'll not cover them in detail here. See the PyTorch documentation for more information on how to define [forward and backward training loop](https://pytorch.org/tutorials/beginner/nn_tutorial.html), how to use [PyTorch DataLoaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html) to load data in for training, and how define PyTorch models using the [`torch.nn.Sequential` Class](https://pytorch.org/docs/stable/generated/torch.nn.Sequential.html)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "P3eQXX9WMj4g" + }, + "outputs": [], + "source": [ + "#@title\n", + "import torch, torchvision\n", + "import torch.nn as nn\n", + "from torchvision.datasets import MNIST\n", + "import torchvision.transforms as T\n", + "\n", + "MNIST.mirrors = [mirror for mirror in MNIST.mirrors if \"http://yann.lecun.com/\" not in mirror]\n", + "\n", + "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n", + "\n", + "def get_dataloader(is_train, batch_size, slice=5):\n", + " \"Get a training dataloader\"\n", + " full_dataset = MNIST(root=\".\", train=is_train, transform=T.ToTensor(), download=True)\n", + " sub_dataset = torch.utils.data.Subset(full_dataset, indices=range(0, len(full_dataset), slice))\n", + " loader = torch.utils.data.DataLoader(dataset=sub_dataset,\n", + " batch_size=batch_size,\n", + " shuffle=True if is_train else False,\n", + " pin_memory=True, num_workers=2)\n", + " return loader\n", + "\n", + "def get_model(dropout):\n", + " \"A simple model\"\n", + " model = nn.Sequential(nn.Flatten(),\n", + " nn.Linear(28*28, 256),\n", + " nn.BatchNorm1d(256),\n", + " nn.ReLU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(256,10)).to(device)\n", + " return model\n", + "\n", + "def validate_model(model, valid_dl, loss_func, log_images=False, batch_idx=0):\n", + " \"Compute performance of the model on the validation dataset and log a wandb.Table\"\n", + " model.eval()\n", + " val_loss = 0.\n", + " with torch.inference_mode():\n", + " correct = 0\n", + " for i, (images, labels) in enumerate(valid_dl):\n", + " images, labels = images.to(device), labels.to(device)\n", + "\n", + " # Forward pass ➡\n", + " outputs = model(images)\n", + " val_loss += loss_func(outputs, labels)*labels.size(0)\n", + "\n", + " # Compute accuracy and accumulate\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " correct += (predicted == labels).sum().item()\n", + "\n", + " # Log one batch of images to the dashboard, always same batch_idx.\n", + " if i==batch_idx and log_images:\n", + " log_image_table(images, predicted, labels, outputs.softmax(dim=1))\n", + " return val_loss / len(valid_dl.dataset), correct / len(valid_dl.dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GQwbYMN-Mj4g" + }, + "source": [ + "### Create a table to compare the predicted values versus the true value\n", + "\n", + "The following cell is unique to W&B, so let's go over it.\n", + "\n", + "In the cell we define a function called `log_image_table`. Though technically, optional, this function creates a W&B Table object. We will use the table object to create a table that shows what the model predicted for each image.\n", + "\n", + "More specifically, each row will conists of the image fed to the model, along with predicted value and the actual value (label)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "mWESzH8JMj4g" + }, + "outputs": [], + "source": [ + "def log_image_table(images, predicted, labels, probs):\n", + " \"Log a wandb.Table with (img, pred, target, scores)\"\n", + " # Create a wandb Table to log images, labels and predictions to\n", + " table = wandb.Table(columns=[\"image\", \"pred\", \"target\"]+[f\"score_{i}\" for i in range(10)])\n", + " for img, pred, targ, prob in zip(images.to(\"cpu\"), predicted.to(\"cpu\"), labels.to(\"cpu\"), probs.to(\"cpu\")):\n", + " table.add_data(wandb.Image(img[0].numpy()*255), pred, targ, *prob.numpy())\n", + " wandb.log({\"predictions_table\":table}, commit=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s6tNl5NIMj4g" + }, + "source": [ + "### Train your model and upload checkpoints\n", + "\n", + "The following code trains and saves model checkpoints to your project. Use model checkpoints like you normally would to assess how the model performed during training.\n", + "\n", + "W&B also makes it easy to share your saved models and model checkpoints with other members of your team or organization. To learn how to share your model and model checkpoints with members outside of your team, see [W&B Registry](https://docs.wandb.ai/guides/registry)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "N7kw0RmGMj4h", + "outputId": "c2f53bb8-c5c0-42ac-b53f-3867811953bf" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124632-fc9u3e89" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run glad-haze-1 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/pytorch-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/pytorch-intro/runs/fc9u3e89" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "100%|██████████| 9.91M/9.91M [00:02<00:00, 4.34MB/s]\n", + "100%|██████████| 28.9k/28.9k [00:00<00:00, 129kB/s]\n", + "100%|██████████| 1.65M/1.65M [00:01<00:00, 1.22MB/s]\n", + "100%|██████████| 4.54k/4.54k [00:00<00:00, 7.58MB/s]\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch: 1, Train Loss: 0.348, Valid Loss: 0.299958, Accuracy: 0.91\n", + "Epoch: 2, Train Loss: 0.220, Valid Loss: 0.247019, Accuracy: 0.93\n", + "Epoch: 3, Train Loss: 0.119, Valid Loss: 0.212555, Accuracy: 0.93\n", + "Epoch: 4, Train Loss: 0.219, Valid Loss: 0.196908, Accuracy: 0.94\n", + "Epoch: 5, Train Loss: 0.125, Valid Loss: 0.193205, Accuracy: 0.94\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


train/epoch▁▁▁▁▂▂▂▂▂▂▂▂▃▃▃▃▃▃▃▄▄▄▄▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇██
train/example_ct▁▁▁▂▂▂▂▂▂▂▂▂▃▃▃▄▄▄▄▄▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇████
train/train_loss█▄▃▃▃▃▃▃▂▂▃▃▂▂▂▂▂▂▁▂▂▂▂▁▂▂▁▂▂▂▁▂▁▁▁▂▁▁▂▁
val/val_accuracy▁▅▆██
val/val_loss█▅▂▁▁

Run summary:


test_accuracy0.8
train/epoch5
train/example_ct60000
train/train_loss0.12506
val/val_accuracy0.937
val/val_loss0.19321

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run glad-haze-1 at: https://wandb.ai/augustinum/pytorch-intro/runs/fc9u3e89
View project at: https://wandb.ai/augustinum/pytorch-intro
Synced 5 W&B file(s), 1 media file(s), 268 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124632-fc9u3e89/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124711-byc3bm8m" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run rich-pyramid-2 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/pytorch-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/pytorch-intro/runs/byc3bm8m" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch: 1, Train Loss: 0.591, Valid Loss: 0.357312, Accuracy: 0.90\n", + "Epoch: 2, Train Loss: 0.363, Valid Loss: 0.291693, Accuracy: 0.92\n", + "Epoch: 3, Train Loss: 0.312, Valid Loss: 0.265649, Accuracy: 0.92\n", + "Epoch: 4, Train Loss: 0.357, Valid Loss: 0.245993, Accuracy: 0.93\n", + "Epoch: 5, Train Loss: 0.336, Valid Loss: 0.232883, Accuracy: 0.93\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


train/epoch▁▁▂▂▂▂▂▃▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇▇█████
train/example_ct▁▁▁▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇█
train/train_loss█▄▄▄▃▃▃▃▃▂▂▂▃▂▂▂▂▂▁▂▂▂▂▂▂▂▁▂▂▂▂▁▂▁▂▂▂▂▁▁
val/val_accuracy▁▄▆▇█
val/val_loss█▄▃▂▁

Run summary:


test_accuracy0.8
train/epoch5
train/example_ct60000
train/train_loss0.3355
val/val_accuracy0.93
val/val_loss0.23288

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run rich-pyramid-2 at: https://wandb.ai/augustinum/pytorch-intro/runs/byc3bm8m
View project at: https://wandb.ai/augustinum/pytorch-intro
Synced 5 W&B file(s), 1 media file(s), 268 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124711-byc3bm8m/logs" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124739-xah8k8iw" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run prime-dream-3 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/pytorch-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/pytorch-intro/runs/xah8k8iw" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch: 1, Train Loss: 0.449, Valid Loss: 0.299326, Accuracy: 0.91\n", + "Epoch: 2, Train Loss: 0.153, Valid Loss: 0.237184, Accuracy: 0.93\n", + "Epoch: 3, Train Loss: 0.136, Valid Loss: 0.219430, Accuracy: 0.94\n", + "Epoch: 4, Train Loss: 0.381, Valid Loss: 0.193879, Accuracy: 0.94\n", + "Epoch: 5, Train Loss: 0.186, Valid Loss: 0.185918, Accuracy: 0.94\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


train/epoch▁▁▁▁▁▂▂▂▂▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇██
train/example_ct▁▁▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▇▇▇▇▇▇████
train/train_loss█▄▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▂▂▁▂▁▁▂▂▁▁▂▁▁▁▁▂▁▁▂▁▁
val/val_accuracy▁▅▆▇█
val/val_loss█▄▃▁▁

Run summary:


test_accuracy0.8
train/epoch5
train/example_ct60000
train/train_loss0.18604
val/val_accuracy0.9445
val/val_loss0.18592

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run prime-dream-3 at: https://wandb.ai/augustinum/pytorch-intro/runs/xah8k8iw
View project at: https://wandb.ai/augustinum/pytorch-intro
Synced 5 W&B file(s), 1 media file(s), 268 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124739-xah8k8iw/logs" + ] + }, + "metadata": {} + } + ], + "source": [ + "# Launch 3 experiments, trying different dropout rates\n", + "for _ in range(3):\n", + " # initialise a wandb run\n", + " wandb.init(\n", + " project=\"pytorch-intro\",\n", + " config={\n", + " \"epochs\": 5,\n", + " \"batch_size\": 128,\n", + " \"lr\": 1e-3,\n", + " \"dropout\": random.uniform(0.01, 0.80),\n", + " })\n", + "\n", + " # Copy your config\n", + " config = wandb.config\n", + "\n", + " # Get the data\n", + " train_dl = get_dataloader(is_train=True, batch_size=config.batch_size)\n", + " valid_dl = get_dataloader(is_train=False, batch_size=2*config.batch_size)\n", + " n_steps_per_epoch = math.ceil(len(train_dl.dataset) / config.batch_size)\n", + "\n", + " # A simple MLP model\n", + " model = get_model(config.dropout)\n", + "\n", + " # Make the loss and optimizer\n", + " loss_func = nn.CrossEntropyLoss()\n", + " optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)\n", + "\n", + " # Training\n", + " example_ct = 0\n", + " step_ct = 0\n", + " for epoch in range(config.epochs):\n", + " model.train()\n", + " for step, (images, labels) in enumerate(train_dl):\n", + " images, labels = images.to(device), labels.to(device)\n", + "\n", + " outputs = model(images)\n", + " train_loss = loss_func(outputs, labels)\n", + " optimizer.zero_grad()\n", + " train_loss.backward()\n", + " optimizer.step()\n", + "\n", + " example_ct += len(images)\n", + " metrics = {\"train/train_loss\": train_loss,\n", + " \"train/epoch\": (step + 1 + (n_steps_per_epoch * epoch)) / n_steps_per_epoch,\n", + " \"train/example_ct\": example_ct}\n", + "\n", + " if step + 1 < n_steps_per_epoch:\n", + " # Log train metrics to wandb\n", + " wandb.log(metrics)\n", + "\n", + " step_ct += 1\n", + "\n", + " val_loss, accuracy = validate_model(model, valid_dl, loss_func, log_images=(epoch==(config.epochs-1)))\n", + "\n", + " # Log train and validation metrics to wandb\n", + " val_metrics = {\"val/val_loss\": val_loss,\n", + " \"val/val_accuracy\": accuracy}\n", + " wandb.log({**metrics, **val_metrics})\n", + "\n", + " # Save the model checkpoint to wandb\n", + " torch.save(model, \"my_model.pt\")\n", + " wandb.log_model(\"./my_model.pt\", \"my_mnist_model\", aliases=[f\"epoch-{epoch+1}_dropout-{round(wandb.config.dropout, 4)}\"])\n", + "\n", + " print(f\"Epoch: {epoch+1}, Train Loss: {train_loss:.3f}, Valid Loss: {val_loss:3f}, Accuracy: {accuracy:.2f}\")\n", + "\n", + " # If you had a test set, this is how you could log it as a Summary metric\n", + " wandb.summary['test_accuracy'] = 0.8\n", + "\n", + " # Close your wandb run\n", + " wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h7YCrFEhMj4h" + }, + "source": [ + "You have now trained your first model using W&B. Click on one of the links above to see your metrics and see your saved model checkpoints in the Artifacts tab in the W&B App UI" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pBTlCpisMj4h" + }, + "source": [ + "## (Optional) Set up a W&B Alert\n", + "\n", + "Create a [W&B Alerts](https://docs.wandb.ai/guides/track/alert) to send alerts to your Slack or email from your Python code.\n", + "\n", + "There are 2 steps to follow the first time you'd like to send a Slack or email alert, triggered from your code:\n", + "\n", + "1) Turn on Alerts in your W&B [User Settings](https://wandb.ai/settings)\n", + "2) Add `wandb.alert()` to your code. For example:\n", + "\n", + "```python\n", + "wandb.alert(\n", + " title=\"Low accuracy\",\n", + " text=f\"Accuracy is below the acceptable threshold\"\n", + ")\n", + "```\n", + "\n", + "The following cell shows a minimal example below to see how to use `wandb.alert`" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 874 + }, + "id": "zoI-oY9EMj4h", + "outputId": "491301fb-71c8-4785-95a9-b4fb7f23b87d" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.30.0" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20260909_124806-c2i4ype4" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run effortless-mountain-4 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/augustinum/pytorch-intro" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/augustinum/pytorch-intro/runs/c2i4ype4" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Accuracy is: 1.12, 0.3\n", + "Accuracy is: 0.71, 0.3\n", + "Accuracy is: 1.213, 0.3\n", + "Accuracy is: 1.03, 0.3\n", + "Accuracy is: 0.855, 0.3\n", + "Accuracy is: 0.948, 0.3\n", + "Accuracy is: 1.223, 0.3\n", + "Accuracy is: 0.798, 0.3\n", + "Accuracy is: 0.653, 0.3\n", + "Accuracy is: 1.209, 0.3\n", + "Accuracy is: 1.296, 0.3\n", + "Accuracy is: 0.937, 0.3\n", + "Accuracy is: 1.496, 0.3\n", + "Accuracy is: 0.947, 0.3\n", + "Accuracy is: 0.916, 0.3\n", + "Accuracy is: 0.983, 0.3\n", + "Accuracy is: 1.001, 0.3\n", + "Accuracy is: 1.073, 0.3\n", + "Accuracy is: 0.954, 0.3\n", + "Accuracy is: 0.792, 0.3\n", + "Accuracy is: 1.217, 0.3\n", + "Accuracy is: 1.595, 0.3\n", + "Accuracy is: 1.835, 0.3\n", + "Accuracy is: 0.945, 0.3\n", + "Accuracy is: 0.428, 0.3\n", + "Accuracy is: 0.227, 0.3\n", + "Alert triggered\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "

Run history:


Accuracy▅▃▅▄▄▄▅▃▃▅▆▄▇▄▄▄▄▅▄▃▅▇█▄▂▁

Run summary:


Accuracy0.227

" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run effortless-mountain-4 at: https://wandb.ai/augustinum/pytorch-intro/runs/c2i4ype4
View project at: https://wandb.ai/augustinum/pytorch-intro
Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Find logs at: ./wandb/run-20260909_124806-c2i4ype4/logs" + ] + }, + "metadata": {} + } + ], + "source": [ + "# Start a wandb run\n", + "wandb.init(project=\"pytorch-intro\")\n", + "\n", + "# Simulating a model training loop\n", + "acc_threshold = 0.3\n", + "for training_step in range(1000):\n", + "\n", + " # Generate a random number for accuracy\n", + " accuracy = round(random.random() + random.random(), 3)\n", + " print(f'Accuracy is: {accuracy}, {acc_threshold}')\n", + "\n", + " # Log accuracy to wandb\n", + " wandb.log({\"Accuracy\": accuracy})\n", + "\n", + " # If the accuracy is below the threshold, fire a W&B Alert and stop the run\n", + " if accuracy <= acc_threshold:\n", + " # Send the wandb Alert\n", + " wandb.alert(\n", + " title='Low Accuracy',\n", + " text=f'Accuracy {accuracy} at step {training_step} is below the acceptable theshold, {acc_threshold}',\n", + " )\n", + " print('Alert triggered')\n", + " break\n", + "\n", + "# Mark the run as finished (useful in Jupyter notebooks)\n", + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Fw29yX4tMj4h" + }, + "source": [ + "You can find the full docs for [W&B Alerts here](https://docs.wandb.ai/guides/track/alert).\n", + "\n", + "## Next steps\n", + "The next tutorial you will learn how to do hyperparameter optimization using W&B Sweeps:\n", + "[Hyperparameters sweeps using PyTorch](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W%26B.ipynb)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [], + "toc_visible": true, + "include_colab_link": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file From 9cc1a48b8c75c7c0f80216fc264f849479ffb640 Mon Sep 17 00:00:00 2001 From: Mark Baumann Date: Wed, 9 Sep 2026 14:52:20 +0200 Subject: [PATCH 20/20] Doppelte Google Colab Buttons entfernt --- examples/handschrifterkennung.ipynb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/examples/handschrifterkennung.ipynb b/examples/handschrifterkennung.ipynb index 9e5e1261b..330f47bc2 100644 --- a/examples/handschrifterkennung.ipynb +++ b/examples/handschrifterkennung.ipynb @@ -10,26 +10,6 @@ "\"Open" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "JlSR4EyvMj4c" - }, - "source": [ - "\"Open\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GllrnZFfMj4d" - }, - "source": [ - "\"Open\n", - "" - ] - }, { "cell_type": "markdown", "metadata": { @@ -1698,4 +1678,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +}