From e645eab5c5a51be5fa5284f3e25b35bc8407bdc8 Mon Sep 17 00:00:00 2001 From: zslchase <1833842212@qq.com> Date: Thu, 8 Jan 2026 01:14:25 +0000 Subject: [PATCH 01/81] fix import --- .github/agents/sage-apps.agent.md | 346 ++++ .github/copilot-instructions.md | 575 +++++++ .vscode/settings.json | 5 + README.md | 238 +++ __init__.py | 12 + apps/README.md | 136 ++ apps/__init__.py | 7 + apps/demo_feature_extraction.py | 160 ++ apps/run_article_monitoring.py | 136 ++ apps/run_auto_scaling_chat.py | 118 ++ apps/run_medical_diagnosis.py | 196 +++ apps/run_smart_home.py | 94 + apps/run_video_intelligence.py | 124 ++ apps/run_work_report.py | 240 +++ data | 1 + requirements.txt | 42 + sage-apps/README.md | 217 +++ sage-apps/pyproject.toml | 184 ++ sage-apps/src/sage/apps/__init__.py | 29 + sage-apps/src/sage/apps/_version.py | 6 + .../sage/apps/article_monitoring/README.md | 176 ++ .../sage/apps/article_monitoring/__init__.py | 20 + .../sage/apps/article_monitoring/operators.py | 328 ++++ .../sage/apps/article_monitoring/pipeline.py | 189 +++ .../src/sage/apps/auto_scaling_chat/README.md | 193 +++ .../sage/apps/auto_scaling_chat/__init__.py | 20 + .../sage/apps/auto_scaling_chat/operators.py | 318 ++++ .../sage/apps/auto_scaling_chat/pipeline.py | 170 ++ .../src/sage/apps/medical_diagnosis/README.md | 220 +++ .../sage/apps/medical_diagnosis/__init__.py | 44 + .../apps/medical_diagnosis/agents/__init__.py | 12 + .../agents/diagnostic_agent.py | 235 +++ .../agents/image_analyzer.py | 1121 ++++++++++++ .../agents/report_generator.py | 497 ++++++ .../config/agent_config.yaml | 126 ++ .../apps/medical_diagnosis/run_diagnosis.py | 332 ++++ .../scripts/download_lumbar_dataset.py | 73 + .../scripts/explore_dataset.py | 135 ++ .../medical_diagnosis/scripts/prepare_data.py | 308 ++++ .../sage/apps/medical_diagnosis/setup_data.sh | 98 ++ .../apps/medical_diagnosis/tools/__init__.py | 5 + .../medical_diagnosis/tools/knowledge_base.py | 643 +++++++ sage-apps/src/sage/apps/py.typed | 0 sage-apps/src/sage/apps/smart_home/README.md | 235 +++ .../src/sage/apps/smart_home/__init__.py | 20 + .../src/sage/apps/smart_home/operators.py | 263 +++ .../src/sage/apps/smart_home/pipeline.py | 131 ++ sage-apps/src/sage/apps/video/__init__.py | 7 + .../apps/video/config/default_config.yaml | 63 + .../src/sage/apps/video/operators/__init__.py | 25 + .../sage/apps/video/operators/analytics.py | 241 +++ .../sage/apps/video/operators/formatters.py | 19 + .../sage/apps/video/operators/integrations.py | 178 ++ .../sage/apps/video/operators/perception.py | 190 +++ .../apps/video/operators/preprocessing.py | 34 + .../src/sage/apps/video/operators/sinks.py | 125 ++ .../src/sage/apps/video/operators/sources.py | 73 + .../apps/video/video_intelligence_pipeline.py | 587 +++++++ .../sage/apps/work_report_generator/README.md | 286 ++++ .../apps/work_report_generator/__init__.py | 20 + .../sage/apps/work_report_generator/models.py | 292 ++++ .../apps/work_report_generator/operators.py | 1507 +++++++++++++++++ .../apps/work_report_generator/pipeline.py | 359 ++++ sage-apps/tests/__init__.py | 1 + .../tests/article_monitoring/__init__.py | 1 + .../article_monitoring/test_operators.py | 322 ++++ sage-apps/tests/auto_scaling_chat/__init__.py | 1 + .../tests/auto_scaling_chat/test_operators.py | 294 ++++ sage-apps/tests/medical_diagnosis/__init__.py | 1 + sage-apps/tests/medical_diagnosis/conftest.py | 72 + .../demo_knowledge_loading.py | 250 +++ .../tests/medical_diagnosis/test_diagnosis.py | 218 +++ .../medical_diagnosis/test_image_analyzer.py | 507 ++++++ .../medical_diagnosis/test_image_quality.py | 187 ++ .../medical_diagnosis/test_knowledge_base.py | 409 +++++ sage-apps/tests/smart_home/__init__.py | 1 + sage-apps/tests/smart_home/test_operators.py | 377 +++++ sage-apps/tests/test_medical_diagnosis.py | 136 ++ sage-apps/tests/test_package.py | 17 + sage-apps/tests/test_video_app.py | 160 ++ .../tests/work_report_generator/__init__.py | 1 + .../work_report_generator/test_models.py | 296 ++++ .../work_report_generator/test_operators.py | 638 +++++++ tutorials/L1-common/README.md | 92 + .../L1-common/embedding_server_example.py | 224 +++ tutorials/L1-common/hello_world.py | 47 + .../unified_inference_client_example.py | 281 +++ tutorials/L2-platform/README.md | 48 + tutorials/L2-platform/environment/README.md | 28 + .../L2-platform/environment/remote_env.py | 290 ++++ tutorials/L3-kernel/README.md | 85 + .../checkpoint_recovery_test.py | 199 +++ .../fault_tolerance/fault_tolerance.py | 227 +++ .../fault_tolerance_examples.py | 412 +++++ .../advanced/hello_future_stream_example.py | 193 +++ .../L3-kernel/advanced/hello_future_world.py | 118 ++ .../hello_realistic_service_example.py | 506 ++++++ .../advanced/parallelism_local_validation.py | 283 ++++ .../advanced/parallelism_remote_validation.py | 326 ++++ .../advanced/pipeline_as_service/README.md | 65 + .../async_client_pipeline_as_service.py | 178 ++ .../hello_pipeline_as_service.py | 315 ++++ .../multi_client_pipeline_as_service.py | 156 ++ .../pipeline_as_service/pipeline_bridge.py | 66 + .../qa_pipeline_as_service.py | 638 +++++++ .../advanced/simple_parallelism_validation.py | 309 ++++ .../batch/hello_batch_operator_examples.py | 274 +++ .../batch/hello_batch_vs_source_comparison.py | 208 +++ .../L3-kernel/batch/hello_local_batch.py | 293 ++++ .../L3-kernel/batch/hello_remote_batch.py | 372 ++++ tutorials/L3-kernel/cpu_node_demo.py | 589 +++++++ .../functions/hello_comap_function_example.py | 220 +++ .../functions/hello_comap_lambda_example.py | 351 ++++ .../hello_wordcount_batch_example.py | 69 + .../hello_wordcount_lambda_example.py | 104 ++ tutorials/L3-kernel/keyed_state_example.py | 403 +++++ .../L3-kernel/operators/hello_comap_world.py | 65 + .../L3-kernel/operators/hello_filter_world.py | 54 + .../operators/hello_flatmap_world.py | 52 + .../L3-kernel/operators/hello_join_world.py | 104 ++ .../operators/hello_three_input_comap.py | 107 ++ .../stream/hello_connected_stream_example.py | 102 ++ .../L3-kernel/stream/hello_onebyone_world.py | 47 + .../L3-kernel/stream/hello_streaming_world.py | 53 + .../stream/hello_wordcount_source_example.py | 77 + tutorials/L3-libs/README.md | 87 + tutorials/L3-libs/agent_sft_demo.py | 179 ++ tutorials/L3-libs/agents/README.md | 34 + tutorials/L3-libs/agents/__init__.py | 1 + tutorials/L3-libs/agents/arxiv_search_tool.py | 239 +++ tutorials/L3-libs/agents/basic_agent.py | 205 +++ tutorials/L3-libs/agents/config/README.md | 24 + .../agents/config/config_agent_min.yaml | 93 + tutorials/L3-libs/agents/data/README.md | 27 + .../L3-libs/agents/data/agent_queries.jsonl | 4 + .../agents/data/agent_queries_test.jsonl | 1 + tutorials/L3-libs/agents/demo_arxiv_search.py | 179 ++ tutorials/L3-libs/agents/tool_use_pipeline.py | 676 ++++++++ tutorials/L3-libs/agents/workflow_demo.py | 260 +++ tutorials/L3-libs/amms_example.py | 116 ++ tutorials/L3-libs/embeddings/README.md | 26 + .../L3-libs/embeddings/cross_modal_search.py | 101 ++ .../L3-libs/embeddings/embedding_demo.py | 260 +++ .../embeddings/embedding_service_demo.py | 448 +++++ .../pipeline_builder_embedding_demo.py | 232 +++ tutorials/L3-libs/embeddings/quickstart.py | 103 ++ tutorials/L3-libs/llm/demo_new_templates.py | 183 ++ .../L3-libs/llm/pipeline_builder_llm_demo.py | 260 +++ .../L3-libs/llm/templates_to_llm_demo.py | 240 +++ tutorials/L3-libs/rag/README.md | 37 + tutorials/L3-libs/rag/adaptive_rag_v2.py | 676 ++++++++ .../L3-libs/rag/advanced_rag_topology.py | 779 +++++++++ tutorials/L3-libs/rag/qa_local_llm.py | 143 ++ tutorials/L3-libs/rag/qa_no_retrieval.py | 135 ++ tutorials/L3-libs/rag/simple_rag.py | 305 ++++ .../L3-libs/rag/usage_1_direct_library.py | 279 +++ .../L3-libs/rag/usage_2_sage_function.py | 226 +++ .../L3-libs/rag/usage_3_memory_service.py | 472 ++++++ tutorials/L3-libs/rag/usage_4_complete_rag.py | 582 +++++++ tutorials/L3-libs/unlearning/README.md | 71 + tutorials/L3-libs/unlearning/__init__.py | 6 + .../unlearning/basic_unlearning_demo.py | 143 ++ .../unlearning/machine_unlearning_examples.py | 366 ++++ tutorials/L4-middleware/README.md | 65 + .../L4-middleware/hello_service_world.py | 51 + tutorials/L4-middleware/rag/examples.py | 270 +++ tutorials/L4-middleware/sage_db/README.md | 42 + .../sage_db/hello_sage_db_app.py | 69 + .../sage_db/hello_sage_db_service.py | 70 + .../L4-middleware/sage_db/workflow_demo.py | 349 ++++ tutorials/L4-middleware/sage_flow/README.md | 38 + .../sage_flow/hello_sage_flow_app.py | 97 ++ .../sage_flow/hello_sage_flow_service.py | 105 ++ tutorials/L4-middleware/sage_tsdb/README.md | 262 +++ tutorials/L4-middleware/sage_tsdb/__init__.py | 9 + .../sage_tsdb/advanced_dag_example.py | 265 +++ .../sage_tsdb/basic_dag_example.py | 211 +++ .../sage_tsdb/stream_join_dag_example.py | 202 +++ tutorials/L5-apps/README.md | 34 + tutorials/L6-interface/README.md | 174 ++ tutorials/QUICK_START.md | 320 ++++ tutorials/README.md | 348 ++++ tutorials/__init__.py | 9 + tutorials/benchmark_control_plane_demo.py | 484 ++++++ tutorials/config/README.md | 266 +++ tutorials/config/config.yaml | 47 + tutorials/docs/QUICK_REFERENCE.md | 318 ++++ tutorials/docs/TROUBLESHOOTING.md | 655 +++++++ tutorials/embedding_server_example.py | 147 ++ tutorials/hello_world.py | 47 + tutorials/l6/memory_backend_demo.py | 111 ++ .../vllm_control_plane_config_examples.md | 253 +++ tutorials/vllm_control_plane_tutorial.py | 160 ++ 193 files changed, 38979 insertions(+) create mode 100644 .github/agents/sage-apps.agent.md create mode 100644 .github/copilot-instructions.md create mode 100644 .vscode/settings.json create mode 100644 README.md create mode 100644 __init__.py create mode 100644 apps/README.md create mode 100644 apps/__init__.py create mode 100644 apps/demo_feature_extraction.py create mode 100644 apps/run_article_monitoring.py create mode 100644 apps/run_auto_scaling_chat.py create mode 100644 apps/run_medical_diagnosis.py create mode 100644 apps/run_smart_home.py create mode 100644 apps/run_video_intelligence.py create mode 100644 apps/run_work_report.py create mode 120000 data create mode 100644 requirements.txt create mode 100644 sage-apps/README.md create mode 100644 sage-apps/pyproject.toml create mode 100644 sage-apps/src/sage/apps/__init__.py create mode 100644 sage-apps/src/sage/apps/_version.py create mode 100644 sage-apps/src/sage/apps/article_monitoring/README.md create mode 100644 sage-apps/src/sage/apps/article_monitoring/__init__.py create mode 100644 sage-apps/src/sage/apps/article_monitoring/operators.py create mode 100644 sage-apps/src/sage/apps/article_monitoring/pipeline.py create mode 100644 sage-apps/src/sage/apps/auto_scaling_chat/README.md create mode 100644 sage-apps/src/sage/apps/auto_scaling_chat/__init__.py create mode 100644 sage-apps/src/sage/apps/auto_scaling_chat/operators.py create mode 100644 sage-apps/src/sage/apps/auto_scaling_chat/pipeline.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/README.md create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/__init__.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/agents/__init__.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/agents/diagnostic_agent.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/agents/image_analyzer.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/agents/report_generator.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/config/agent_config.yaml create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/run_diagnosis.py create mode 100755 sage-apps/src/sage/apps/medical_diagnosis/scripts/download_lumbar_dataset.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/scripts/explore_dataset.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/scripts/prepare_data.py create mode 100755 sage-apps/src/sage/apps/medical_diagnosis/setup_data.sh create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/tools/__init__.py create mode 100644 sage-apps/src/sage/apps/medical_diagnosis/tools/knowledge_base.py create mode 100644 sage-apps/src/sage/apps/py.typed create mode 100644 sage-apps/src/sage/apps/smart_home/README.md create mode 100644 sage-apps/src/sage/apps/smart_home/__init__.py create mode 100644 sage-apps/src/sage/apps/smart_home/operators.py create mode 100644 sage-apps/src/sage/apps/smart_home/pipeline.py create mode 100644 sage-apps/src/sage/apps/video/__init__.py create mode 100644 sage-apps/src/sage/apps/video/config/default_config.yaml create mode 100644 sage-apps/src/sage/apps/video/operators/__init__.py create mode 100644 sage-apps/src/sage/apps/video/operators/analytics.py create mode 100644 sage-apps/src/sage/apps/video/operators/formatters.py create mode 100644 sage-apps/src/sage/apps/video/operators/integrations.py create mode 100644 sage-apps/src/sage/apps/video/operators/perception.py create mode 100644 sage-apps/src/sage/apps/video/operators/preprocessing.py create mode 100644 sage-apps/src/sage/apps/video/operators/sinks.py create mode 100644 sage-apps/src/sage/apps/video/operators/sources.py create mode 100644 sage-apps/src/sage/apps/video/video_intelligence_pipeline.py create mode 100644 sage-apps/src/sage/apps/work_report_generator/README.md create mode 100644 sage-apps/src/sage/apps/work_report_generator/__init__.py create mode 100644 sage-apps/src/sage/apps/work_report_generator/models.py create mode 100644 sage-apps/src/sage/apps/work_report_generator/operators.py create mode 100644 sage-apps/src/sage/apps/work_report_generator/pipeline.py create mode 100644 sage-apps/tests/__init__.py create mode 100644 sage-apps/tests/article_monitoring/__init__.py create mode 100644 sage-apps/tests/article_monitoring/test_operators.py create mode 100644 sage-apps/tests/auto_scaling_chat/__init__.py create mode 100644 sage-apps/tests/auto_scaling_chat/test_operators.py create mode 100644 sage-apps/tests/medical_diagnosis/__init__.py create mode 100644 sage-apps/tests/medical_diagnosis/conftest.py create mode 100644 sage-apps/tests/medical_diagnosis/demo_knowledge_loading.py create mode 100644 sage-apps/tests/medical_diagnosis/test_diagnosis.py create mode 100644 sage-apps/tests/medical_diagnosis/test_image_analyzer.py create mode 100644 sage-apps/tests/medical_diagnosis/test_image_quality.py create mode 100644 sage-apps/tests/medical_diagnosis/test_knowledge_base.py create mode 100644 sage-apps/tests/smart_home/__init__.py create mode 100644 sage-apps/tests/smart_home/test_operators.py create mode 100644 sage-apps/tests/test_medical_diagnosis.py create mode 100644 sage-apps/tests/test_package.py create mode 100644 sage-apps/tests/test_video_app.py create mode 100644 sage-apps/tests/work_report_generator/__init__.py create mode 100644 sage-apps/tests/work_report_generator/test_models.py create mode 100644 sage-apps/tests/work_report_generator/test_operators.py create mode 100644 tutorials/L1-common/README.md create mode 100644 tutorials/L1-common/embedding_server_example.py create mode 100644 tutorials/L1-common/hello_world.py create mode 100644 tutorials/L1-common/unified_inference_client_example.py create mode 100644 tutorials/L2-platform/README.md create mode 100644 tutorials/L2-platform/environment/README.md create mode 100644 tutorials/L2-platform/environment/remote_env.py create mode 100644 tutorials/L3-kernel/README.md create mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/checkpoint_recovery_test.py create mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance.py create mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance_examples.py create mode 100644 tutorials/L3-kernel/advanced/hello_future_stream_example.py create mode 100644 tutorials/L3-kernel/advanced/hello_future_world.py create mode 100644 tutorials/L3-kernel/advanced/hello_realistic_service_example.py create mode 100644 tutorials/L3-kernel/advanced/parallelism_local_validation.py create mode 100644 tutorials/L3-kernel/advanced/parallelism_remote_validation.py create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/README.md create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/async_client_pipeline_as_service.py create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/hello_pipeline_as_service.py create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/multi_client_pipeline_as_service.py create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/pipeline_bridge.py create mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/qa_pipeline_as_service.py create mode 100644 tutorials/L3-kernel/advanced/simple_parallelism_validation.py create mode 100644 tutorials/L3-kernel/batch/hello_batch_operator_examples.py create mode 100644 tutorials/L3-kernel/batch/hello_batch_vs_source_comparison.py create mode 100644 tutorials/L3-kernel/batch/hello_local_batch.py create mode 100644 tutorials/L3-kernel/batch/hello_remote_batch.py create mode 100644 tutorials/L3-kernel/cpu_node_demo.py create mode 100644 tutorials/L3-kernel/functions/hello_comap_function_example.py create mode 100644 tutorials/L3-kernel/functions/hello_comap_lambda_example.py create mode 100644 tutorials/L3-kernel/functions/hello_wordcount_batch_example.py create mode 100644 tutorials/L3-kernel/functions/hello_wordcount_lambda_example.py create mode 100644 tutorials/L3-kernel/keyed_state_example.py create mode 100644 tutorials/L3-kernel/operators/hello_comap_world.py create mode 100644 tutorials/L3-kernel/operators/hello_filter_world.py create mode 100644 tutorials/L3-kernel/operators/hello_flatmap_world.py create mode 100644 tutorials/L3-kernel/operators/hello_join_world.py create mode 100644 tutorials/L3-kernel/operators/hello_three_input_comap.py create mode 100644 tutorials/L3-kernel/stream/hello_connected_stream_example.py create mode 100644 tutorials/L3-kernel/stream/hello_onebyone_world.py create mode 100644 tutorials/L3-kernel/stream/hello_streaming_world.py create mode 100644 tutorials/L3-kernel/stream/hello_wordcount_source_example.py create mode 100644 tutorials/L3-libs/README.md create mode 100644 tutorials/L3-libs/agent_sft_demo.py create mode 100644 tutorials/L3-libs/agents/README.md create mode 100644 tutorials/L3-libs/agents/__init__.py create mode 100644 tutorials/L3-libs/agents/arxiv_search_tool.py create mode 100644 tutorials/L3-libs/agents/basic_agent.py create mode 100644 tutorials/L3-libs/agents/config/README.md create mode 100644 tutorials/L3-libs/agents/config/config_agent_min.yaml create mode 100644 tutorials/L3-libs/agents/data/README.md create mode 100644 tutorials/L3-libs/agents/data/agent_queries.jsonl create mode 100644 tutorials/L3-libs/agents/data/agent_queries_test.jsonl create mode 100644 tutorials/L3-libs/agents/demo_arxiv_search.py create mode 100644 tutorials/L3-libs/agents/tool_use_pipeline.py create mode 100644 tutorials/L3-libs/agents/workflow_demo.py create mode 100644 tutorials/L3-libs/amms_example.py create mode 100644 tutorials/L3-libs/embeddings/README.md create mode 100644 tutorials/L3-libs/embeddings/cross_modal_search.py create mode 100644 tutorials/L3-libs/embeddings/embedding_demo.py create mode 100644 tutorials/L3-libs/embeddings/embedding_service_demo.py create mode 100644 tutorials/L3-libs/embeddings/pipeline_builder_embedding_demo.py create mode 100644 tutorials/L3-libs/embeddings/quickstart.py create mode 100644 tutorials/L3-libs/llm/demo_new_templates.py create mode 100644 tutorials/L3-libs/llm/pipeline_builder_llm_demo.py create mode 100644 tutorials/L3-libs/llm/templates_to_llm_demo.py create mode 100644 tutorials/L3-libs/rag/README.md create mode 100644 tutorials/L3-libs/rag/adaptive_rag_v2.py create mode 100644 tutorials/L3-libs/rag/advanced_rag_topology.py create mode 100644 tutorials/L3-libs/rag/qa_local_llm.py create mode 100644 tutorials/L3-libs/rag/qa_no_retrieval.py create mode 100644 tutorials/L3-libs/rag/simple_rag.py create mode 100644 tutorials/L3-libs/rag/usage_1_direct_library.py create mode 100644 tutorials/L3-libs/rag/usage_2_sage_function.py create mode 100644 tutorials/L3-libs/rag/usage_3_memory_service.py create mode 100644 tutorials/L3-libs/rag/usage_4_complete_rag.py create mode 100644 tutorials/L3-libs/unlearning/README.md create mode 100644 tutorials/L3-libs/unlearning/__init__.py create mode 100644 tutorials/L3-libs/unlearning/basic_unlearning_demo.py create mode 100644 tutorials/L3-libs/unlearning/machine_unlearning_examples.py create mode 100644 tutorials/L4-middleware/README.md create mode 100644 tutorials/L4-middleware/hello_service_world.py create mode 100644 tutorials/L4-middleware/rag/examples.py create mode 100644 tutorials/L4-middleware/sage_db/README.md create mode 100644 tutorials/L4-middleware/sage_db/hello_sage_db_app.py create mode 100644 tutorials/L4-middleware/sage_db/hello_sage_db_service.py create mode 100644 tutorials/L4-middleware/sage_db/workflow_demo.py create mode 100644 tutorials/L4-middleware/sage_flow/README.md create mode 100644 tutorials/L4-middleware/sage_flow/hello_sage_flow_app.py create mode 100644 tutorials/L4-middleware/sage_flow/hello_sage_flow_service.py create mode 100644 tutorials/L4-middleware/sage_tsdb/README.md create mode 100644 tutorials/L4-middleware/sage_tsdb/__init__.py create mode 100644 tutorials/L4-middleware/sage_tsdb/advanced_dag_example.py create mode 100644 tutorials/L4-middleware/sage_tsdb/basic_dag_example.py create mode 100644 tutorials/L4-middleware/sage_tsdb/stream_join_dag_example.py create mode 100644 tutorials/L5-apps/README.md create mode 100644 tutorials/L6-interface/README.md create mode 100644 tutorials/QUICK_START.md create mode 100644 tutorials/README.md create mode 100644 tutorials/__init__.py create mode 100644 tutorials/benchmark_control_plane_demo.py create mode 100644 tutorials/config/README.md create mode 100644 tutorials/config/config.yaml create mode 100644 tutorials/docs/QUICK_REFERENCE.md create mode 100644 tutorials/docs/TROUBLESHOOTING.md create mode 100644 tutorials/embedding_server_example.py create mode 100644 tutorials/hello_world.py create mode 100644 tutorials/l6/memory_backend_demo.py create mode 100644 tutorials/vllm_control_plane_config_examples.md create mode 100644 tutorials/vllm_control_plane_tutorial.py diff --git a/.github/agents/sage-apps.agent.md b/.github/agents/sage-apps.agent.md new file mode 100644 index 0000000..718c047 --- /dev/null +++ b/.github/agents/sage-apps.agent.md @@ -0,0 +1,346 @@ +--- +description: 'SAGE Apps Development Agent - Specialized for developing production-ready applications using SAGE framework' +tools: ['vscode', 'execute', 'read', 'edit', 'search', 'web', 'agent', 'ms-python.python/getPythonEnvironmentInfo', 'ms-python.python/getPythonExecutableCommand', 'ms-python.python/installPythonPackage', 'ms-python.python/configurePythonEnvironment', 'todo'] +--- + +# SAGE Apps Development Agent + +## Purpose + +This agent specializes in developing and maintaining the **sage-apps** package and application examples in the sage-examples repository. It understands SAGE's 6-layer architecture, follows project-specific coding principles, and helps create production-ready AI applications. + +## When to Use This Agent + +✅ **Use this agent for:** +- Developing new applications in `apps/` or `sage-apps/src/sage/apps/` +- Creating production-ready examples (video intelligence, medical diagnosis, smart home, etc.) +- Writing tests for sage-apps in `sage-apps/tests/` +- Debugging application-level issues +- Implementing L5 (Applications) layer features +- Working with SAGE dependencies (isage-common, isage-llm-core, isage-libs, etc.) +- Adding new application examples to the repository + +❌ **Do NOT use this agent for:** +- Core framework development (belongs to SAGE main repository) +- Tutorial examples (use general Copilot for `tutorials/` directory) +- Infrastructure or CI/CD changes +- Benchmark development (belongs to sage-benchmark repository) + +## Core Principles (CRITICAL) + +### 🚨 Principle 1: NO MANUAL PIP INSTALL + +**All dependencies MUST be declared in pyproject.toml or requirements.txt** + +❌ **FORBIDDEN:** +```bash +pip install transformers +pip install torch==2.7.0 +pip install sage-libs +``` + +✅ **CORRECT:** +```toml +# In pyproject.toml or requirements.txt +dependencies = [ + "isage-libs>=0.2.0", + "isage-llm-core>=0.2.0", + "transformers>=4.52.0", +] +``` + +Then install with: `pip install -e .` or `pip install -r requirements.txt` + +### 🚨 Principle 2: NO FALLBACK LOGIC + +**Never use try-except fallback patterns. Let errors propagate with clear messages.** + +❌ **BAD:** +```python +try: + from ._version import __version__ +except ImportError: + __version__ = "unknown" # Hides missing file + +config = os.getenv("API_KEY") or "default_key" # Hides missing env var +``` + +✅ **GOOD:** +```python +# Let exceptions propagate +config = load_config("config.yaml") # FileNotFoundError if missing +api_key = os.environ["API_KEY"] # KeyError if missing + +# Or provide helpful error messages +if not os.path.exists("config.yaml"): + raise FileNotFoundError( + "config.yaml not found. Please create it from config.yaml.template" + ) +``` + +**Rationale:** Fail fast with clear errors. Silent fallbacks hide bugs and make debugging harder. + +## SAGE Architecture Knowledge + +### Layer Structure (L1-L6) + +``` +L6: sage-cli, sage-studio, sage-tools, sage-llm-gateway, sage-edge +L5: sage-apps ← THIS AGENT'S PRIMARY FOCUS +L4: sage-middleware +L3: sage-kernel, sage-libs +L2: sage-platform +L1: sage-common, sage-llm-core +``` + +### Dependencies + +**Core dependencies for sage-apps:** +```toml +dependencies = [ + "isage-common>=0.2.0", # L1: Foundation + "isage-llm-core>=0.2.0", # L1: LLM control plane + client + "isage-libs>=0.2.0", # L3: Algorithms, RAG, Agents +] + +# Optional for advanced features +[project.optional-dependencies] +middleware = ["isage-middleware>=0.2.0"] +full = [ + "isage-middleware>=0.2.0", + "isage-vdb>=0.2.0", + "isage-benchmark>=0.2.0", +] +``` + +**Note:** All packages use 'isage-' prefix on PyPI (since 'sage' is taken). + +## Workflow + +### Before Making Changes + +1. **Read documentation first** - Don't guess: + - `apps/README.md` - Application overview + - `sage-apps/README.md` - Package structure + - Root `README.md` - Project overview + - Use `grep_search` or `semantic_search` to find relevant docs + +2. **Check existing examples:** + - Look at similar apps in `apps/` + - Review tests in `sage-apps/tests/` + - Follow established patterns + +3. **Plan with TODO list:** + - Use `manage_todo_list` for multi-step work + - Break down complex tasks + - Mark progress as you go + +### Development Steps + +1. **Setup:** + - Configure Python environment with `configure_python_environment` + - Verify dependencies in `pyproject.toml` + - Check `.env` for required API keys + +2. **Implementation:** + - Follow code style (ruff formatting) + - Add type hints + - Write clear docstrings + - No fallback logic! + +3. **Testing:** + - Write tests in `sage-apps/tests/` + - Run with `pytest tests/` + - Test both success and failure cases + +4. **Documentation:** + - Update README if adding new app + - Add inline comments for complex logic + - Provide usage examples + +### Example Application Template + +```python +""" +[App Name] - Brief description + +This application demonstrates [key features]. + +Requirements: + - isage-libs>=0.2.0 + - [other dependencies] + +Environment: + OPENAI_API_KEY: Required for LLM inference + [other env vars] + +Usage: + python apps/run_[app_name].py +""" + +from sage.llm import UnifiedInferenceClient +from sage.libs.rag import SimpleRAG +from sage.libs.agents import Agent + + +def main(): + """Main application entry point.""" + # Step 1: Setup (no fallbacks!) + client = UnifiedInferenceClient.create() + + # Step 2: Core logic + agent = Agent(client=client) + response = agent.run("Your task") + + # Step 3: Display results + print(f"Result: {response}") + + +if __name__ == "__main__": + main() +``` + +## Input/Output Expectations + +### Ideal Inputs from User + +- Clear description of the application to build/modify +- Target use case and requirements +- Specific files or features to work on +- API keys or configuration details (if needed) + +### Expected Outputs + +- Working application code in `apps/` or `sage-apps/src/` +- Comprehensive tests in `sage-apps/tests/` +- Updated README with usage instructions +- Code following ruff style guidelines +- No manual dependency installation commands + +## Tools This Agent May Call + +### Core Development Tools +- `read_file` - Read source code and docs +- `replace_string_in_file` / `multi_replace_string_in_file` - Edit code +- `create_file` - Create new applications or tests +- `grep_search` / `semantic_search` - Find relevant code/docs + +### Python Environment +- `configure_python_environment` - Setup Python env +- `get_python_environment_details` - Check installed packages +- `install_python_packages` - Install dependencies (via pyproject.toml) +- `mcp_pylance_mcp_s_pylanceRunCodeSnippet` - Test code snippets + +### Testing & Validation +- `run_in_terminal` - Run tests, execute apps +- `get_errors` - Check for linting/type errors +- `mcp_pylance_mcp_s_pylanceFileSyntaxErrors` - Validate syntax + +### Planning & Progress +- `manage_todo_list` - Track multi-step work +- `list_dir` / `file_search` - Explore project structure + +## Reporting Progress + +### For Simple Tasks +- Briefly confirm completion: "Added new smart home sensor handler" +- No verbose explanations needed + +### For Complex Tasks +1. **Start:** Create TODO list showing planned steps +2. **During:** Mark tasks in-progress, provide brief updates +3. **Complete:** Mark all tasks done, summarize what was built + +### When Asking for Help +- **Missing info:** "Need API key for testing. Please set OPENAI_API_KEY in .env" +- **Ambiguous:** "Found 3 similar apps. Should this use RAG, agents, or both?" +- **Errors:** "Test failing with [error]. Need clarification on expected behavior" + +## Edge Cases & Boundaries + +### What This Agent WON'T Do + +❌ Modify core SAGE framework packages (L1-L4 internals) +❌ Change CI/CD configurations or GitHub workflows +❌ Create tutorials (belongs to `tutorials/` with general Copilot) +❌ Develop benchmark tools (belongs to sage-benchmark repo) +❌ Use manual `pip install` commands +❌ Add silent try-except fallbacks + +### What This Agent WILL Do + +✅ Build production applications using SAGE APIs +✅ Write comprehensive tests for applications +✅ Follow SAGE architecture best practices +✅ Use proper dependency management +✅ Provide clear error messages +✅ Document code and usage patterns + +## Common Application Patterns + +### 1. RAG Applications +```python +from sage.libs.rag import SimpleRAG + +rag = SimpleRAG( + embedding_client=embedding_client, + llm_client=llm_client +) +response = rag.query("What is SAGE?") +``` + +### 2. Agent Applications +```python +from sage.libs.agents import Agent + +agent = Agent(client=llm_client, tools=tools) +result = agent.run("Analyze this medical image") +``` + +### 3. Batch Processing +```python +from sage.kernel.operators import BatchOperator + +class DataProcessor(BatchOperator): + def process_batch(self, batch): + # Process data + return results +``` + +## Environment Setup Requirements + +### Required Files +- `.env` - API keys and configuration (from `.env.template`) +- `pyproject.toml` - Package dependencies +- `requirements.txt` - Alternative dependency file + +### Required Environment Variables +```bash +# Minimal for most apps +OPENAI_API_KEY=sk-xxx + +# Optional for local SAGE services +SAGE_CHAT_BASE_URL=http://localhost:8889/v1 +SAGE_CHAT_MODEL=Qwen/Qwen2.5-7B-Instruct + +# Optional for cloud services +# SAGE_CHAT_API_KEY=sk-xxx +# SAGE_CHAT_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +## Resources + +- **Main repo:** https://github.com/intellistream/SAGE +- **Examples repo:** https://github.com/intellistream/sage-examples +- **Documentation:** https://sage.intellistream.com +- **PyPI packages:** https://pypi.org/search/?q=isage + +## Final Reminder + +**Trust the documentation.** Before coding: +1. Read `apps/README.md` and `sage-apps/README.md` +2. Search for similar examples in `apps/` directory +3. Check test patterns in `sage-apps/tests/` +4. Follow the two core principles: NO manual pip install, NO fallback logic + +When in doubt, search the docs first. They exist to guide you. \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..44ff243 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,575 @@ +# SAGE Examples Copilot Instructions + +## Overview + +**sage-examples** 是 SAGE 框架的示例代码仓库，包含完整的教程和应用案例。这个仓库从 SAGE 主仓库独立出来，专注于提供学习资源和生产应用示例。 + +## 🚨 CRITICAL Principles + +### ❌ NEVER MANUAL PIP INSTALL - ALWAYS USE pyproject.toml + +**所有依赖必须声明在 pyproject.toml 中，禁止使用手动 `pip install` 命令。** + +#### ❌ FORBIDDEN Operations: + +```bash +pip install transformers # ❌ 手动安装 +pip install torch==2.7.0 # ❌ 手动版本 +pip install sage-libs # ❌ 手动依赖 +``` + +#### ✅ CORRECT Operations: + +```toml +# 在 pyproject.toml 或 requirements.txt 中声明 +dependencies = [ + "isage-libs>=0.2.0", # ✅ 声明在 pyproject.toml + "isage-llm-core>=0.2.0", # ✅ 版本约束 + "transformers>=4.52.0", # ✅ 可选依赖 +] +``` + +```bash +# 然后安装包 +pip install -e . +# 或 +pip install -r requirements.txt +``` + +**Why**: 确保可重现性，在 git 中跟踪依赖变更，防止版本冲突，维护单一信息源。 + +### ❌ NO FALLBACK LOGIC - PROJECT-WIDE RULE + +**禁止在代码的任何地方使用 try-except fallback 模式。** + +这是**项目级原则**，不仅适用于版本管理。Fallback 会隐藏问题，使调试更加困难。 + +#### ❌ BAD Examples (不要这样做): + +```python +# 版本导入 +try: + from ._version import __version__ +except ImportError: + __version__ = "unknown" # ❌ NO - 隐藏缺失文件 + +# 配置加载 +try: + config = load_config("config.yaml") +except FileNotFoundError: + config = {} # ❌ NO - 隐藏缺失配置 + +# 环境变量 +api_key = os.getenv("API_KEY") or "default_key" # ❌ NO - 隐藏缺失变量 +``` + +#### ✅ GOOD Examples (这样做): + +```python +# 让异常传播并提供清晰的错误信息 +config = load_config("config.yaml") # 如果缺失则 FileNotFoundError +api_key = os.environ["API_KEY"] # 如果缺失则 KeyError + +# 或提供有用的错误信息 +if not os.path.exists("config.yaml"): + raise FileNotFoundError( + "config.yaml not found. Please create it from config.yaml.template" + ) +``` + +**Rationale**: 快速失败，明确报错。静默 fallback 会隐藏 bug，使调试更困难，在生产环境中不可接受。 + +## SAGE Dependencies + +### Core Dependencies + +sage-examples 依赖于 SAGE 核心包（通过 PyPI 安装）： + +```toml +# 核心包（必需） +dependencies = [ + "isage-common>=0.2.0", # L1: Foundation + "isage-llm-core>=0.2.0", # L1: LLM control plane + client + "isage-libs>=0.2.0", # L3: Algorithms, RAG, Agents +] + +# 中间件包（可选，用于高级示例） +[project.optional-dependencies] +middleware = [ + "isage-middleware>=0.2.0", # L4: Operators (C++ extensions) +] + +# 应用包（可选，用于应用示例） +apps = [ + "isage-apps>=0.2.0", # L5: Production applications +] + +# 完整安装 +full = [ + "isage-middleware>=0.2.0", + "isage-apps>=0.2.0", + "isage-vdb>=0.2.0", # Vector database + "isage-benchmark>=0.2.0", # Benchmark tools +] +``` + +### SAGE Architecture Reference + +SAGE 采用 6 层分层架构（L1-L6）： + +``` +L6: sage-cli, sage-studio, sage-tools, sage-llm-gateway, sage-edge +L5: sage-apps +L4: sage-middleware +L3: sage-kernel, sage-libs +L2: sage-platform +L1: sage-common, sage-llm-core +``` + +**PyPI Package Names**: +- `isage-common` (sage-common) +- `isage-llm-core` (sage-llm-core) +- `isage-llm-gateway` (sage-llm-gateway) +- `isage-libs` (sage-libs) +- `isage-middleware` (sage-middleware) +- `isage-apps` (sage-apps) +- `isage-vdb` (SageVDB 向量数据库) +- `isage-benchmark` (benchmark 工具) + +**注意**: 由于 'sage' 在 PyPI 已被占用，所有包名使用 'isage-' 前缀。 + +## Directory Structure + +``` +sage-examples/ +├── apps/ # 🎯 生产应用示例 +│ ├── run_video_intelligence.py +│ ├── run_medical_diagnosis.py +│ ├── run_smart_home.py +│ └── README.md +│ +├── tutorials/ # 📚 教程（按架构分层） +│ ├── hello_world.py +│ ├── QUICK_START.md +│ ├── README.md +│ ├── L1-common/ # 基础层教程 +│ ├── L2-platform/ # 平台层教程 +│ ├── L3-kernel/ # 核心层教程（执行引擎） +│ ├── L3-libs/ # 核心层教程（RAG、Agents） +│ ├── L4-middleware/ # 中间件层教程 +│ ├── L5-apps/ # 应用层教程 +│ ├── L6-interface/ # 接口层教程 +│ ├── config/ # 配置示例 +│ └── docs/ # 文档 +│ +├── sage-apps/ # sage-apps 包的独立开发 +│ ├── pyproject.toml +│ ├── src/sage/apps/ +│ └── tests/ +│ +├── data/ # 共享数据文件 +├── requirements.txt # Python 依赖 +├── README.md +└── LICENSE +``` + +## How Copilot Should Learn SAGE Examples + +### Documentation-First Approach + +在回答问题或修改代码之前，Copilot **必须先依赖项目文档/README，而不是猜测**。 + +**在进行任何非平凡工作之前，Copilot 应该至少浏览：** + +- Root overview: `README.md` (示例总览、快速开始) +- Tutorials overview: `tutorials/README.md` (教程结构、学习路径) +- Apps overview: `apps/README.md` (应用列表、使用方法) +- Quick start: `tutorials/QUICK_START.md` (5分钟快速入门) + +**在处理特定层级/主题时，Copilot 应该额外阅读：** + +- 对应层级的 README，例如： + - `tutorials/L1-common/README.md` + - `tutorials/L2-platform/README.md` + - `tutorials/L3-kernel/README.md` / `L3-libs/README.md` + - `tutorials/L4-middleware/README.md` + - `tutorials/L5-apps/README.md` + - `tutorials/L6-interface/README.md` + +**🔍 遇到困难或不确定时：** + +- **首先**检查相关文档是否存在 +- **使用工具**如 `grep_search` 或 `semantic_search` 在文档中查找 +- **先阅读后行动** - 文档是为了指导你，不是可选参考 +- **常见文档位置：** + - 教程文档：`tutorials/docs/` + - 配置示例：`tutorials/config/` + - 应用说明：`apps/README.md` + - 快速入门：`tutorials/QUICK_START.md` + +**规则**: 不要猜测架构决策或策略。阅读文档。文档就是为此而存在的。 + +## SAGE Examples Scope + +### What sage-examples Contains + +**✅ 包含内容：** + +1. **Tutorials (tutorials/)**: 分层教程，覆盖 L1-L6 各层 + - 基础概念演示 + - 最佳实践示例 + - 循序渐进的学习路径 + - 配置文件示例 + +2. **Applications (apps/)**: 生产级应用示例 + - 视频智能分析 + - 医疗诊断系统 + - 智能家居 + - 文章监控 + - 自动扩展聊天 + +3. **sage-apps Package (sage-apps/)**: 独立开发的应用包 + - 源代码：`src/sage/apps/` + - 测试：`tests/` + - 打包配置：`pyproject.toml` + +### What sage-examples Does NOT Contain + +**❌ 不包含内容（在 SAGE 主仓库）：** + +1. **核心框架代码**: 在 `SAGE/packages/` 中 +2. **工具和脚本**: 在 `SAGE/tools/` 中 +3. **CI/CD 配置**: 在 `SAGE/.github/` 中 +4. **开发者文档**: 在 `SAGE/docs-public/` 中 +5. **Benchmark 框架**: 独立仓库 `sage-benchmark` + +### Relationship with SAGE Main Repository + +``` +┌──────────────────────────────────────────────┐ +│ SAGE Main Repository │ +│ (intellistream/SAGE) │ +├──────────────────────────────────────────────┤ +│ • 核心框架包 (packages/) │ +│ • 开发工具 (tools/) │ +│ • 文档 (docs-public/) │ +│ • CI/CD (.github/) │ +└──────────────────────────────────────────────┘ + ↓ + PyPI 发布 + (isage-* packages) + ↓ +┌──────────────────────────────────────────────┐ +│ sage-examples Repository │ +│ (intellistream/sage-examples) │ +├──────────────────────────────────────────────┤ +│ • 教程代码 (tutorials/) │ +│ • 应用示例 (apps/) │ +│ • 独立 sage-apps 包 (sage-apps/) │ +│ • 依赖 PyPI 上的 isage-* 包 │ +└──────────────────────────────────────────────┘ +``` + +## Installation + +### Minimal (Tutorials only) + +```bash +# 克隆仓库 +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples + +# 安装核心依赖 +pip install -r requirements.txt + +# 或只安装教程所需 +pip install isage-common isage-llm-core isage-libs +``` + +### Full (All examples including apps) + +```bash +# 安装所有依赖 +pip install -r requirements.txt + +# 或使用 pip extras +pip install -e ".[full]" +``` + +### Development (sage-apps package) + +```bash +# 安装 sage-apps 包用于开发 +cd sage-apps +pip install -e ".[dev]" +``` + +## Environment Setup + +### Environment Variables + +创建 `.env` 文件（从 `.env.template` 复制）： + +```bash +# === LLM API Keys === +OPENAI_API_KEY=sk-xxx +HF_TOKEN=hf_xxx + +# === SAGE Services (可选，使用本地服务时) === +# SAGE_CHAT_BASE_URL=http://localhost:8889/v1 # Gateway URL +# SAGE_CHAT_MODEL=Qwen/Qwen2.5-7B-Instruct + +# === DashScope (可选，使用阿里云时) === +# SAGE_CHAT_API_KEY=sk-xxx +# SAGE_CHAT_MODEL=qwen-turbo-2025-02-11 +# SAGE_CHAT_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +**注意**: +- 本地开发优先使用 SAGE Gateway (`localhost:8889`) +- DashScope 等云端 API 仅用于无本地服务时 +- HuggingFace Token 用于下载模型（会自动检测网络并配置镜像） + +## Running Examples + +### Quick Start + +```bash +# 1. 最简单的示例（30秒） +python tutorials/hello_world.py + +# 2. Embedding 示例（2分钟） +python tutorials/embedding_server_example.py + +# 3. Agent 示例（5分钟） +python tutorials/L3-libs/agents/basic_agent.py +``` + +### By Category + +```bash +# === 基础教程 === +cd tutorials/L1-common && python hello_world.py + +# === 平台服务 === +cd tutorials/L2-platform/scheduler && python scheduler_comparison.py + +# === 核心引擎 === +cd tutorials/L3-kernel/batch && python hello_local_batch.py + +# === RAG & Agents === +cd tutorials/L3-libs/rag && python simple_rag.py +cd tutorials/L3-libs/agents && python basic_agent.py + +# === 中间件 === +cd tutorials/L4-middleware/vector_db && python sagedb_demo.py + +# === 应用 === +cd apps && python run_smart_home.py +``` + +### Applications + +```bash +# 视频智能分析 +python apps/run_video_intelligence.py --video path/to/video.mp4 + +# 医疗诊断 +python apps/run_medical_diagnosis.py + +# 智能家居 +python apps/run_smart_home.py + +# 文章监控 +python apps/run_article_monitoring.py + +# 自动扩展聊天 +python apps/run_auto_scaling_chat.py +``` + +## Common Issues + +### Import Errors + +**问题**: `ModuleNotFoundError: No module named 'sage'` + +**解决**: +```bash +# 确保安装了 SAGE 核心包 +pip install isage-common isage-llm-core isage-libs + +# 或安装完整依赖 +pip install -r requirements.txt +``` + +### API Key Errors + +**问题**: `ValueError: OPENAI_API_KEY not found` + +**解决**: +```bash +# 1. 创建 .env 文件 +cp .env.template .env + +# 2. 编辑 .env 添加 API key +echo "OPENAI_API_KEY=sk-xxx" >> .env + +# 3. 或使用环境变量 +export OPENAI_API_KEY=sk-xxx +``` + +### Service Connection Errors + +**问题**: `Connection refused to http://localhost:8889` + +**解决**: +```bash +# 确保 SAGE Gateway 正在运行（在 SAGE 主仓库） +cd /path/to/SAGE +sage gateway start + +# 或使用云端 API（设置 .env） +SAGE_CHAT_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +### Model Download Issues + +**问题**: HuggingFace 模型下载慢或失败 + +**解决**: +```bash +# 中国大陆自动使用镜像，无需手动配置 +# 如需强制使用镜像： +export HF_ENDPOINT=https://hf-mirror.com + +# 或预下载模型（在 SAGE 主仓库） +sage llm model download Qwen/Qwen2.5-7B-Instruct +``` + +## Testing + +```bash +# 运行所有测试 +pytest + +# 运行特定测试 +pytest tests/test_tutorials.py +pytest tests/test_apps.py + +# 运行 sage-apps 包测试 +cd sage-apps +pytest tests/ +``` + +## Code Style + +### Formatting & Linting + +```bash +# 使用 ruff 格式化 +ruff format . + +# 检查代码质量 +ruff check . + +# 自动修复 +ruff check --fix . +``` + +### Pre-commit Hooks + +```bash +# 安装 pre-commit +pip install pre-commit + +# 安装 hooks +pre-commit install + +# 手动运行 +pre-commit run --all-files +``` + +## Contributing + +### Adding New Examples + +1. **选择正确的位置**: + - 教程 → `tutorials/L{1-6}-*/` + - 应用 → `apps/` + - 配置 → `tutorials/config/` + +2. **遵循命名规范**: + - 使用描述性名称：`basic_agent.py`, `simple_rag.py` + - 添加 README：每个目录应有 README.md + - 示例数据：放在 `data/` 或 `tutorials/*/data/` + +3. **文档要求**: + - 代码注释：解释关键步骤 + - README：说明示例目的和使用方法 + - Docstrings：函数和类的文档字符串 + +4. **依赖管理**: + - 更新 `requirements.txt` 或 `pyproject.toml` + - 不要手动 `pip install` + - 声明版本约束 + +### Example Template + +```python +""" +Brief description of what this example demonstrates. + +Requirements: + - isage-libs>=0.2.0 + - transformers>=4.52.0 + +Usage: + python example_name.py +""" + +from sage.llm import UnifiedInferenceClient +from sage.libs.rag import SimpleRAG + + +def main(): + """Main function with clear steps.""" + # Step 1: Setup + client = UnifiedInferenceClient.create() + + # Step 2: Execute + response = client.chat([ + {"role": "user", "content": "Hello!"} + ]) + + # Step 3: Display results + print(f"Response: {response}") + + +if __name__ == "__main__": + main() +``` + +## Resources + +- **SAGE 主仓库**: https://github.com/intellistream/SAGE +- **SAGE 文档**: https://sage.intellistream.com +- **PyPI 包**: https://pypi.org/search/?q=isage +- **sage-benchmark**: https://github.com/intellistream/sage-benchmark +- **Issue Tracker**: https://github.com/intellistream/sage-examples/issues + +## Final Reminder for Copilot + +**Trust these instructions** - search only if incomplete, errors occur, or deep architecture needed. + +**🔍 When encountering difficulties or uncertainties:** + +1. **First**, check if there's relevant documentation in `tutorials/docs/` or layer READMEs +2. **Use tools** like `grep_search` or `semantic_search` to find documentation before making assumptions +3. **Read before acting** - documentation exists to guide you, not as optional reference +4. **Common documentation locations:** + - Root: `README.md` + - Tutorials: `tutorials/README.md`, `tutorials/QUICK_START.md` + - Layer guides: `tutorials/L{1-6}-*/README.md` + - Apps: `apps/README.md` + +**Remember**: Don't guess. Read the docs. They exist for this reason. diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a8c2003 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1446575 --- /dev/null +++ b/README.md @@ -0,0 +1,238 @@ +# SAGE Examples Collection + +Complete examples and tutorials demonstrating SAGE's capabilities. + +## 🚀 Quick Start + +**New to SAGE?** Start with tutorials: + +```bash +# Your first SAGE program (30 seconds) +python examples/tutorials/hello_world.py + +# Learn embeddings (2 minutes) +python examples/tutorials/embedding_demo.py + +# Build an agent (5 minutes) +python examples/tutorials/agents/basic_agent.py +``` + +## 📁 Directory Structure + +``` +examples/ +├── tutorials/ # 📚 Learning tutorials (START HERE!) +│ ├── hello_world.py +│ ├── agents/ # Agent tutorials +│ │ ├── config/ # Agent configurations +│ │ └── data/ # Agent test data +│ ├── multimodal/ # Text+Image+Video +│ ├── memory/ # Memory systems +│ │ ├── config/ # Memory configurations +│ │ └── data/ # Memory test data +│ ├── service/ # Service integration +│ ├── scheduler/ # Task scheduling +│ ├── sage_db/ # Vector database +│ ├── config/ # General configurations +│ ├── data/ # General data utilities +│ └── ... +│ +├── apps/ # 🎯 Production applications +│ ├── run_video_intelligence.py +│ └── run_medical_diagnosis.py +│ +└── memory/ # � Advanced memory examples (DEPRECATED - use tutorials/memory) +``` + +**Note**: RAG examples and benchmarks have been moved to `packages/sage-benchmark/`. See +`packages/sage-benchmark/README.md` for details. + +## 📚 Examples by Level + +### 🟢 Beginner (< 30 minutes) + +Simple, focused tutorials to learn SAGE basics. + +**Location**: `examples/tutorials/` + +- **Hello World**: Your first pipeline +- **Embeddings**: Text embeddings basics +- **Basic Agent**: Create an AI agent +- **Simple RAG**: Question answering +- **Service Basics**: Embedding service, pipeline service + +**Run**: See `examples/tutorials/README.md` + +### 🟡 Intermediate (30 min - 2 hours) + +Production-ready patterns and integrations. + +**Location**: `packages/sage-benchmark/`, `examples/memory/` + +- **Advanced RAG**: Dense/sparse retrieval, reranking, multimodal fusion (see `sage-benchmark`) +- **RAG Benchmarking**: Performance evaluation and metrics (see `sage-benchmark/benchmark_rag`) +- **Memory Systems**: RAG with memory, persistence patterns +- **Vector Databases**: Milvus, ChromaDB, FAISS integration (see `sage-benchmark`) +- **Distributed RAG**: Ray-based parallel processing (see `sage-benchmark`) + +**See**: `packages/sage-benchmark/README.md` for RAG and benchmark examples + +### 🔴 Advanced (2+ hours) + +Complete applications and complex workflows. + +**Location**: `examples/apps/` + +- **Video Intelligence**: Multi-model video analysis +- **Medical Diagnosis**: AI medical imaging +- **Multi-agent Systems**: Coordinated AI agents + +## 📦 Installation + +### Minimal (Tutorials only) + +```bash +pip install -e packages/sage-libs +``` + +### Full (All examples) + +```bash +# All applications +pip install -e packages/sage-apps[all] + +# RAG and benchmarking +pip install -e packages/sage-benchmark + +# Or specific apps +pip install -e packages/sage-apps[video] +pip install -e packages/sage-apps[medical] + +# All examples dependencies +pip install -r examples/requirements.txt +``` + +## ⚠️ Examples vs Tests + +**This directory contains examples and demos, NOT tests.** + +- **Examples** (`examples/`): How to use SAGE features +- **Unit Tests** (`packages/*/tests/`): Verify code correctness +- **Integration Tests** (`tools/tests/`): Test example execution + +## 🎯 Learning Paths + +### Path 1: RAG Developer + +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/rag_simple.py` - Learn + basics +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_milvus.py` + \- Add vector search +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_rerank.py` - Improve + results +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` + \- Add multimodal +1. `tutorials/memory/rag_memory_pipeline.py` - Add memory + +See `packages/sage-benchmark/README.md` for complete RAG documentation. + +**Note**: All RAG examples have been moved from `examples/rag/` to `packages/sage-benchmark/`. + +### Path 2: Agent Builder + +1. `tutorials/agents/basic_agent.py` - Agent basics +1. `tutorials/agents/workflow_demo.py` - Workflows +1. `tutorials/agents/arxiv_search_tool.py` - Custom tools +1. `apps/run_medical_diagnosis.py` - Multi-agent app + +### Path 3: Service Developer + +1. `tutorials/service/embedding_service_demo.py` - Service basics +1. `tutorials/service/pipeline_as_service/` - Pipeline services +1. `tutorials/service/sage_db/` - Vector database service +1. `tutorials/service/sage_flow/` - Stream processing service + +## 💡 Tips + +**Running examples:** + +```bash +# Always run from project root +cd /path/to/SAGE +python examples/tutorials/hello_world.py +``` + +**API Keys:** + +```bash +# Copy and configure +cp .env.example .env +# Edit .env with your keys +``` + +**Configurations:** + +```bash +# Tutorials have their own config directories +ls examples/tutorials/agents/config/*.yaml +ls examples/tutorials/memory/config/*.yaml + +# RAG configs are in sage-benchmark +ls packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/*.yaml + +# See respective README files for details +cat packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/README.md +``` + +**Troubleshooting:** + +- Missing dependencies? Check `requirements.txt` in each category +- Import errors? Make sure you installed SAGE: `pip install -e packages/sage-libs` +- Need data? Check the `data/` subdirectory in each example category +- Need config? Check the `config/` subdirectory in each example category +- Need help? See `docs/COMMUNITY.md` + +## 📖 Documentation + +- **Tutorials README**: `examples/tutorials/README.md` +- **Apps README**: `examples/apps/README.md` +- **RAG & Benchmarks**: `packages/sage-benchmark/README.md` +- **Main Docs**: `docs/` and `docs-public/` +- **API Docs**: Docstrings in `packages/sage-libs/` + +## 🔍 Quick Reference + +### By Feature + +- **Agents**: `tutorials/agents/` +- **RAG**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/` +- **Multimodal**: `tutorials/multimodal/`, + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` +- **Memory**: `tutorials/memory/` +- **Services**: `tutorials/service/` +- **Streaming**: `tutorials/stream_mode/` +- **Distributed**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_ray.py` +- **Benchmarking**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/` + +### By Technology + +- **ChromaDB**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_qa_chroma.yaml` +- **Milvus**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_*_milvus.yaml` +- **Ray**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_ray.yaml` +- **OpenAI**: Most RAG examples in sage-benchmark +- **Hugging Face**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_hf_model.py` +- **Local LLMs**: Various examples in sage-benchmark + +### By Use Case + +- **Question Answering**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_*.py` +- **Document Search**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/build_*_index.py` +- **Image Search**: `tutorials/multimodal/` +- **Video Analysis**: `apps/run_video_intelligence.py` +- **Medical AI**: `apps/run_medical_diagnosis.py` +- **Web Services**: `tutorials/service/pipeline_as_service/` +- **RAG Benchmarking**: `sage-benchmark/src/sage/benchmark/benchmark_rag/` diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..3537cd7 --- /dev/null +++ b/__init__.py @@ -0,0 +1,12 @@ +""" +SAGE 应用示例模块 + +包含各种类型的示例应用： +- tutorials: 基础教程和入门示例 +- apps: 应用示例 +- kernel: 内核示例 +- unlearning: 机器遗忘示例 +""" + +# 这些是示例目录，不需要在 __all__ 中声明 +# 用户可以直接访问这些目录中的示例文件 diff --git a/apps/README.md b/apps/README.md new file mode 100644 index 0000000..3535b3e --- /dev/null +++ b/apps/README.md @@ -0,0 +1,136 @@ +# SAGE Applications Examples + +This directory contains example scripts for running applications from the `sage-apps` package. + +## Overview + +The `sage-apps` package provides production-ready AI applications built on the SAGE framework. These +example scripts demonstrate how to use them. + +## Available Applications + +### 1. Video Intelligence Pipeline + +Multi-model video analysis combining CLIP and MobileNetV3 for comprehensive video understanding. + +**Run:** + +```bash +python examples/apps/run_video_intelligence.py --video path/to/video.mp4 +``` + +**Features:** + +- Scene understanding with CLIP +- Action recognition with MobileNetV3 +- Frame-by-frame analysis +- Comprehensive video insights + +### 2. Medical Diagnosis System + +AI-assisted medical imaging analysis using multi-agent systems. + +**Run:** + +```bash +python examples/apps/run_medical_diagnosis.py +``` + +**Features:** + +- Multi-agent diagnostic workflow +- Medical image analysis +- Knowledge base integration +- Diagnostic report generation + +### 3. Article Monitoring + +Automated article/news monitoring and analysis pipeline. + +**Run:** + +```bash +python examples/apps/run_article_monitoring.py +``` + +**Features:** + +- Article content extraction +- Sentiment analysis +- Topic classification +- Automated summarization + +### 4. Auto Scaling Chat + +Scalable chat application with automatic resource scaling. + +**Run:** + +```bash +python examples/apps/run_auto_scaling_chat.py +``` + +**Features:** + +- Dynamic scaling based on load +- Multi-turn conversation support +- LLM integration +- Resource-aware execution + +### 5. Smart Home + +IoT-enabled smart home automation using SAGE pipelines. + +**Run:** + +```bash +python examples/apps/run_smart_home.py +``` + +**Features:** + +- Sensor data processing +- Device control automation +- Event-driven workflows +- Real-time monitoring + +## Installation + +### Install All Applications + +```bash +pip install -e packages/sage-apps[all] +``` + +### Install Specific Applications + +```bash +# Video Intelligence only +pip install -e packages/sage-apps[video] + +# Medical Diagnosis only +pip install -e packages/sage-apps[medical] +``` + +## Configuration + +Each application can be configured using YAML files located in `examples/config/`: + +- Video Intelligence: `config_video_intelligence.yaml` +- Medical Diagnosis: Custom configurations in the medical diagnosis data directory + +## Development + +For developing new applications or modifying existing ones, see: + +- `packages/sage-apps/README.md` - Package overview +- `packages/sage-apps/MIGRATION.md` - Migration guide +- `packages/sage-apps/PACKAGE_CREATION_SUMMARY.md` - Implementation details + +## Support + +For questions or issues: + +1. Check the application-specific documentation in `packages/sage-apps/src/sage/apps//` +1. Review the main SAGE documentation +1. Open an issue on GitHub diff --git a/apps/__init__.py b/apps/__init__.py new file mode 100644 index 0000000..7049b2d --- /dev/null +++ b/apps/__init__.py @@ -0,0 +1,7 @@ +""" +SAGE Applications Examples + +This package contains example scripts for running applications from sage-apps. +""" + +__version__ = "0.1.0" diff --git a/apps/demo_feature_extraction.py b/apps/demo_feature_extraction.py new file mode 100644 index 0000000..04bf44e --- /dev/null +++ b/apps/demo_feature_extraction.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +""" +Demo: Feature Extraction with Pretrained Models + +This script demonstrates the use of CLIP and DINOv2 models for feature extraction +in the medical diagnosis application. + +Issue #894: Use pretrained models for feature extraction instead of random features +""" + +import numpy as np + +from sage.apps.medical_diagnosis.agents.image_analyzer import ( + TORCH_AVAILABLE, + ImageAnalyzer, +) + + +def demo_clip_features(): + """Demonstrate CLIP feature extraction""" + print("\n" + "=" * 80) + print("🔬 Demo: CLIP Feature Extraction") + print("=" * 80) + + config = { + "models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}, + "image_processing": {"feature_extraction": {"method": "clip", "dimension": 512}}, + } + + if not TORCH_AVAILABLE: + print("⚠️ PyTorch not available, will use mock features") + else: + print("✅ PyTorch available, attempting to load CLIP model...") + + analyzer = ImageAnalyzer(config) + + # Create a mock medical image + mock_mri_image = np.random.randint(0, 255, (512, 512), dtype=np.uint8) + print("\n📸 Created mock MRI image (512x512)") + + # Extract features + print("\n🔍 Extracting features...") + features = analyzer._extract_features(mock_mri_image) + + if features is not None: + print("\n✅ Feature extraction successful!") + print(f" - Feature dimension: {features.shape[0]}") + print(f" - Feature type: {features.dtype}") + print(f" - Feature norm: {np.linalg.norm(features):.4f}") + print(f" - Feature range: [{features.min():.4f}, {features.max():.4f}]") + + # Show a few feature values + print("\n📊 First 10 feature values:") + print(f" {features[:10]}") + else: + print("\n❌ Feature extraction failed") + + +def demo_dinov2_features(): + """Demonstrate DINOv2 feature extraction""" + print("\n" + "=" * 80) + print("🔬 Demo: DINOv2 Feature Extraction") + print("=" * 80) + + config = { + "models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}, + "image_processing": {"feature_extraction": {"method": "dinov2", "dimension": 768}}, + } + + if not TORCH_AVAILABLE: + print("⚠️ PyTorch not available, will use mock features") + else: + print("✅ PyTorch available, attempting to load DINOv2 model...") + + analyzer = ImageAnalyzer(config) + + # Create a mock medical image + mock_mri_image = np.random.randint(0, 255, (512, 512), dtype=np.uint8) + print("\n📸 Created mock MRI image (512x512)") + + # Extract features + print("\n🔍 Extracting features...") + features = analyzer._extract_features(mock_mri_image) + + if features is not None: + print("\n✅ Feature extraction successful!") + print(f" - Feature dimension: {features.shape[0]}") + print(f" - Feature type: {features.dtype}") + print(f" - Feature norm: {np.linalg.norm(features):.4f}") + print(f" - Feature range: [{features.min():.4f}, {features.max():.4f}]") + + # Show a few feature values + print("\n📊 First 10 feature values:") + print(f" {features[:10]}") + else: + print("\n❌ Feature extraction failed") + + +def demo_full_analysis(): + """Demonstrate full image analysis pipeline""" + print("\n" + "=" * 80) + print("🔬 Demo: Full Image Analysis Pipeline") + print("=" * 80) + + config = { + "models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}, + "image_processing": {"feature_extraction": {"method": "clip", "dimension": 768}}, + } + + analyzer = ImageAnalyzer(config) + + # Analyze a non-existent image (will use mock data) + print("\n🔍 Running full analysis (using mock data)...") + result = analyzer.analyze("test_patient_mri.dcm") + + print("\n📊 Analysis Results:") + print(f" - Image quality: {result['image_quality']:.2f}") + print(f" - Vertebrae detected: {len(result['vertebrae'])}") + print(f" - Discs detected: {len(result['discs'])}") + print(f" - Abnormalities found: {len(result['abnormalities'])}") + + if result["image_embedding"] is not None: + print("\n🧬 Image Embedding:") + print(f" - Dimension: {result['image_embedding'].shape[0]}") + print(f" - Norm: {np.linalg.norm(result['image_embedding']):.4f}") + + print("\n📝 Detected Abnormalities:") + for i, abnormality in enumerate(result["abnormalities"], 1): + print(f" {i}. {abnormality['description']}") + print(f" Location: {abnormality['location']}, Severity: {abnormality['severity']}") + + +def main(): + """Main demo function""" + print("\n" + "=" * 80) + print("🎯 Medical Image Feature Extraction Demo") + print("=" * 80) + print("\nThis demo showcases the integration of pretrained models (CLIP, DINOv2)") + print("for feature extraction in medical image analysis.") + print("\nIssue #894: https://github.com/intellistream/SAGE/issues/894") + + # Demo 1: CLIP + demo_clip_features() + + # Demo 2: DINOv2 + demo_dinov2_features() + + # Demo 3: Full analysis + demo_full_analysis() + + print("\n" + "=" * 80) + print("✅ Demo Complete!") + print("=" * 80) + print("\n💡 Note: When models are not available (e.g., no PyTorch or GPU),") + print(" the system automatically falls back to mock features for testing.") + print("\n") + + +if __name__ == "__main__": + main() diff --git a/apps/run_article_monitoring.py b/apps/run_article_monitoring.py new file mode 100644 index 0000000..4d9d5fb --- /dev/null +++ b/apps/run_article_monitoring.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +Article Monitoring System Example + +This script demonstrates the Article Monitoring application from sage-apps. +It monitors arXiv for relevant research papers using keyword and semantic filtering. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python apps/run_article_monitoring.py + python apps/run_article_monitoring.py --keywords "transformer,attention" + python apps/run_article_monitoring.py --category cs.CL --max-articles 20 + +Test Configuration: + @test_category: apps + @test_speed: medium + @test_requires: [network] # May need network for arXiv API +""" + +import argparse +import sys +from pathlib import Path + +try: + from sage.apps.article_monitoring import run_article_monitoring_pipeline + from sage.common.utils.logging.custom_logger import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.article_monitoring: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the article monitoring pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Article Monitoring System - Monitor and filter arXiv papers", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Use default settings + python %(prog)s + + # Custom keywords + python %(prog)s --keywords "transformer,attention,bert,nlp" + + # Different category + python %(prog)s --category cs.CL --max-articles 20 + + # Verbose mode + python %(prog)s --verbose + +Features: + - Real-time arXiv paper fetching + - Keyword-based filtering + - Semantic similarity filtering + - Ranked recommendations + """, + ) + + parser.add_argument( + "--keywords", + "-k", + type=str, + help="Comma-separated list of keywords", + ) + + parser.add_argument( + "--topics", + "-t", + type=str, + help="Comma-separated list of interest topics", + ) + + parser.add_argument( + "--category", + "-c", + type=str, + default="cs.AI", + help="arXiv category to monitor (default: cs.AI)", + ) + + parser.add_argument( + "--max-articles", + "-m", + type=int, + default=10, + help="Maximum number of articles to fetch (default: 10)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Parse keywords + keywords = None + if args.keywords: + keywords = [kw.strip() for kw in args.keywords.split(",")] + + # Parse topics + topics = None + if args.topics: + topics = [t.strip() for t in args.topics.split(",")] + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + print("=" * 70) + print("SAGE Article Monitoring System - Example") + print("=" * 70) + print() + + # Run pipeline + try: + run_article_monitoring_pipeline( + keywords=keywords, + interest_topics=topics, + category=args.category, + max_articles=args.max_articles, + verbose=args.verbose, + ) + except Exception as e: + print(f"\nError running article monitoring pipeline: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/run_auto_scaling_chat.py b/apps/run_auto_scaling_chat.py new file mode 100644 index 0000000..07820e3 --- /dev/null +++ b/apps/run_auto_scaling_chat.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Auto-Scaling Chat System Example + +This script demonstrates the Auto-Scaling Chat application from sage-apps. +It simulates elastic scaling of chat servers based on user load. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python apps/run_auto_scaling_chat.py + python apps/run_auto_scaling_chat.py --duration 60 --peak-rate 80 + python apps/run_auto_scaling_chat.py --verbose + +Test Configuration: + @test_category: apps + @test_speed: medium +""" + +import argparse +import sys +from pathlib import Path + +try: + from sage.apps.auto_scaling_chat import run_auto_scaling_demo + from sage.common.utils.logging.custom_logger import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.auto_scaling_chat: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the auto-scaling chat system demo.""" + parser = argparse.ArgumentParser( + description="SAGE Auto-Scaling Chat System - Elastic resource management demo", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run default simulation (30s) + python %(prog)s + + # Short simulation with high peak + python %(prog)s --duration 20 --peak-rate 80 + + # Long simulation with moderate load + python %(prog)s --duration 60 --peak-rate 40 + + # Verbose mode with scaling events + python %(prog)s --verbose + +Features: + - Simulated variable user load + - Automatic server scaling (up/down) + - Load balancing across servers + - Real-time metrics collection + """, + ) + + parser.add_argument( + "--duration", + "-d", + type=int, + default=30, + help="Simulation duration in seconds (default: 30)", + ) + + parser.add_argument( + "--base-rate", + "-b", + type=int, + default=5, + help="Base user load (default: 5)", + ) + + parser.add_argument( + "--peak-rate", + "-p", + type=int, + default=50, + help="Peak user load (default: 50)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + print("=" * 70) + print("SAGE Auto-Scaling Chat System - Example") + print("=" * 70) + print() + + # Run demo + try: + run_auto_scaling_demo( + duration=args.duration, + base_rate=args.base_rate, + peak_rate=args.peak_rate, + verbose=args.verbose, + ) + except Exception as e: + print(f"\nError running auto-scaling demo: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/run_medical_diagnosis.py b/apps/run_medical_diagnosis.py new file mode 100644 index 0000000..e16bbf0 --- /dev/null +++ b/apps/run_medical_diagnosis.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +Medical Diagnosis System Example + +This script demonstrates how to use the Medical Diagnosis application from sage-apps. +It provides AI-assisted medical imaging analysis using a multi-agent system. + +Requirements: + pip install -e sage-apps[medical] + # Or: pip install isage-apps[medical] + +Usage: + python apps/run_medical_diagnosis.py + python apps/run_medical_diagnosis.py --case-id case_0001 + python apps/run_medical_diagnosis.py --interactive + +Test Configuration: + @test_category: apps + @test_speed: slow + @test_requires: [medical, data] + @test_skip_ci: true + @test:skip - Requires medical dataset to be downloaded +""" + +import argparse +import subprocess +import sys +from pathlib import Path + +try: + from sage.apps.medical_diagnosis.run_diagnosis import main as diagnosis_main +except ImportError as e: + print(f"Error importing sage.apps.medical_diagnosis: {e}") + print("\nPlease install sage-apps with medical dependencies:") + print(" cd sage-apps && pip install -e .[medical]") + print(" Or: pip install isage-apps[medical]") + sys.exit(1) + + +def check_and_setup_data(data_dir: str, auto_setup: bool = False) -> bool: + """检查数据是否存在，如果不存在则提示用户自动设置""" + data_path = Path(data_dir) + processed_dir = data_path / "processed" + + # 检查数据是否存在 + if processed_dir.exists() and (processed_dir / "train_index.json").exists(): + return True + + print(f"\n{'=' * 60}") + print("数据集未找到") + print("=" * 60) + print(f"期望的数据目录: {data_dir}") + print("") + + # 查找 setup_data.sh 脚本 + setup_script = data_path.parent / "setup_data.sh" + + if not setup_script.exists(): + # 尝试其他可能的位置 + setup_script = ( + project_root / "packages/sage-apps/src/sage/apps/medical_diagnosis/setup_data.sh" + ) + + if not setup_script.exists(): + print("❌ 数据设置脚本未找到") + print("") + print("期望结构:") + print(" {data_dir}/processed/images/") + print(" {data_dir}/processed/train_index.json") + print(" {data_dir}/processed/test_index.json") + return False + + print(f"找到数据设置脚本: {setup_script}") + print("") + print("🤖 自动下载并准备数据集...") + print("提示: 如果不想自动下载，请使用 Ctrl+C 取消") + print("") + print("开始自动设置数据集...") + print("=" * 60) + + try: + # 运行 setup_data.sh + subprocess.run( + ["bash", str(setup_script)], + cwd=str(setup_script.parent), + check=True, + text=True, + ) + + print("=" * 60) + print("✅ 数据集设置完成！") + print("") + return True + + except subprocess.CalledProcessError: + print("=" * 60) + print("❌ 数据集设置失败") + print("") + print("您可以手动运行以下命令来设置数据:") + print(f" bash {setup_script}") + print("") + return False + except Exception as e: + print(f"❌ 发生错误: {e}") + return False + + +def main(): + """Run the medical diagnosis system.""" + parser = argparse.ArgumentParser( + description="SAGE Medical Diagnosis System - AI-assisted medical imaging analysis", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run with default demo case + python %(prog)s + + # Analyze a specific case + python %(prog)s --case-id case_0001 + + # Interactive mode + python %(prog)s --interactive + + # Use custom data directory + python %(prog)s --data-dir path/to/medical/data + +Features: + - Multi-agent diagnostic workflow + - Medical image analysis + - Knowledge base integration + - Diagnostic report generation + - Interactive consultation mode + """, + ) + + parser.add_argument("--case-id", type=str, help="Specific case ID to analyze (e.g., case_0001)") + + parser.add_argument( + "--data-dir", + type=str, + default="packages/sage-apps/src/sage/apps/medical_diagnosis/data", + help="Path to medical diagnosis data directory", + ) + + parser.add_argument( + "--interactive", + action="store_true", + help="Run in interactive consultation mode", + ) + + parser.add_argument("--output", type=str, help="Output directory for diagnostic reports") + + parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") + + parser.add_argument( + "--auto-setup", + action="store_true", + help="Automatically download and setup data without prompting", + ) + + args = parser.parse_args() + + # Check and setup data if needed + if not check_and_setup_data(args.data_dir, auto_setup=args.auto_setup): + print("\n⚠️ 警告: 数据集未就绪") + print("系统将尝试使用模拟数据运行...") + print("") + + print("=" * 60) + print("SAGE Medical Diagnosis System") + print("=" * 60) + print(f"Data Directory: {args.data_dir}") + if args.case_id: + print(f"Case ID: {args.case_id}") + if args.interactive: + print("Mode: Interactive") + else: + print("Mode: Automated Analysis") + if args.output: + print(f"Output: {args.output}") + print("=" * 60) + print() + + # Call the medical diagnosis main function + try: + diagnosis_main() + except Exception as e: + print(f"Error running medical diagnosis system: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/run_smart_home.py b/apps/run_smart_home.py new file mode 100644 index 0000000..a72f34b --- /dev/null +++ b/apps/run_smart_home.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Smart Home System Example + +This script demonstrates the Smart Home application from sage-apps. +It simulates IoT device coordination for automated laundry workflow. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python apps/run_smart_home.py + python apps/run_smart_home.py --cycles 3 + python apps/run_smart_home.py --verbose + +Test Configuration: + @test_category: apps + @test_speed: fast +""" + +import argparse +import sys +from pathlib import Path + +try: + from sage.apps.smart_home import run_smart_home_demo + from sage.common.utils.logging.custom_logger import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.smart_home: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the smart home automation demo.""" + parser = argparse.ArgumentParser( + description="SAGE Smart Home System - IoT device automation demo", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run single laundry cycle + python %(prog)s + + # Run multiple cycles + python %(prog)s --cycles 3 + + # Verbose mode with event logging + python %(prog)s --cycles 2 --verbose + +Features: + - IoT device simulation (robot, washer, dryer, sensors) + - Automated workflow coordination + - Environmental monitoring + - Event-driven architecture + """, + ) + + parser.add_argument( + "--cycles", + "-c", + type=int, + default=1, + help="Number of laundry cycles to run (default: 1)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + print("=" * 70) + print("SAGE Smart Home System - Example") + print("=" * 70) + print() + + # Run demo + try: + run_smart_home_demo(num_cycles=args.cycles, verbose=args.verbose) + except Exception as e: + print(f"\nError running smart home demo: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/run_video_intelligence.py b/apps/run_video_intelligence.py new file mode 100644 index 0000000..3d253a0 --- /dev/null +++ b/apps/run_video_intelligence.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +Video Intelligence Pipeline Example + +This script demonstrates how to use the Video Intelligence application from sage-apps. +It performs multi-model video analysis using CLIP and MobileNetV3. + +Requirements: + pip install -e sage-apps[video] + # Or: pip install isage-apps[video] + +Usage: + python apps/run_video_intelligence.py --video path/to/video.mp4 + python apps/run_video_intelligence.py --video path/to/video.mp4 --config custom_config.yaml + +Test Configuration: + @test_category: apps + @test_speed: slow + @test_requires: [video] + @test_skip_ci: true + @test:skip - Requires video file parameter +""" + +import argparse +import sys +from pathlib import Path + +try: + from sage.apps.video.video_intelligence_pipeline import main as video_main +except ImportError as e: + print(f"Error importing sage.apps.video: {e}") + print("\nPlease install sage-apps with video dependencies:") + print(" cd sage-apps && pip install -e .[video]") + print(" Or: pip install isage-apps[video]") + sys.exit(1) + + +def main(): + """Run the video intelligence pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Video Intelligence Pipeline - Multi-model video analysis", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Analyze a video file + python %(prog)s --video path/to/video.mp4 + + # Use custom configuration + python %(prog)s --video path/to/video.mp4 --config config_video_intelligence.yaml + + # Process multiple frames + python %(prog)s --video path/to/video.mp4 --max-frames 100 + +Features: + - Scene understanding with CLIP model + - Action recognition with MobileNetV3 + - Frame-by-frame analysis + - Comprehensive video insights + - Configurable processing pipeline + """, + ) + + parser.add_argument("--video", type=str, help="Path to the video file to analyze") + + parser.add_argument("--config", type=str, default=None, help="Path to configuration file") + + parser.add_argument("--max-frames", type=int, help="Maximum number of frames to process") + + parser.add_argument("--output", type=str, help="Output directory for results") + + args = parser.parse_args() + + # If no video is provided, show help and exit successfully (for testing purposes) + if not args.video: + print("=" * 60) + print("SAGE Video Intelligence Pipeline") + print("=" * 60) + print("No video file provided. This example requires a video file to process.") + print("\nUsage:") + print(" python examples/apps/run_video_intelligence.py --video path/to/video.mp4") + print("\nFor more information, use --help") + print("=" * 60) + sys.exit(0) + + # Validate video file if provided + if args.video and not Path(args.video).exists(): + print(f"Error: Video file not found: {args.video}") + sys.exit(1) + + # Validate config file if provided + if args.config and not Path(args.config).exists(): + print(f"Error: Config file not found: {args.config}") + sys.exit(1) + + print("=" * 60) + print("SAGE Video Intelligence Pipeline") + print("=" * 60) + if args.video: + print(f"Video: {args.video}") + if args.config: + print(f"Config: {args.config}") + else: + print("Config: Using default configuration") + if args.max_frames: + print(f"Max Frames: {args.max_frames}") + if args.output: + print(f"Output: {args.output}") + print("=" * 60) + print() + + # Call the video intelligence main function + # The actual implementation will handle the arguments + try: + video_main() + except Exception as e: + print(f"Error running video intelligence pipeline: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/apps/run_work_report.py b/apps/run_work_report.py new file mode 100644 index 0000000..e5bc5d8 --- /dev/null +++ b/apps/run_work_report.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +""" +Work Report Generator Example + +This script demonstrates the Work Report Generator application from sage-apps. +It generates weekly/daily work reports by combining GitHub data with optional diary entries. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python apps/run_work_report.py + python apps/run_work_report.py --days 7 + python apps/run_work_report.py --format json --output report.json + python apps/run_work_report.py --no-submodules --branch main + +Test Configuration: + @test_category: apps + @test_speed: medium + @test_requires: [network] # Needs GitHub API access +""" + +import argparse +import os +import sys +from pathlib import Path + +try: + from sage.apps.work_report_generator import run_work_report_pipeline + from sage.apps.work_report_generator.operators import GitHubDataSource + from sage.common.utils.logging.custom_logger import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.work_report_generator: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the work report generator pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Work Report Generator - Generate weekly/daily work reports", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Use default settings (all SAGE repos, main-dev branch, 7 days) + python %(prog)s + + # Main repo only, without submodules + python %(prog)s --no-submodules + + # Specify custom repositories + python %(prog)s --repos intellistream/SAGE,intellistream/sageLLM + + # Use different branch (e.g., main instead of main-dev) + python %(prog)s --branch main + + # Custom time range + python %(prog)s --days 14 + + # JSON output + python %(prog)s --format json --output reports/weekly.json + + # English report with verbose output + python %(prog)s --language en --verbose + + # Without LLM summarization (faster) + python %(prog)s --no-llm + +Environment Variables: + GITHUB_TOKEN or GIT_TOKEN: GitHub personal access token + +Features: + - Fetches commits and PRs from GitHub repositories + - Supports fetching from specific branches (default: main-dev) + - Includes all SAGE submodules by default + - Aggregates contributions by contributor + - Generates AI-powered summaries (optional) + - Supports markdown, JSON, and console output + """, + ) + + parser.add_argument( + "--repos", + "-r", + type=str, + default=None, + help="Comma-separated list of repositories. If not specified, fetches all SAGE repos.", + ) + + parser.add_argument( + "--branch", + "-b", + type=str, + default="main-dev", + help="Branch name to fetch commits from (default: main-dev)", + ) + + parser.add_argument( + "--days", + "-d", + type=int, + default=7, + help="Number of days to look back (default: 7)", + ) + + parser.add_argument( + "--format", + "-f", + type=str, + choices=["console", "markdown", "json"], + default="markdown", + help="Output format (default: markdown)", + ) + + parser.add_argument( + "--output", + "-o", + type=str, + help="Output file path (default: .sage/reports/weekly_report_.)", + ) + + parser.add_argument( + "--diary-path", + type=str, + help="Path to diary directory or file for personal notes", + ) + + parser.add_argument( + "--language", + "-l", + type=str, + choices=["zh", "en"], + default="zh", + help="Report language (default: zh for Chinese)", + ) + + parser.add_argument( + "--token", + "-t", + type=str, + help="GitHub personal access token (can also use GITHUB_TOKEN env var)", + ) + + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Enable verbose logging", + ) + + parser.add_argument( + "--no-llm", + action="store_true", + help="Disable LLM summary generation for faster reports", + ) + + parser.add_argument( + "--no-submodules", + action="store_true", + help="Only fetch from main SAGE repo, skip submodules", + ) + + args = parser.parse_args() + + # Setup logging + logger = CustomLogger.get_logger(__name__) + + # Parse repositories + repos = None + if args.repos: + repos = [r.strip() for r in args.repos.split(",")] + + # Get GitHub token + github_token = args.token or os.environ.get("GITHUB_TOKEN") or os.environ.get("GIT_TOKEN") + + if not github_token: + logger.warning( + "No GitHub token provided. Set GITHUB_TOKEN environment variable " + "or use --token option for full functionality. Using mock data." + ) + + # Determine repos to display + display_repos = ( + repos + if repos + else (GitHubDataSource.SAGE_REPOS if not args.no_submodules else ["intellistream/SAGE"]) + ) + + # Print configuration + print() + print("Configuration:") + print(f" Target Branch: {args.branch}") + print(f" Repositories ({len(display_repos)}):") + for repo in display_repos: + print(f" - {repo}") + print(f" Time Range: {args.days} days") + print(f" Output Format: {args.format}") + print(f" Language: {'Chinese' if args.language == 'zh' else 'English'}") + print(f" LLM Summary: {'Disabled' if args.no_llm else 'Enabled'}") + print(f" GitHub Token: {'Provided' if github_token else 'Not provided (mock data)'}") + print() + + # Run pipeline + try: + result = run_work_report_pipeline( + repos=repos, + branch=args.branch, + days=args.days, + output_format=args.format, + output_path=args.output, + diary_path=args.diary_path, + language=args.language, + github_token=github_token, + verbose=args.verbose, + use_llm=not args.no_llm, + include_submodules=not args.no_submodules, + ) + + if result: + print(f"\n Report saved to: {result}") + print("\nTo view the report:") + print(f" cat {result}") + + except KeyboardInterrupt: + print("\n\n Pipeline interrupted by user.") + sys.exit(1) + except Exception as e: + logger.error(f"Pipeline failed: {e}") + if args.verbose: + import traceback + + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/data b/data new file mode 120000 index 0000000..33923d8 --- /dev/null +++ b/data @@ -0,0 +1 @@ +tutorials/agents/data \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..874fce7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,42 @@ +# Requirements for SAGE Examples +# +# This file contains dependencies for running examples in sage-examples repository. +# +# Installation: +# pip install -r requirements.txt +# +# For development, also install sage-apps package: +# cd sage-apps && pip install -e .[dev] + +# === SAGE Core Packages (from PyPI) === +isage-common>=0.2.0 # L1: Foundation utilities and config +isage-llm-core>=0.2.0 # L1: LLM control plane and UnifiedInferenceClient +isage-libs>=0.2.0 # L3: RAG, Agents, and algorithms + +# === Optional SAGE Packages === +# Uncomment if needed for advanced examples +# isage-middleware>=0.2.0 # L4: Operators and middleware (C++ extensions) +# isage-vdb>=0.2.0 # Vector database +# isage-benchmark>=0.2.0 # Benchmark tools + +# === Application Dependencies === +# These are needed for apps/ examples +pyyaml>=6.0 # Config files +pillow>=10.0.0 # Image processing +numpy>=1.26.0,<2.3.0 # Array operations + +# Video Intelligence Examples (apps/run_video_intelligence.py) +opencv-python-headless>=4.5.0 # Video processing (headless to avoid GUI deps) +torch>=2.7.0,<3.0.0 # PyTorch for models +torchvision>=0.22.0,<1.0.0 # Vision models +transformers>=4.52.0,<4.54.0 # CLIP and transformers (compatible with vllm 0.9.2) + +# Medical Diagnosis Examples (apps/run_medical_diagnosis.py) +scikit-learn>=1.3.0 # ML utilities + +# === Development Dependencies (Optional) === +# Uncomment for development work +# pytest>=7.0.0 +# pytest-cov>=4.0.0 +# ruff==0.14.6 +# mypy>=1.0.0 diff --git a/sage-apps/README.md b/sage-apps/README.md new file mode 100644 index 0000000..ab5a580 --- /dev/null +++ b/sage-apps/README.md @@ -0,0 +1,217 @@ +# SAGE Applications + +Real-world AI applications built on the SAGE framework, showcasing end-to-end solutions for various +domains. + +## Overview + +`isage-apps` provides production-ready applications demonstrating SAGE's capabilities: + +- **Video Intelligence**: Multi-model video analysis pipeline with CLIP and MobileNetV3 +- **Medical Diagnosis**: AI-assisted medical image analysis for healthcare + +## Installation + +### Basic Installation + +```bash +pip install isage-apps +``` + +### Install with Specific Applications + +```bash +# Video intelligence only +pip install isage-apps[video] + +# Medical diagnosis only +pip install isage-apps[medical] + +# All applications +pip install isage-apps[all] +``` + +### Development Installation + +```bash +cd packages/sage-apps +pip install -e ".[dev]" +``` + +## 📖 Quick Start + +```bash +# Run video intelligence demo +pip install isage-apps[video] +python -m sage.apps.video.video_intelligence_pipeline --video path/to/video.mp4 + +# Run medical diagnosis demo +pip install isage-apps[medical] +python -m sage.apps.medical_diagnosis.run_diagnosis +``` + +## Applications + +### 1. Video Intelligence + +Advanced video analysis pipeline combining multiple AI models: + +- **Frame sampling and preprocessing** +- **Zero-shot scene understanding** (CLIP) +- **Object classification** (MobileNetV3) +- **Temporal anomaly detection** +- **Sliding-window summarization** +- **Keyed event aggregation** + +**Quick Start:** + +```bash +pip install isage-apps[video] +python -m sage.apps.video.video_intelligence_pipeline --video path/to/video.mp4 +``` + +**Features:** + +- Multi-model inference pipeline +- Real-time processing with SAGE operators +- Structured JSON output (timeline, summary, events) +- Console progress monitoring +- Graceful degradation (works offline with cached models) + +**Documentation:** See `sage/apps/video/README_intelligence_demo.md` + +### 2. Medical Diagnosis + +AI-assisted diagnostic system for medical imaging: + +- **Multi-agent architecture** (diagnostic, image analysis, report generation) +- **Knowledge-based reasoning** +- **Structured medical reports** +- **Training and evaluation tools** + +**Quick Start:** + +```bash +pip install isage-apps[medical] +python -m sage.apps.medical_diagnosis.run_diagnosis +``` + +**Features:** + +- Agent-based diagnostic workflow +- Medical knowledge base integration +- Configurable diagnostic criteria +- Report generation + +**Documentation:** See `sage/apps/medical_diagnosis/README.md` + +## Package Structure + +``` +sage-apps/ +├── src/sage/apps/ +│ ├── __init__.py +│ ├── video/ # Video intelligence application +│ │ ├── video_intelligence_pipeline.py +│ │ ├── operators/ # SAGE operators for video +│ │ ├── config/ # Configuration files +│ │ └── README_intelligence_demo.md +│ └── medical_diagnosis/ # Medical diagnosis application +│ ├── run_diagnosis.py +│ ├── agents/ # Diagnostic agents +│ ├── config/ # Agent configurations +│ ├── data/ # Medical datasets +│ └── README.md +└── tests/ # Application tests +``` + +## Dependencies + +### Core Framework + +- `isage-common` - Common utilities +- `isage-kernel` - Runtime and operators +- `isage-middleware` - Services (SageVDB, SageFlow, NeuroMem) +- `isage-libs` - Operator libraries + +### Application-Specific + +**Video Intelligence:** + +- `opencv-python` - Video processing +- `torch` - Deep learning +- `transformers` - CLIP and language models + +**Medical Diagnosis:** + +- `pillow` - Image processing +- `scikit-learn` - ML utilities + +## Usage Examples + +### Video Intelligence + +```python +from sage.apps.video.video_intelligence_pipeline import main + +# Run with custom video +main(["--video", "my_video.mp4", "--max-frames", "100"]) +``` + +### Medical Diagnosis + +```python +from sage.apps.medical_diagnosis.run_diagnosis import run_diagnosis + +# Run diagnostic pipeline +run_diagnosis(config_path="config/agent_config.yaml") +``` + +## Development + +### Running Tests + +```bash +pytest tests/ +``` + +### Code Quality + +```bash +# Format code +black src/ + +# Lint +ruff check src/ + +# Type checking +mypy src/ +``` + +## CI/CD Notes + +**Video Intelligence:** + +- Requires HuggingFace model downloads (~170MB) +- Tagged with `@test:skip` in CI due to network restrictions +- Test locally with: `python -m sage.apps.video.video_intelligence_pipeline` + +**Medical Diagnosis:** + +- Works in CI (uses local data) +- Test with: `pytest tests/test_medical_diagnosis.py` + +## Contributing + +See [CONTRIBUTING.md](../../CONTRIBUTING.md) for guidelines. + +## License + +MIT License - see [LICENSE](../../LICENSE) for details. + +## Related Documentation + +- [SAGE Framework](../../README.md) +- [Video Intelligence Demo](src/sage/apps/video/README_intelligence_demo.md) +- [Medical Diagnosis](src/sage/apps/medical_diagnosis/README.md) +- [CI Test Fix](src/sage/apps/video/CI_TEST_FIX.md) diff --git a/sage-apps/pyproject.toml b/sage-apps/pyproject.toml new file mode 100644 index 0000000..7018553 --- /dev/null +++ b/sage-apps/pyproject.toml @@ -0,0 +1,184 @@ +[build-system] +requires = [ + "setuptools>=64", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "isage-apps" +dynamic = [ + "version", +] +description = "SAGE Applications - Real-world AI applications built on SAGE framework" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "IntelliStream Team", email = "shuhao_zhang@hust.edu.cn" }, +] +keywords = [ + "sage", + "applications", + "ai", + "video-intelligence", + "medical-diagnosis", + "computer-vision", + "healthcare", + "intellistream", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: Healthcare Industry", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Image Recognition", + "Topic :: Scientific/Engineering :: Medical Science Apps.", + "Topic :: Multimedia :: Video", +] +dependencies = [ + # 外部依赖 + "pyyaml>=6.0", + "pillow>=10.0.0", + "numpy>=1.26.0,<2.3.0", + # Video 应用依赖 + # 使用 headless 版本避免系统 GUI 依赖 (libGL, libglib 等) + "opencv-python-headless>=4.5.0", + "torch>=2.7.0,<3.0.0", + "torchvision>=0.22.0,<1.0.0", + "transformers>=4.52.0,<4.54.0", # 兼容 vllm 0.9.2 (避免 aimv2 配置冲突) + # Medical 应用依赖 + "scikit-learn>=1.3.0", +] + +license = {text = "MIT"} + +[project.optional-dependencies] +# SAGE 内部依赖 +sage-deps = [ + "isage-common>=0.2.0", + "isage-kernel>=0.1.0", + "isage-middleware>=0.1.0", +] +video = [ + "opencv-python-headless>=4.5.0", + "torch>=2.7.0,<3.0.0", + "torchvision>=0.22.0,<1.0.0", + "transformers>=4.52.0,<4.54.0", # 兼容 vllm 0.9.2 (避免 aimv2 配置冲突) +] +medical = [ + "pillow>=10.0.0", + "scikit-learn>=1.3.0", + "pydicom>=2.3.0", +] +all = [ + "isage-apps[sage-deps,video,medical]", +] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "ruff==0.14.6", # Pinned to match tools/pre-commit-config.yaml + "mypy>=1.0.0", +] + +[project.urls] +Homepage = "https://github.com/intellistream/SAGE" +Documentation = "https://intellistream.github.io/SAGE" +Repository = "https://github.com/intellistream/SAGE" +Issues = "https://github.com/intellistream/SAGE/issues" + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["sage*"] +namespaces = true + +[tool.setuptools.dynamic.version] +attr = "sage.apps._version.__version__" + +[tool.setuptools.package-data] +"sage.apps" = [ + "py.typed", +] +"sage.apps.video" = [ + "config/*.yaml", + "data/**/*", +] +"sage.apps.medical_diagnosis" = [ + "config/*.yaml", + "data/**/*", +] + +[tool.black] +line-length = 100 +target-version = [ + "py310", + "py311", +] +include = "\\.pyi?$" + +[tool.mypy] +python_version = "3.10" +cache_dir = "../../.sage/cache/mypy" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.ruff] +extend = "../../tools/ruff.toml" + +[tool.pytest.ini_options] +testpaths = [ + "tests", + "src", +] +python_files = [ + "test_*.py", + "*_test.py", +] +python_classes = [ + "Test*", +] +python_functions = [ + "test_*", +] +addopts = [ + "--benchmark-storage=../../.sage/benchmarks", + "-o", + "cache_dir=../../.sage/cache/pytest", + "--strict-markers", + "--strict-config", + "--verbose", + "-ra", +] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "integration: marks tests as integration tests", + "unit: marks tests as unit tests", + "video: marks tests as video intelligence tests", + "medical: marks tests as medical diagnosis tests", +] + +[tool.coverage.run] +source = [ + "src/sage", +] +omit = [ + "*/tests/*", + "*/test_*.py", + "*/_test_*.py", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", +] diff --git a/sage-apps/src/sage/apps/__init__.py b/sage-apps/src/sage/apps/__init__.py new file mode 100644 index 0000000..d0176de --- /dev/null +++ b/sage-apps/src/sage/apps/__init__.py @@ -0,0 +1,29 @@ +"""SAGE Applications - Real-world AI applications built on SAGE framework. + +Layer: L5 (Applications) +Dependencies: sage.middleware (L4), sage.libs (L3), sage.kernel (L3), sage.platform (L2), sage.common (L1) + +This package contains production-ready applications demonstrating SAGE's +capabilities across various domains: + +- video: Video intelligence and analysis +- medical_diagnosis: AI-assisted medical imaging diagnosis +- work_report_generator: Weekly/daily work report generator with GitHub integration + +Architecture: +- L5 应用层，组合使用下层功能构建完整应用 +- 依赖 L4 领域组件和 L3 核心引擎 +- 提供端到端的应用解决方案 +""" + +__layer__ = "L5" + +from . import medical_diagnosis, video, work_report_generator +from ._version import __version__ + +__all__ = [ + "__version__", + "medical_diagnosis", + "video", + "work_report_generator", +] diff --git a/sage-apps/src/sage/apps/_version.py b/sage-apps/src/sage/apps/_version.py new file mode 100644 index 0000000..4eee951 --- /dev/null +++ b/sage-apps/src/sage/apps/_version.py @@ -0,0 +1,6 @@ +"""Version information for sage-apps package.""" + +# 独立硬编码版本 +__version__ = "0.2.0.0" +__author__ = "IntelliStream Team" +__email__ = "shuhao_zhang@hust.edu.cn" diff --git a/sage-apps/src/sage/apps/article_monitoring/README.md b/sage-apps/src/sage/apps/article_monitoring/README.md new file mode 100644 index 0000000..26915a0 --- /dev/null +++ b/sage-apps/src/sage/apps/article_monitoring/README.md @@ -0,0 +1,176 @@ +# Article Monitoring System + +An intelligent article monitoring system built on SAGE framework that continuously fetches papers +from arXiv, performs multi-level filtering, and provides personalized recommendations. + +## Features + +- **Real-time Data Stream Processing**: Built on SAGE operators for efficient stream processing +- **Multi-level Filtering**: + - Keyword-based filtering using bag-of-words matching + - Semantic filtering using text similarity +- **Personalized Recommendations**: Ranks articles based on user interests +- **SAGE Integration**: Uses BatchFunction, MapFunction, and SinkFunction +- **Configurable**: Easy to customize keywords and topics + +## Quick Start + +```python +from sage.apps.article_monitoring import run_article_monitoring_pipeline + +# Run with default settings +run_article_monitoring_pipeline() + +# Or customize parameters +run_article_monitoring_pipeline( + keywords=["machine learning", "deep learning", "neural network"], + interest_topics=[ + "artificial intelligence and machine learning applications", + "natural language processing and text analysis" + ], + category="cs.AI", + max_articles=20 +) +``` + +## Command Line Usage + +```bash +# Use default settings +python -m sage.apps.article_monitoring.pipeline + +# Custom keywords and topics +python -m sage.apps.article_monitoring.pipeline \ + --keywords "transformer,attention,bert" \ + --topics "natural language processing and transformers" \ + --category cs.CL \ + --max-articles 30 + +# Verbose mode +python -m sage.apps.article_monitoring.pipeline --verbose +``` + +## Use Cases + +- Academic researchers tracking papers in their field +- Research teams monitoring specific topics +- Students discovering relevant publications +- Literature review automation + +## Configuration + +### Categories + +Common arXiv categories: + +- `cs.AI` - Artificial Intelligence +- `cs.LG` - Machine Learning +- `cs.CL` - Computation and Language +- `cs.CV` - Computer Vision +- `stat.ML` - Machine Learning (Statistics) + +### Filtering Parameters + +- `keywords`: List of keywords for initial filtering +- `interest_topics`: List of topics for semantic analysis +- `max_articles`: Maximum number of articles to fetch per run + +## Architecture + +The pipeline consists of three main stages: + +1. **Article Fetching**: Retrieves recent papers from arXiv API +1. **Keyword Filtering**: Filters based on exact keyword matches +1. **Semantic Filtering**: Further filters using semantic similarity + +``` +arXiv API → Fetch Articles → Keyword Filter → Semantic Filter → Ranked Results +``` + +## Example Output + +``` +====================================================================== +SAGE Article Monitoring System +====================================================================== +Category: cs.AI +Keywords: ['machine learning', 'deep learning'] +Interest Topics: ['artificial intelligence applications'] +====================================================================== + +## Pipeline Architecture + +The system uses SAGE stream processing operators: + +``` + +ArxivSource (BatchFunction) ↓ KeywordFilter (MapFunction) ↓ SemanticFilter (MapFunction) ↓ +ArticleScorer (MapFunction) ↓ ArticleRankingSink (SinkFunction) + +``` + +### Operators + +- **ArxivSource**: Fetches articles from arXiv API +- **KeywordFilter**: Filters based on keyword matching +- **SemanticFilter**: Filters based on semantic similarity (Jaccard) +- **ArticleScorer**: Calculates final ranking scores +- **ArticleRankingSink**: Collects and displays ranked results + +## Example Output + +``` + +# ====================================================================== 🔍 SAGE Article Monitoring System + +# Category: cs.AI Max Articles: 10 Keywords: ['machine learning', 'deep learning', 'neural network'] Interest Topics: artificial intelligence and machine... + +📡 Starting pipeline... + +# ====================================================================== 📚 Recommended Articles (8 found) + +1. Deep Learning for Time Series Forecasting in Stream Processing Authors: John Doe, Jane Smith + Score: 3.45 (keyword: 3.0, semantic: 0.45) URL: http://arxiv.org/abs/2401.00001 Abstract: We + propose a novel deep learning approach for time series... + +1. Machine Learning Pipeline Optimization Authors: David Lee Score: 2.30 (keyword: 2.0, semantic: + 0.30) URL: http://arxiv.org/abs/2401.00004 Abstract: Optimizing machine learning pipelines for + better performance... + +# ====================================================================== ✅ Pipeline completed in 2.34s Recommended 8 articles + +``` +``` + +## Testing + +Run the example script: + +```bash +python examples/apps/run_article_monitoring.py +``` + +Or with custom parameters: + +```bash +python examples/apps/run_article_monitoring.py \ + --keywords "transformer,attention,bert" \ + --topics "natural language processing" \ + --category cs.CL \ + --max-articles 30 +``` + +## Dependencies + +- Python 3.8+ +- Standard library only (no external dependencies for basic functionality) +- Optional: SAGE framework for advanced stream processing + +## Future Enhancements + +- Integration with embedding models for true semantic search +- User feedback learning +- Multi-source support (beyond arXiv) +- Email/Slack notifications +- Web dashboard +- Persistent storage with SageDB diff --git a/sage-apps/src/sage/apps/article_monitoring/__init__.py b/sage-apps/src/sage/apps/article_monitoring/__init__.py new file mode 100644 index 0000000..4efca59 --- /dev/null +++ b/sage-apps/src/sage/apps/article_monitoring/__init__.py @@ -0,0 +1,20 @@ +""" +Article Monitoring System + +An intelligent article monitoring system that continuously fetches papers from arXiv, +performs multi-level filtering (keyword-based and semantic), and provides personalized +recommendations to users. + +Built on SAGE framework using stream processing operators. + +Features: +- Real-time article fetching from arXiv +- Keyword-based filtering using bag-of-words +- Semantic filtering using text similarity +- Personalized recommendations +- Stream processing with SAGE operators +""" + +from .pipeline import run_article_monitoring_pipeline + +__all__ = ["run_article_monitoring_pipeline"] diff --git a/sage-apps/src/sage/apps/article_monitoring/operators.py b/sage-apps/src/sage/apps/article_monitoring/operators.py new file mode 100644 index 0000000..453a1ef --- /dev/null +++ b/sage-apps/src/sage/apps/article_monitoring/operators.py @@ -0,0 +1,328 @@ +""" +SAGE Operators for Article Monitoring System + +Implements source, map, filter, and sink operators for the article monitoring pipeline. +""" + +from __future__ import annotations + +import re +import time +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from typing import Any + +from sage.common.core import BatchFunction, MapFunction, SinkFunction + + +@dataclass +class Article: + """Article metadata from arXiv.""" + + id: str + title: str + authors: list[str] + abstract: str + published: str + categories: list[str] + url: str + keyword_score: float = 0.0 + semantic_score: float = 0.0 + total_score: float = 0.0 + + +class ArxivSource(BatchFunction): + """ + BatchFunction source that fetches articles from arXiv API. + + Emits one article at a time as a data packet. + """ + + def __init__( + self, + category: str = "cs.AI", + max_results: int = 10, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.category = category + self.max_results = max_results + self.articles: list[dict[str, Any]] = [] + self.index = 0 + self.fetched = False + + def _fetch_articles(self) -> list[dict[str, Any]]: + """Fetch articles from arXiv API.""" + query = f"cat:{self.category}" + params = { + "search_query": query, + "start": 0, + "max_results": self.max_results, + "sortBy": "submittedDate", + "sortOrder": "descending", + } + + url = f"http://export.arxiv.org/api/query?{urllib.parse.urlencode(params)}" + + try: + with urllib.request.urlopen(url, timeout=10) as response: + data = response.read().decode("utf-8") + return self._parse_arxiv_response(data) + except Exception as e: + self.logger.warning(f"Error fetching from arXiv: {e}, using mock data") + return self._get_mock_articles() + + def _parse_arxiv_response(self, xml_data: str) -> list[dict[str, Any]]: + """Parse arXiv API XML response.""" + articles = [] + root = ET.fromstring(xml_data) + ns = {"atom": "http://www.w3.org/2005/Atom"} + + for entry in root.findall("atom:entry", ns): + try: + id_elem = entry.find("atom:id", ns) + title_elem = entry.find("atom:title", ns) + abstract_elem = entry.find("atom:summary", ns) + published_elem = entry.find("atom:published", ns) + + if not all([id_elem, title_elem, abstract_elem, published_elem]): + continue + + article_id = id_elem.text.split("/")[-1] # type: ignore + title = title_elem.text.strip() # type: ignore + abstract = abstract_elem.text.strip() # type: ignore + published = published_elem.text # type: ignore + url = id_elem.text # type: ignore + + authors = [ + author.find("atom:name", ns).text # type: ignore + for author in entry.findall("atom:author", ns) + if author.find("atom:name", ns) is not None + ] + + categories = [cat.attrib["term"] for cat in entry.findall("atom:category", ns)] + + articles.append( + { + "id": article_id, + "title": title, + "authors": authors, + "abstract": abstract, + "published": published, + "categories": categories, + "url": url, + } + ) + except Exception as e: + self.logger.warning(f"Error parsing entry: {e}") + continue + + return articles + + def _get_mock_articles(self) -> list[dict[str, Any]]: + """Generate mock articles for testing/offline mode.""" + return [ + { + "id": "2401.00001", + "title": "Deep Learning for Time Series Forecasting in Stream Processing", + "authors": ["John Doe", "Jane Smith"], + "abstract": "We propose a novel deep learning approach for time series forecasting in distributed stream processing systems...", + "published": "2024-01-01T00:00:00Z", + "categories": ["cs.LG", "cs.AI"], + "url": "http://arxiv.org/abs/2401.00001", + }, + { + "id": "2401.00002", + "title": "Semantic Analysis in Natural Language Processing", + "authors": ["Alice Johnson"], + "abstract": "This paper explores advanced semantic analysis techniques for NLP tasks using transformer models...", + "published": "2024-01-01T01:00:00Z", + "categories": ["cs.CL", "cs.AI"], + "url": "http://arxiv.org/abs/2401.00002", + }, + { + "id": "2401.00003", + "title": "Distributed Systems for Real-time Stream Processing", + "authors": ["Bob Chen", "Carol White"], + "abstract": "We present a distributed architecture for processing data streams in real-time with low latency...", + "published": "2024-01-01T02:00:00Z", + "categories": ["cs.DC", "cs.DB"], + "url": "http://arxiv.org/abs/2401.00003", + }, + { + "id": "2401.00004", + "title": "Machine Learning Pipeline Optimization", + "authors": ["David Lee"], + "abstract": "Optimizing machine learning pipelines for better performance and resource utilization in production systems...", + "published": "2024-01-01T03:00:00Z", + "categories": ["cs.LG", "cs.AI"], + "url": "http://arxiv.org/abs/2401.00004", + }, + { + "id": "2401.00005", + "title": "Graph Neural Networks for Social Network Analysis", + "authors": ["Emma Wilson", "Frank Zhang"], + "abstract": "Applying graph neural networks to analyze complex social network structures and dynamics...", + "published": "2024-01-01T04:00:00Z", + "categories": ["cs.SI", "cs.LG"], + "url": "http://arxiv.org/abs/2401.00005", + }, + ] + + def execute(self) -> dict[str, Any] | None: + """Execute batch function to emit one article at a time.""" + # Fetch articles on first execution + if not self.fetched: + self.logger.info(f"Fetching articles from arXiv (category: {self.category})...") + self.articles = self._fetch_articles() + self.fetched = True + self.logger.info(f"Fetched {len(self.articles)} articles") + + # Emit one article per call + if self.index < len(self.articles): + article = self.articles[self.index] + self.index += 1 + return article + + # All articles emitted + return None + + +class KeywordFilter(MapFunction): + """ + MapFunction that filters articles based on keyword matching. + + Returns the article with keyword_score if it passes, otherwise returns None. + """ + + def __init__(self, keywords: list[str], min_score: float = 1.0, **kwargs) -> None: + super().__init__(**kwargs) + self.keywords = [kw.lower() for kw in keywords] + self.min_score = min_score + + def execute(self, data: dict[str, Any]) -> dict[str, Any] | None: + """Filter article based on keyword matching.""" + text = (data.get("title", "") + " " + data.get("abstract", "")).lower() + score = sum(1 for kw in self.keywords if kw in text) + + if score >= self.min_score: + data["keyword_score"] = float(score) + return data + return None + + +class SemanticFilter(MapFunction): + """ + MapFunction that filters articles based on semantic similarity. + + Uses simplified word overlap as a proxy for semantic similarity. + """ + + def __init__(self, interest_topics: list[str], min_similarity: float = 0.05, **kwargs) -> None: + super().__init__(**kwargs) + self.interest_topics = [topic.lower() for topic in interest_topics] + self.min_similarity = min_similarity + + def execute(self, data: dict[str, Any]) -> dict[str, Any] | None: + """Filter article based on semantic similarity.""" + text = (data.get("title", "") + " " + data.get("abstract", "")).lower() + text_words = set(re.findall(r"\w+", text)) + + max_similarity = 0.0 + for topic in self.interest_topics: + topic_words = set(re.findall(r"\w+", topic)) + if not topic_words: + continue + + # Simple Jaccard similarity + intersection = len(text_words.intersection(topic_words)) + union = len(text_words.union(topic_words)) + similarity = intersection / union if union > 0 else 0.0 + max_similarity = max(max_similarity, similarity) + + if max_similarity >= self.min_similarity: + data["semantic_score"] = float(max_similarity * 10) # Scale for display + data["total_score"] = data.get("keyword_score", 0) + data["semantic_score"] + return data + return None + + +class ArticleScorer(MapFunction): + """ + MapFunction that calculates and adds final score to articles. + """ + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Calculate total score for ranking.""" + keyword_score = data.get("keyword_score", 0.0) + semantic_score = data.get("semantic_score", 0.0) + data["total_score"] = keyword_score + semantic_score + return data + + +class ArticleRankingSink(SinkFunction): + """ + SinkFunction that collects and displays ranked articles. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.articles: list[dict[str, Any]] = [] + self.start_time = time.time() + + def execute(self, data: dict[str, Any]) -> None: + """Collect article for final ranking.""" + self.articles.append(data) + + def close(self) -> None: + """Display ranked results when pipeline completes.""" + elapsed = time.time() - self.start_time + + # Sort by total score + self.articles.sort(key=lambda x: x.get("total_score", 0), reverse=True) + + print("\n" + "=" * 70) + print(f"📚 Recommended Articles ({len(self.articles)} found)") + print("=" * 70) + + for i, article in enumerate(self.articles, 1): + print(f"\n{i}. {article.get('title', 'Unknown')}") + authors = article.get("authors", []) + if authors: + print(f" Authors: {', '.join(authors[:3])}") + if len(authors) > 3: + print(f" (+{len(authors) - 3} more)") + print( + f" Score: {article.get('total_score', 0):.2f} " + f"(keyword: {article.get('keyword_score', 0):.1f}, " + f"semantic: {article.get('semantic_score', 0):.2f})" + ) + print(f" URL: {article.get('url', 'N/A')}") + abstract = article.get("abstract", "") + if abstract: + preview = abstract[:150] + "..." if len(abstract) > 150 else abstract + print(f" Abstract: {preview}") + + print("\n" + "=" * 70) + print(f"✅ Pipeline completed in {elapsed:.2f}s") + print(f" Recommended {len(self.articles)} articles") + print("=" * 70 + "\n") + + +class ArticleLogSink(SinkFunction): + """ + SinkFunction that logs each processed article. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.count = 0 + + def execute(self, data: dict[str, Any]) -> None: + """Log article processing.""" + self.count += 1 + title = data.get("title", "Unknown") + score = data.get("total_score", 0) + self.logger.info(f"[{self.count}] Processed: {title[:50]}... (score: {score:.2f})") diff --git a/sage-apps/src/sage/apps/article_monitoring/pipeline.py b/sage-apps/src/sage/apps/article_monitoring/pipeline.py new file mode 100644 index 0000000..32d0c94 --- /dev/null +++ b/sage-apps/src/sage/apps/article_monitoring/pipeline.py @@ -0,0 +1,189 @@ +""" +Article Monitoring Pipeline + +Main pipeline implementation using SAGE operators for article monitoring. +""" + +from __future__ import annotations + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + +from .operators import ( + ArticleLogSink, + ArticleRankingSink, + ArticleScorer, + ArxivSource, + KeywordFilter, + SemanticFilter, +) + + +def run_article_monitoring_pipeline( + keywords: list[str] | None = None, + interest_topics: list[str] | None = None, + category: str = "cs.AI", + max_articles: int = 10, + verbose: bool = False, +) -> None: + """ + Run the article monitoring pipeline using SAGE framework. + + Args: + keywords: List of keywords for initial filtering + interest_topics: List of topics for semantic filtering + category: arXiv category to monitor + max_articles: Maximum number of articles to fetch + verbose: Enable verbose logging + + Example: + >>> run_article_monitoring_pipeline( + ... keywords=["machine learning", "deep learning"], + ... interest_topics=["artificial intelligence applications"], + ... category="cs.AI", + ... max_articles=20 + ... ) + """ + # Default parameters + if keywords is None: + keywords = [ + "machine learning", + "deep learning", + "neural network", + "stream processing", + ] + + if interest_topics is None: + interest_topics = [ + "artificial intelligence and machine learning applications", + "data stream processing and real-time systems", + "distributed computing and scalability", + ] + + print("=" * 70) + print("🔍 SAGE Article Monitoring System") + print("=" * 70) + print(f"Category: {category}") + print(f"Max Articles: {max_articles}") + print(f"Keywords: {keywords}") + print( + f"Interest Topics: {', '.join(t[:40] + '...' if len(t) > 40 else t for t in interest_topics)}" + ) + print("=" * 70) + print() + + # Create SAGE environment + env = LocalEnvironment("article_monitoring") + + # Build pipeline: + # 1. ArxivSource: Fetch articles from arXiv + # 2. KeywordFilter: Filter by keywords + # 3. SemanticFilter: Filter by semantic similarity + # 4. ArticleScorer: Calculate final scores + # 5. ArticleRankingSink: Collect and display results + pipeline = ( + env.from_batch( + ArxivSource, + category=category, + max_results=max_articles, + ) + .map(KeywordFilter, keywords=keywords, min_score=1.0) + .map(SemanticFilter, interest_topics=interest_topics, min_similarity=0.05) + .map(ArticleScorer) + .sink(ArticleRankingSink) + ) + + # Optionally add logging sink + if verbose: + pipeline.sink(ArticleLogSink) + + # Execute pipeline + print("📡 Starting pipeline...") + print() + env.submit(autostop=True) + + +def main(): + """Main entry point for command-line usage.""" + import argparse + + parser = argparse.ArgumentParser( + description="SAGE Article Monitoring System - Monitor and filter arXiv papers", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Use default settings + python -m sage.apps.article_monitoring.pipeline + + # Custom keywords and topics + python -m sage.apps.article_monitoring.pipeline \\ + --keywords "transformer,attention,bert" \\ + --topics "natural language processing and transformers" \\ + --category cs.CL \\ + --max-articles 30 + + # Verbose mode + python -m sage.apps.article_monitoring.pipeline --verbose + """, + ) + + parser.add_argument( + "--keywords", + "-k", + type=str, + help="Comma-separated list of keywords (default: machine learning,deep learning,neural network)", + ) + + parser.add_argument( + "--topics", + "-t", + type=str, + help="Comma-separated list of interest topics", + ) + + parser.add_argument( + "--category", + "-c", + type=str, + default="cs.AI", + help="arXiv category to monitor (default: cs.AI)", + ) + + parser.add_argument( + "--max-articles", + "-m", + type=int, + default=10, + help="Maximum number of articles to fetch (default: 10)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Parse keywords + keywords = None + if args.keywords: + keywords = [kw.strip() for kw in args.keywords.split(",")] + + # Parse topics + topics = None + if args.topics: + topics = [t.strip() for t in args.topics.split(",")] + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + # Run pipeline + run_article_monitoring_pipeline( + keywords=keywords, + interest_topics=topics, + category=args.category, + max_articles=args.max_articles, + verbose=args.verbose, + ) + + +if __name__ == "__main__": + main() diff --git a/sage-apps/src/sage/apps/auto_scaling_chat/README.md b/sage-apps/src/sage/apps/auto_scaling_chat/README.md new file mode 100644 index 0000000..f96de31 --- /dev/null +++ b/sage-apps/src/sage/apps/auto_scaling_chat/README.md @@ -0,0 +1,193 @@ +# Auto-Scaling Chat System + +An intelligent auto-scaling chat system built on SAGE framework demonstrating elastic resource +management and load balancing capabilities. + +## Features + +- **Simulated User Traffic**: Variable load patterns (ramp up, peak, ramp down) +- **Automatic Scaling**: Dynamic server scaling based on load thresholds +- **Load Balancing**: Round-robin distribution across available servers +- **Resource Monitoring**: Real-time metrics and performance tracking +- **SAGE Integration**: Built with BatchFunction, MapFunction, SinkFunction +- **Configurable Policies**: Customizable scaling thresholds and limits + +## Quick Start + +```python +from sage.apps.auto_scaling_chat import run_auto_scaling_demo + +# Run default simulation +run_auto_scaling_demo() + +# Custom load profile +run_auto_scaling_demo( + duration=60, # seconds + base_rate=5, # min concurrent users + peak_rate=80, # max concurrent users + verbose=True # show scaling events +) +``` + +## Command Line Usage + +```bash +# Default simulation (30s) +python -m sage.apps.auto_scaling_chat.pipeline + +# Custom parameters +python -m sage.apps.auto_scaling_chat.pipeline \ + --duration 60 \ + --base-rate 10 \ + --peak-rate 80 \ + --verbose + +# Quick test +python -m sage.apps.auto_scaling_chat.pipeline --duration 15 --peak-rate 40 +``` + +## Pipeline Architecture + +The system uses SAGE stream processing operators: + +``` +UserTrafficSource (BatchFunction) + ↓ +AutoScaler (MapFunction) + ↓ +LoadBalancer (MapFunction) + ↓ +RequestProcessor (MapFunction) + ↓ +MetricsCollector (SinkFunction) + ↓ +ScalingEventsSink (SinkFunction) [optional] +``` + +### Operators + +- **UserTrafficSource**: Generates simulated user requests with varying load patterns +- **AutoScaler**: Makes scaling decisions based on load/server ratio +- **LoadBalancer**: Distributes requests across available servers (round-robin) +- **RequestProcessor**: Simulates request processing on servers +- **MetricsCollector**: Collects and displays performance metrics +- **ScalingEventsSink**: Logs all scaling events for analysis + +## Scaling Policy + +Default thresholds: + +- **Scale UP**: When average load per server > 30 users +- **Scale DOWN**: When average load per server < 10 users +- **Min servers**: 2 +- **Max servers**: 10 +- **Cooldown**: 5 seconds between scaling actions + +## Example Output + +``` +⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ + SAGE Auto-Scaling Chat System Demo +⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡⚡ + +====================================================================== +📋 Configuration +====================================================================== + Simulation duration: 30s + Base load: 5 concurrent users + Peak load: 50 concurrent users + Verbose logging: True +====================================================================== + +🚀 Starting auto-scaling simulation... + +⚠️ Scaling Threshold Reached! Load: 35, Scaling up to 3 servers +📊 Load: 20 users | Servers: 2 | Requests: 10 +📊 Load: 35 users | Servers: 3 | Requests: 20 +⚠️ Scaling Threshold Reached! Load: 48, Scaling up to 4 servers +📊 Load: 50 users | Servers: 4 | Requests: 30 + +====================================================================== +📊 Auto-Scaling System Metrics +====================================================================== + Total requests processed: 245 + Total duration: 30.15s + Average throughput: 8.13 req/s + Peak load: 50 concurrent users + Peak servers: 4 + Scaling events: 5 + Average load: 28.3 users + Average servers: 3.2 +====================================================================== + +🔄 Scaling Events: + 1. 📈 SCALE_UP: 3 servers (load: 35) + 2. 📈 SCALE_UP: 4 servers (load: 48) + 3. 📉 SCALE_DOWN: 3 servers (load: 18) + 4. 📉 SCALE_DOWN: 2 servers (load: 8) +``` + +## Use Cases + +- **Cloud Applications**: Elastic scaling for web applications +- **Chat Systems**: Handle variable user loads efficiently +- **API Gateways**: Auto-scale based on request rates +- **Gaming Servers**: Match-making and lobby systems +- **Streaming Services**: Handle peak viewing times + +## Configuration Options + +Customize scaling behavior: + +```python +from sage.apps.auto_scaling_chat.operators import AutoScaler + +# Custom scaler with different thresholds +scaler = AutoScaler( + scale_up_threshold=25, # Scale up earlier + scale_down_threshold=5, # Scale down more aggressively + min_servers=1, # Allow single server + max_servers=20, # Allow more servers +) +``` + +## Testing + +Run the example script: + +```bash +python examples/apps/run_auto_scaling_chat.py +``` + +Or run tests: + +```bash +python -m pytest packages/sage-apps/tests/auto_scaling_chat/ -v +``` + +## Performance Metrics + +The system tracks: + +- Total requests processed +- Average throughput (requests/second) +- Peak concurrent load +- Server utilization +- Scaling event frequency +- Average response time + +## Dependencies + +- Python 3.8+ +- SAGE framework (sage-common, sage-kernel) +- Standard library only + +## Future Enhancements + +- Integration with real cloud providers (AWS, GCP, Azure) +- Predictive scaling using machine learning +- Cost optimization algorithms +- Multi-region deployment +- Advanced load balancing (weighted, least-connections) +- Health checks and failure recovery +- Integration with monitoring systems (Prometheus, Grafana) diff --git a/sage-apps/src/sage/apps/auto_scaling_chat/__init__.py b/sage-apps/src/sage/apps/auto_scaling_chat/__init__.py new file mode 100644 index 0000000..9b8b298 --- /dev/null +++ b/sage-apps/src/sage/apps/auto_scaling_chat/__init__.py @@ -0,0 +1,20 @@ +""" +Auto-Scaling Chat System + +An intelligent chat system demonstrating SAGE's resource optimization capabilities +through automatic scaling based on user load. Shows cloud infrastructure elasticity +with load balancing and resource monitoring. + +Built on SAGE framework using stream processing operators. + +Features: +- Simulated user traffic generation +- Automatic scaling based on load thresholds +- Load balancing across server instances +- Resource monitoring and metrics +- Visual feedback on scaling decisions +""" + +from .pipeline import run_auto_scaling_demo + +__all__ = ["run_auto_scaling_demo"] diff --git a/sage-apps/src/sage/apps/auto_scaling_chat/operators.py b/sage-apps/src/sage/apps/auto_scaling_chat/operators.py new file mode 100644 index 0000000..14f0c4f --- /dev/null +++ b/sage-apps/src/sage/apps/auto_scaling_chat/operators.py @@ -0,0 +1,318 @@ +""" +SAGE Operators for Auto-Scaling Chat System + +Implements operators for simulated chat system with auto-scaling capabilities. +""" + +from __future__ import annotations + +import random +import time +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +from sage.common.core import BatchFunction, MapFunction, SinkFunction + + +@dataclass +class UserRequest: + """Represents a user chat request.""" + + request_id: int + user_id: int + timestamp: str + message: str + processing_time: float = 0.0 + server_id: int | None = None + + +class UserTrafficSource(BatchFunction): + """ + BatchFunction source that generates simulated user traffic. + + Simulates variable user load with peaks and valleys. + """ + + # Probability scaling constant for request generation + PROBABILITY_SCALE = 100.0 + + def __init__( + self, + duration: int = 30, + base_rate: int = 5, + peak_rate: int = 50, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.duration = duration + self.base_rate = base_rate + self.peak_rate = peak_rate + self.start_time = time.time() + self.request_count = 0 + + def execute(self) -> dict[str, Any] | None: + """Generate user requests based on current load pattern.""" + elapsed = time.time() - self.start_time + + if elapsed >= self.duration: + return None + + # Simulate traffic pattern: gradual increase, peak, then decrease + progress = elapsed / self.duration + if progress < 0.3: + # Ramp up phase + rate = self.base_rate + (self.peak_rate - self.base_rate) * (progress / 0.3) + elif progress < 0.7: + # Peak phase + rate = self.peak_rate + else: + # Ramp down phase + rate = self.peak_rate - (self.peak_rate - self.base_rate) * ((progress - 0.7) / 0.3) + + # Determine if we should emit a request based on rate + # Higher rate = more requests per second + probability = rate / self.PROBABILITY_SCALE + if random.random() < probability: + self.request_count += 1 + return { + "request_id": self.request_count, + "user_id": random.randint(1, 1000), + "timestamp": datetime.now().isoformat(), + "message": f"User message #{self.request_count}", + "current_load": int(rate), + } + + # Small delay to control emission rate + time.sleep(0.05) + return None + + +class LoadBalancer(MapFunction): + """ + MapFunction that distributes requests across available servers. + + Implements simple round-robin load balancing. + """ + + def __init__(self, initial_servers: int = 2, **kwargs) -> None: + super().__init__(**kwargs) + self.num_servers = initial_servers + self.current_server = 0 + self.server_loads: dict[int, int] = dict.fromkeys(range(initial_servers), 0) + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Assign request to a server.""" + # Round-robin assignment + server_id = self.current_server + self.current_server = (self.current_server + 1) % self.num_servers + + # Update server load + self.server_loads[server_id] = self.server_loads.get(server_id, 0) + 1 + + # Assign to request + data["server_id"] = server_id + data["num_servers"] = self.num_servers + + return data + + def scale_servers(self, new_count: int) -> None: + """Scale number of servers.""" + old_count = self.num_servers + self.num_servers = new_count + + # Initialize new servers + for i in range(old_count, new_count): + self.server_loads[i] = 0 + + self.logger.info(f"Scaled servers: {old_count} → {new_count}") + + +class AutoScaler(MapFunction): + """ + MapFunction that makes auto-scaling decisions based on load. + """ + + def __init__( + self, + scale_up_threshold: int = 30, + scale_down_threshold: int = 10, + min_servers: int = 2, + max_servers: int = 10, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.scale_up_threshold = scale_up_threshold + self.scale_down_threshold = scale_down_threshold + self.min_servers = min_servers + self.max_servers = max_servers + self.current_servers = min_servers + self.last_scale_time = time.time() + self.cooldown = 5.0 # seconds + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Make scaling decision based on current load.""" + current_load = data.get("current_load", 0) + now = time.time() + + # Check if we're in cooldown period + if now - self.last_scale_time < self.cooldown: + data["scaling_action"] = "cooldown" + return data + + # Calculate average load per server + avg_load_per_server = current_load / self.current_servers if self.current_servers > 0 else 0 + + scaling_action = "none" + + # Scale up if load is high + if ( + avg_load_per_server > self.scale_up_threshold + and self.current_servers < self.max_servers + ): + new_servers = min(self.current_servers + 1, self.max_servers) + scaling_action = "scale_up" + self.current_servers = new_servers + self.last_scale_time = now + self.logger.warning( + f"⚠️ Scaling Threshold Reached! " + f"Load: {current_load}, Scaling up to {new_servers} servers" + ) + + # Scale down if load is low + elif ( + avg_load_per_server < self.scale_down_threshold + and self.current_servers > self.min_servers + ): + new_servers = max(self.current_servers - 1, self.min_servers) + scaling_action = "scale_down" + self.current_servers = new_servers + self.last_scale_time = now + self.logger.info(f"Scaling down to {new_servers} servers (low load)") + + data["scaling_action"] = scaling_action + data["total_servers"] = self.current_servers + data["avg_load_per_server"] = avg_load_per_server + + return data + + +class RequestProcessor(MapFunction): + """ + MapFunction that simulates request processing on servers. + """ + + def __init__(self, processing_time: float = 0.1, **kwargs) -> None: + super().__init__(**kwargs) + self.processing_time = processing_time + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Process request.""" + # Simulate processing time + time.sleep(random.uniform(0, self.processing_time)) + + data["processing_time"] = self.processing_time + data["processed_at"] = datetime.now().isoformat() + + return data + + +class MetricsCollector(SinkFunction): + """ + SinkFunction that collects and displays system metrics. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.total_requests = 0 + self.scaling_events = 0 + self.peak_servers = 0 + self.peak_load = 0 + self.start_time = time.time() + self.load_history: list[int] = [] + self.server_history: list[int] = [] + + def execute(self, data: dict[str, Any]) -> None: + """Collect metrics from requests.""" + self.total_requests += 1 + + current_load = data.get("current_load", 0) + num_servers = data.get("total_servers", 0) + scaling_action = data.get("scaling_action", "none") + + # Update peak metrics + self.peak_load = max(self.peak_load, current_load) + self.peak_servers = max(self.peak_servers, num_servers) + + # Track scaling events + if scaling_action in ["scale_up", "scale_down"]: + self.scaling_events += 1 + + # Record history + self.load_history.append(current_load) + self.server_history.append(num_servers) + + # Log every 10 requests + if self.total_requests % 10 == 0: + self.logger.info( + f"📊 Load: {current_load:3d} users | " + f"Servers: {num_servers:2d} | " + f"Requests: {self.total_requests:4d}" + ) + + def close(self) -> None: + """Display final metrics summary.""" + elapsed = time.time() - self.start_time + + print("\n" + "=" * 70) + print("📊 Auto-Scaling System Metrics") + print("=" * 70) + print(f" Total requests processed: {self.total_requests}") + print(f" Total duration: {elapsed:.2f}s") + print(f" Average throughput: {self.total_requests / elapsed:.2f} req/s") + print(f" Peak load: {self.peak_load} concurrent users") + print(f" Peak servers: {self.peak_servers}") + print(f" Scaling events: {self.scaling_events}") + + if self.load_history: + avg_load = sum(self.load_history) / len(self.load_history) + print(f" Average load: {avg_load:.1f} users") + + if self.server_history: + avg_servers = sum(self.server_history) / len(self.server_history) + print(f" Average servers: {avg_servers:.1f}") + + print("=" * 70 + "\n") + + +class ScalingEventsSink(SinkFunction): + """ + SinkFunction that logs all scaling events. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.events: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> None: + """Log scaling events.""" + scaling_action = data.get("scaling_action", "none") + if scaling_action in ["scale_up", "scale_down"]: + event = { + "action": scaling_action, + "servers": data.get("total_servers", 0), + "load": data.get("current_load", 0), + "timestamp": data.get("timestamp", ""), + } + self.events.append(event) + + def close(self) -> None: + """Display scaling events log.""" + if self.events: + print("\n🔄 Scaling Events:") + for i, event in enumerate(self.events, 1): + action_icon = "📈" if event["action"] == "scale_up" else "📉" + print( + f" {i}. {action_icon} {event['action'].upper()}: " + f"{event['servers']} servers (load: {event['load']})" + ) diff --git a/sage-apps/src/sage/apps/auto_scaling_chat/pipeline.py b/sage-apps/src/sage/apps/auto_scaling_chat/pipeline.py new file mode 100644 index 0000000..a518439 --- /dev/null +++ b/sage-apps/src/sage/apps/auto_scaling_chat/pipeline.py @@ -0,0 +1,170 @@ +""" +Auto-Scaling Chat System Pipeline + +Main pipeline implementation using SAGE operators for auto-scaling demonstration. +""" + +from __future__ import annotations + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + +from .operators import ( + AutoScaler, + LoadBalancer, + MetricsCollector, + RequestProcessor, + ScalingEventsSink, + UserTrafficSource, +) + + +def run_auto_scaling_demo( + duration: int = 30, + base_rate: int = 5, + peak_rate: int = 50, + verbose: bool = False, +) -> None: + """ + Run the auto-scaling chat system demonstration. + + Args: + duration: Simulation duration in seconds + base_rate: Base user load (concurrent users) + peak_rate: Peak user load (concurrent users) + verbose: Enable verbose logging + + Example: + >>> from sage.apps.auto_scaling_chat import run_auto_scaling_demo + >>> run_auto_scaling_demo(duration=30, peak_rate=50) + """ + print("\n" + "⚡" * 35) + print(" " * 10 + "SAGE Auto-Scaling Chat System Demo") + print("⚡" * 35) + print() + print("=" * 70) + print("📋 Configuration") + print("=" * 70) + print(f" Simulation duration: {duration}s") + print(f" Base load: {base_rate} concurrent users") + print(f" Peak load: {peak_rate} concurrent users") + print(f" Verbose logging: {verbose}") + print("=" * 70) + print() + + print("=" * 70) + print("🎯 System Behavior") + print("=" * 70) + print(" Phase 1 (0-30%): Gradual load increase") + print(" Phase 2 (30-70%): Peak load period") + print(" Phase 3 (70-100%): Load decrease") + print() + print(" Scaling Policy:") + print(" - Scale UP when avg load/server > 30 users") + print(" - Scale DOWN when avg load/server < 10 users") + print(" - Min servers: 2, Max servers: 10") + print("=" * 70) + print() + + # Create SAGE environment + env = LocalEnvironment("auto_scaling_chat") + + # Build pipeline: + # 1. UserTrafficSource: Generate simulated user requests + # 2. AutoScaler: Make scaling decisions based on load + # 3. LoadBalancer: Distribute requests across servers + # 4. RequestProcessor: Process requests on servers + # 5. MetricsCollector: Collect and display metrics + # 6. ScalingEventsSink: Log scaling events + pipeline = ( + env.from_batch( + UserTrafficSource, + duration=duration, + base_rate=base_rate, + peak_rate=peak_rate, + ) + .map(AutoScaler, scale_up_threshold=30, scale_down_threshold=10) + .map(LoadBalancer, initial_servers=2) + .map(RequestProcessor, processing_time=0.05) + .sink(MetricsCollector) + ) + + if verbose: + pipeline.sink(ScalingEventsSink) + + # Execute pipeline + print("🚀 Starting auto-scaling simulation...") + print() + env.submit(autostop=True) + + print("\n⚡" * 35) + print(" " * 20 + "Demo completed!") + print("⚡" * 35 + "\n") + + +def main(): + """Main entry point for command-line usage.""" + import argparse + + parser = argparse.ArgumentParser( + description="SAGE Auto-Scaling Chat System - Elastic resource management demo", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run default simulation (30s) + python -m sage.apps.auto_scaling_chat.pipeline + + # Short simulation with high peak + python -m sage.apps.auto_scaling_chat.pipeline --duration 20 --peak-rate 80 + + # Long simulation with moderate load + python -m sage.apps.auto_scaling_chat.pipeline --duration 60 --peak-rate 40 + + # Verbose mode with scaling events + python -m sage.apps.auto_scaling_chat.pipeline --verbose + """, + ) + + parser.add_argument( + "--duration", + "-d", + type=int, + default=30, + help="Simulation duration in seconds (default: 30)", + ) + + parser.add_argument( + "--base-rate", + "-b", + type=int, + default=5, + help="Base user load (default: 5)", + ) + + parser.add_argument( + "--peak-rate", + "-p", + type=int, + default=50, + help="Peak user load (default: 50)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + # Run demo + run_auto_scaling_demo( + duration=args.duration, + base_rate=args.base_rate, + peak_rate=args.peak_rate, + verbose=args.verbose, + ) + + +if __name__ == "__main__": + main() diff --git a/sage-apps/src/sage/apps/medical_diagnosis/README.md b/sage-apps/src/sage/apps/medical_diagnosis/README.md new file mode 100644 index 0000000..56598ee --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/README.md @@ -0,0 +1,220 @@ +# Medical Diagnosis Application + +SAGE 医疗诊断应用示例 - 展示如何使用 SAGE 构建多智能体医疗诊断系统。 + +## 📋 概述 + +这个应用展示了如何使用 SAGE 框架构建一个协作式医疗诊断系统，其中多个专业医生智能体协同工作，分析患者症状并提供诊断建议。 + +## 🏗️ 架构 + +``` +medical_diagnosis/ +├── agents/ # 智能体定义 +│ ├── doctor.py # 主治医生智能体 +│ └── specialist.py # 专科医生智能体 +├── config/ # 配置文件 +│ └── agents.yaml # 智能体配置 +├── tools/ # 工具函数 +│ ├── medical_knowledge.py # 医学知识库 +│ └── symptom_analyzer.py # 症状分析工具 +├── data/ # 数据文件 +│ └── patient_cases/ # 患者案例 +├── scripts/ # 脚本文件 +└── run_diagnosis.py # 主运行脚本 +``` + +## 🚀 快速开始 + +### 安装依赖 + +```bash +# 安装 SAGE +pip install isage-apps + +# 或从源码安装 +cd packages/sage-apps +pip install -e . +``` + +### 准备数据 + +```bash +# 运行数据准备脚本 +cd src/sage/apps/medical_diagnosis +./setup_data.sh +``` + +### 运行诊断 + +```bash +# 单个案例诊断 +python run_diagnosis.py --case patient_001 + +# 批量诊断 +python run_diagnosis.py --batch --input-dir data/patient_cases/ + +# 交互式诊断 +python run_diagnosis.py --interactive +``` + +## 📊 示例 + +### 单个患者诊断 + +```python +from sage.apps.medical_diagnosis import MedicalDiagnosisSystem + +# 创建诊断系统 +system = MedicalDiagnosisSystem() + +# 患者症状 +patient_data = { + "age": 45, + "gender": "male", + "symptoms": ["头痛", "发烧", "咳嗽"], + "duration": "3天", + "medical_history": ["高血压"] +} + +# 执行诊断 +result = system.diagnose(patient_data) + +# 查看结果 +print(f"初步诊断: {result['diagnosis']}") +print(f"建议检查: {result['recommended_tests']}") +print(f"治疗方案: {result['treatment_plan']}") +``` + +### 批量处理 + +```python +from sage.apps.medical_diagnosis import run_batch_diagnosis + +# 批量诊断 +results = run_batch_diagnosis( + input_dir="data/patient_cases/", + output_dir="results/" +) + +# 生成报告 +for patient_id, diagnosis in results.items(): + print(f"患者 {patient_id}: {diagnosis['summary']}") +``` + +## 🎯 功能特性 + +- **多智能体协作**: 主治医生和专科医生协同诊断 +- **知识库集成**: 整合医学知识库和诊断指南 +- **症状分析**: 自动分析患者症状并生成诊断假设 +- **治疗建议**: 基于诊断结果提供治疗方案 +- **批量处理**: 支持批量处理多个患者案例 +- **可视化结果**: 生成诊断报告和可视化图表 + +## 🔧 配置 + +编辑 `config/agents.yaml` 来配置智能体行为： + +```yaml +agents: + - name: primary_doctor + role: 主治医生 + llm_config: + model: gpt-4 + temperature: 0.7 + + - name: cardiologist + role: 心脏科专家 + llm_config: + model: gpt-4 + temperature: 0.5 +``` + +## 📝 数据格式 + +### 输入格式 + +患者案例应使用 JSON 格式： + +```json +{ + "patient_id": "P001", + "age": 45, + "gender": "male", + "chief_complaint": "胸痛", + "symptoms": [ + {"name": "胸痛", "severity": "severe", "duration": "2小时"}, + {"name": "呼吸困难", "severity": "moderate", "duration": "30分钟"} + ], + "vital_signs": { + "blood_pressure": "140/90", + "heart_rate": 95, + "temperature": 37.2 + }, + "medical_history": ["高血压", "糖尿病"] +} +``` + +### 输出格式 + +诊断结果： + +```json +{ + "diagnosis": { + "primary": "急性心肌梗死可能性高", + "differential": ["不稳定型心绞痛", "主动脉夹层"], + "confidence": 0.85 + }, + "recommended_tests": [ + "心电图", + "心肌酶谱", + "冠状动脉造影" + ], + "treatment_plan": { + "immediate": ["氧疗", "硝酸甘油", "阿司匹林"], + "follow_up": ["心脏超声", "冠脉介入"] + }, + "reasoning": "基于患者症状、体征和病史，考虑急性冠脉综合征..." +} +``` + +## 🧪 测试 + +```bash +# 运行所有测试 +pytest tests/medical_diagnosis/ + +# 运行单个测试 +pytest tests/medical_diagnosis/test_diagnosis.py -v + +# 运行带覆盖率的测试 +pytest tests/medical_diagnosis/ --cov=sage.apps.medical_diagnosis +``` + +## 📚 相关文档 + +- [SAGE 框架文档](../../docs/) +- [多智能体系统指南](../../docs/guides/multi-agent.md) +- [医疗 AI 应用最佳实践](../../docs/tutorials/medical-ai.md) + +## ⚠️ 免责声明 + +**重要**: 这是一个演示应用，仅用于教学和研究目的。**不应**用于实际医疗诊断。任何医疗决策都应由专业医疗人员做出。 + +## 📄 许可证 + +本应用遵循 SAGE 项目的 MIT 许可证。 + +## 🤝 贡献 + +欢迎贡献！请参阅 [CONTRIBUTING.md](../../../../CONTRIBUTING.md) 了解如何参与项目。 + +## 📧 联系方式 + +- 问题反馈: [GitHub Issues](https://github.com/intellistream/SAGE/issues) +- 讨论: [GitHub Discussions](https://github.com/intellistream/SAGE/discussions) + +______________________________________________________________________ + +**构建于** [SAGE](https://github.com/intellistream/SAGE) - 下一代智能体框架 diff --git a/sage-apps/src/sage/apps/medical_diagnosis/__init__.py b/sage-apps/src/sage/apps/medical_diagnosis/__init__.py new file mode 100644 index 0000000..5289d58 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/__init__.py @@ -0,0 +1,44 @@ +""" +Medical Diagnosis Application - Lumbar Spine MRI Analysis + +A complete medical diagnosis system for analyzing lumbar spine MRI images +and generating diagnostic reports using multi-agent collaboration. + +Quick Start: + >>> from sage.apps.medical_diagnosis import DiagnosticAgent + >>> + >>> agent = DiagnosticAgent() + >>> result = agent.diagnose( + ... image_path="path/to/mri.jpg", + ... patient_info={"age": 45, "gender": "male", "symptoms": "back pain"} + ... ) + >>> print(result.report) + +Components: + - DiagnosticAgent: Main coordinator agent + - ImageAnalyzer: MRI image analysis agent + - ReportGenerator: Diagnostic report generation agent + - MedicalKnowledgeBase: Medical knowledge retrieval + +For full documentation, see README.md in this directory. +""" + +from .agents.diagnostic_agent import DiagnosisResult, DiagnosticAgent +from .agents.image_analyzer import ImageAnalyzer, ImageFeatures +from .agents.report_generator import DiagnosisReport, ReportGenerator +from .tools.knowledge_base import MedicalKnowledgeBase + +__all__ = [ + # Main Agent + "DiagnosticAgent", + "DiagnosisResult", + # Sub Agents + "ImageAnalyzer", + "ImageFeatures", + "ReportGenerator", + "DiagnosisReport", + # Tools + "MedicalKnowledgeBase", +] + +__version__ = "1.0.0" diff --git a/sage-apps/src/sage/apps/medical_diagnosis/agents/__init__.py b/sage-apps/src/sage/apps/medical_diagnosis/agents/__init__.py new file mode 100644 index 0000000..137f735 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/agents/__init__.py @@ -0,0 +1,12 @@ +"""医疗诊断Agent包""" + +from .diagnostic_agent import DiagnosisResult, DiagnosticAgent +from .image_analyzer import ImageAnalyzer +from .report_generator import ReportGenerator + +__all__ = [ + "DiagnosticAgent", + "DiagnosisResult", + "ImageAnalyzer", + "ReportGenerator", +] diff --git a/sage-apps/src/sage/apps/medical_diagnosis/agents/diagnostic_agent.py b/sage-apps/src/sage/apps/medical_diagnosis/agents/diagnostic_agent.py new file mode 100644 index 0000000..963ec10 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/agents/diagnostic_agent.py @@ -0,0 +1,235 @@ +""" +医疗诊断Agent - 主Agent +负责协调影像分析、知识检索和报告生成 +""" + +from pathlib import Path +from typing import Any + +import yaml + +# 导入 DiagnosisReport，DiagnosisResult 作为别名以保持向后兼容 +from .report_generator import DiagnosisReport + +# 向后兼容别名 +DiagnosisResult = DiagnosisReport + + +class DiagnosticAgent: + """ + 腰椎MRI诊断Agent + + 功能: + 1. 接收MRI影像和患者信息 + 2. 调用影像分析Agent提取特征 + 3. 检索相关医学知识和相似病例 + 4. 生成诊断报告 + """ + + def __init__(self, config_path: str | None = None): + """ + 初始化诊断Agent + + Args: + config_path: 配置文件路径 + """ + self.config = self._load_config(config_path) + self.image_analyzer = None + self.knowledge_base = None + self.report_generator = None + self._setup_components() + + def _load_config(self, config_path: str | None) -> dict: + """加载配置""" + if config_path and Path(config_path).exists(): + with open(config_path, encoding="utf-8") as f: + return yaml.safe_load(f) + + # 默认配置 + return { + "agent": { + "name": "LumbarSpineDiagnosticAgent", + "version": "1.0.0", + "max_iterations": 5, + }, + "models": { + "vision_model": "Qwen/Qwen2-VL-7B-Instruct", + "llm_model": "Qwen/Qwen2.5-7B-Instruct", + "embedding_model": "BAAI/bge-large-zh-v1.5", + }, + "services": { + "vllm": {"enabled": True, "gpu_memory_utilization": 0.9}, + "embedding": {"method": "hf", "cache_enabled": True}, + "vector_db": {"collection_name": "lumbar_spine_cases", "top_k": 5}, + }, + } + + def _setup_components(self): + """设置组件""" + from sage.apps.medical_diagnosis.agents.image_analyzer import ImageAnalyzer + from sage.apps.medical_diagnosis.agents.report_generator import ReportGenerator + from sage.apps.medical_diagnosis.tools.knowledge_base import MedicalKnowledgeBase + + # 初始化各个组件 + self.image_analyzer = ImageAnalyzer(self.config) + self.knowledge_base = MedicalKnowledgeBase(self.config) + self.report_generator = ReportGenerator(self.config) + + print("✅ DiagnosticAgent 初始化完成") + print(f" Vision Model: {self.config['models']['vision_model']}") + print(f" LLM Model: {self.config['models']['llm_model']}") + + def diagnose( + self, + image_path: str, + patient_info: dict[str, Any] | None = None, + verbose: bool = True, + ) -> DiagnosisResult: + """ + 执行诊断 + + Args: + image_path: MRI影像路径 + patient_info: 患者信息（年龄、性别、症状等） + verbose: 是否打印详细信息 + + Returns: + DiagnosisResult: 诊断结果 + """ + if verbose: + print(f"\n{'=' * 60}") + print("🏥 开始诊断分析") + print(f"{'=' * 60}") + print(f"📄 影像路径: {image_path}") + if patient_info: + print(f"👤 患者信息: {patient_info}") + + # Step 1: 影像分析 + if verbose: + print("\n📊 Step 1: 影像特征提取...") + + if not self.image_analyzer: + raise RuntimeError("Image analyzer not initialized") + image_features = self.image_analyzer.analyze(image_path) + + if verbose: + print(f" ✓ 检测到 {len(image_features.get('vertebrae', []))} 个椎体") + print(f" ✓ 检测到 {len(image_features.get('discs', []))} 个椎间盘") + if image_features.get("abnormalities"): + print(f" ⚠ 发现 {len(image_features['abnormalities'])} 处异常") + + # Step 2: 知识库检索 + if verbose: + print("\n🔍 Step 2: 检索相关知识和病例...") + + # 构建查询 + query = self._build_query(image_features, patient_info) + + # 检索相似病例 + if not self.knowledge_base: + raise RuntimeError("Knowledge base not initialized") + similar_cases = self.knowledge_base.retrieve_similar_cases( + query=query, + image_features=image_features, + top_k=self.config["services"]["vector_db"]["top_k"], + ) + + if verbose: + print(f" ✓ 检索到 {len(similar_cases)} 个相似病例") + + # 检索医学知识 + medical_knowledge = self.knowledge_base.retrieve_knowledge(query=query, top_k=3) + + # Step 3: 生成诊断报告 + if verbose: + print("\n📝 Step 3: 生成诊断报告...") + + if not self.report_generator: + raise RuntimeError("Report generator not initialized") + diagnosis_result = self.report_generator.generate( + image_features=image_features, + patient_info=patient_info, + similar_cases=similar_cases, + medical_knowledge=medical_knowledge, + ) + + if verbose: + print(" ✓ 报告生成完成") + print(f"\n{'=' * 60}") + print("📋 诊断结果") + print(f"{'=' * 60}") + print(f"\n{diagnosis_result.report}") + + return diagnosis_result + + def _build_query(self, image_features: dict, patient_info: dict | None) -> str: + """构建检索查询""" + query_parts = [] + + # 添加影像发现 + if image_features.get("abnormalities"): + findings = [a["description"] for a in image_features["abnormalities"]] + query_parts.append(f"影像发现: {', '.join(findings)}") + + # 添加患者症状 + if patient_info and "symptoms" in patient_info: + query_parts.append(f"症状: {patient_info['symptoms']}") + + # 添加年龄信息 + if patient_info and "age" in patient_info: + age = patient_info["age"] + if age > 60: + query_parts.append("老年患者退行性变化") + elif age > 40: + query_parts.append("中年腰椎病变") + + return " ".join(query_parts) + + def batch_diagnose( + self, cases: list[dict[str, Any]], output_dir: str | None = None + ) -> list[DiagnosisResult]: + """ + 批量诊断 + + Args: + cases: 病例列表，每个包含 image_path 和 patient_info + output_dir: 输出目录 + + Returns: + 诊断结果列表 + """ + results = [] + + print(f"\n🏥 批量诊断开始 - 共 {len(cases)} 个病例") + + for i, case in enumerate(cases, 1): + print(f"\n处理病例 {i}/{len(cases)}") + + result = self.diagnose( + image_path=case["image_path"], + patient_info=case.get("patient_info"), + verbose=False, + ) + + results.append(result) + + # 保存结果 + if output_dir: + output_path = Path(output_dir) / f"case_{i:03d}_report.txt" + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + f.write(result.report) + + print("\n✅ 批量诊断完成！") + return results + + +if __name__ == "__main__": + # 测试代码 + agent = DiagnosticAgent() + + # 示例诊断 + result = agent.diagnose( + image_path="data/medical/test/sample_mri.jpg", + patient_info={"age": 45, "gender": "male", "symptoms": "下背部疼痛，左腿麻木"}, + ) diff --git a/sage-apps/src/sage/apps/medical_diagnosis/agents/image_analyzer.py b/sage-apps/src/sage/apps/medical_diagnosis/agents/image_analyzer.py new file mode 100644 index 0000000..830e21c --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/agents/image_analyzer.py @@ -0,0 +1,1121 @@ +""" +影像分析Agent +负责MRI影像的特征提取和初步分析 +""" + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + +try: + import torch + from PIL import Image as PILImage + from transformers import ( + AutoImageProcessor, + AutoModel, + CLIPModel, + CLIPProcessor, + ) + + TORCH_AVAILABLE = True +except ImportError: + TORCH_AVAILABLE = False + torch = None + PILImage = None + AutoImageProcessor = None + AutoModel = None + CLIPModel = None + CLIPProcessor = None + +try: + from numpy.lib.stride_tricks import sliding_window_view +except ImportError: # pragma: no cover - older NumPy fallback + sliding_window_view = None # type: ignore[assignment] + + +DEFAULT_DISC_LEVELS = ["L1/L2", "L2/L3", "L3/L4", "L4/L5", "L5/S1"] + + +@dataclass +class ImageFeatures: + """影像特征""" + + vertebrae: list[dict[str, Any]] # 椎体信息 + discs: list[dict[str, Any]] # 椎间盘信息 + abnormalities: list[dict[str, Any]] # 异常发现 + image_quality: float # 影像质量评分 + image_embedding: np.ndarray | None = None # 影像嵌入向量 + + +class ImageAnalyzer: + """ + MRI影像分析器 + + 功能: + 1. 图像预处理和质量评估 + 2. 椎体和椎间盘分割定位 + 3. 病变区域检测 + 4. 影像特征提取和向量化 + """ + + _LAPLACIAN_KERNEL = np.array([[0, 1, 0], [1, -4, 1], [0, 1, 0]], dtype=np.float32) + _SOBEL_X = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], dtype=np.float32) + _SOBEL_Y = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]], dtype=np.float32) + _BLUR_KERNEL = np.ones((3, 3), dtype=np.float32) / 9.0 + _DEFAULT_QUALITY_WEIGHTS = { + "contrast": 0.25, + "sharpness": 0.35, + "noise": 0.25, + "brightness": 0.15, + } + + def __init__(self, config: dict): + """ + 初始化影像分析器 + + Args: + config: 配置字典 + """ + self.config = config + self.vision_model = None + self.segmentation_model = None + self.segmenter_type = None + self.segmenter_device = "cpu" + self.segmentation_config: dict[str, Any] = {} + self.segmentation_enabled = False + self.disc_levels = DEFAULT_DISC_LEVELS.copy() + self.segmentation_thresholds: dict[str, float] = {} + self.pixel_spacing_mm = 180.0 + self.baseline_disc_height_mm = 9.0 + self._setup_models() + + def _setup_models(self): + """设置视觉模型""" + # TODO: 集成 SAGE VLLMService 或其他视觉模型 + # Issue URL: https://github.com/intellistream/SAGE/issues/899 + print(f" Loading vision model: {self.config['models']['vision_model']}") + + # 这里可以集成: + # 1. Qwen2-VL for general vision understanding + # 2. SAM (Segment Anything Model) for segmentation + # 3. Medical imaging specific models + + self.vision_model = "placeholder" # 实际应该加载模型 + + image_processing_cfg = self.config.get("image_processing", {}) + + # Avoid heavy downloads in tests unless explicitly enabled + test_mode = os.getenv("SAGE_TEST_MODE", "").lower() in {"1", "true", "yes", "on"} + skip_heavy_models = bool(self.config.get("skip_model_download", test_mode)) + + # 设置特征提取模型 + self.feature_model = None + self.feature_processor = None + self.feature_method = image_processing_cfg.get("feature_extraction", {}).get( + "method", "clip" + ) + + if skip_heavy_models: + print(" Skipping feature extractor setup (test/skip_model_download)") + elif TORCH_AVAILABLE: + try: + if self.feature_method == "clip": + self._setup_clip_model() + elif self.feature_method == "dinov2": + self._setup_dinov2_model() + else: + print( + f" Warning: Unknown feature method '{self.feature_method}', using mock features" + ) + except Exception as e: + print(f" Warning: Failed to load feature extraction model: {e}") + print(" Falling back to mock features") + else: + print(" Warning: PyTorch not available, feature extraction will use mock features") + + # 设置分割参数 + self.segmentation_config = image_processing_cfg.get("segmentation", {}) or {} + self.segmentation_enabled = ( + bool(self.segmentation_config.get("enabled", False)) and not skip_heavy_models + ) + disc_levels_cfg = self.segmentation_config.get("disc_levels") + self.disc_levels = list(disc_levels_cfg) if disc_levels_cfg else DEFAULT_DISC_LEVELS.copy() + self.pixel_spacing_mm = float( + self.segmentation_config.get("pixel_spacing_mm", self.pixel_spacing_mm) + ) + self.baseline_disc_height_mm = float( + self.segmentation_config.get( + "baseline_disc_height_mm", + self.segmentation_config.get("expected_height_mm", self.baseline_disc_height_mm), + ) + ) + self.segmentation_thresholds = { + "min_height_ratio": 0.02, + "max_height_ratio": 0.18, + "min_aspect_ratio": 1.5, + "min_width_ratio": 0.12, + "min_vertical_pos": 0.08, + "max_vertical_pos": 0.95, + "min_confidence": 0.35, + "min_separation_ratio": 0.05, + } + thresholds_cfg = self.segmentation_config.get("thresholds", {}) or {} + for key, value in thresholds_cfg.items(): + try: + self.segmentation_thresholds[key] = float(value) + except (TypeError, ValueError): + continue + + if self.segmentation_enabled: + self._setup_segmentation_model() + + def _setup_clip_model(self): + """设置 CLIP 模型用于特征提取""" + try: + model_name = "openai/clip-vit-base-patch32" + print(f" Loading CLIP model: {model_name}") + self.feature_processor = CLIPProcessor.from_pretrained(model_name) + self.feature_model = CLIPModel.from_pretrained(model_name) + + # 移动到 GPU (如果可用) + if torch.cuda.is_available(): + self.feature_model = self.feature_model.cuda() + print(" CLIP model loaded on GPU") + else: + print(" CLIP model loaded on CPU") + + self.feature_model.eval() # 设置为评估模式 + except Exception as e: + print(f" Failed to load CLIP model: {e}") + raise + + def _setup_dinov2_model(self): + """设置 DINOv2 模型用于特征提取""" + try: + model_name = "facebook/dinov2-base" + print(f" Loading DINOv2 model: {model_name}") + self.feature_processor = AutoImageProcessor.from_pretrained(model_name) + self.feature_model = AutoModel.from_pretrained(model_name) + + # 移动到 GPU (如果可用) + if torch.cuda.is_available(): + self.feature_model = self.feature_model.cuda() + print(" DINOv2 model loaded on GPU") + else: + print(" DINOv2 model loaded on CPU") + + self.feature_model.eval() # 设置为评估模式 + except Exception as e: + print(f" Failed to load DINOv2 model: {e}") + raise + + def _setup_segmentation_model(self): + """设置椎间盘分割模型""" + if not TORCH_AVAILABLE: + print(" Warning: Segmentation requires PyTorch, falling back to heuristics") + return + + model_name = (self.segmentation_config.get("model") or "sam").lower() + try: + if model_name.startswith("sam") or model_name.startswith("medsam"): + self.segmentation_model = self._setup_sam_segmenter(model_name) + self.segmenter_type = "sam" + else: + print( + f" Warning: Unsupported segmentation model '{model_name}', using heuristics" + ) + self.segmentation_model = None + except Exception as exc: + print(f" Warning: Failed to initialize segmentation model '{model_name}': {exc}") + self.segmentation_model = None + self.segmenter_type = None + + def _setup_sam_segmenter(self, model_name: str): + """初始化 Segment Anything 模型""" + try: + from segment_anything import SamAutomaticMaskGenerator, sam_model_registry + except ImportError as exc: # pragma: no cover - 可选依赖 + raise RuntimeError( + "segment_anything is not installed. Please install it to use SAM segmentation" + ) from exc + + checkpoint_path = self.segmentation_config.get("checkpoint_path") + if not checkpoint_path: + raise ValueError("Segmentation config missing 'checkpoint_path'") + + checkpoint = Path(checkpoint_path).expanduser() + if not checkpoint.exists(): + raise FileNotFoundError(f"Segmentation checkpoint not found: {checkpoint}") + + variant = self.segmentation_config.get("sam_variant") + variant = variant or { + "sam": "vit_h", + "sam-vit-h": "vit_h", + "sam-vit-b": "vit_b", + "sam-vit-l": "vit_l", + "medsam": "vit_b", + }.get(model_name, model_name.replace("sam-", "")) + + if variant not in sam_model_registry: + raise ValueError(f"Unsupported SAM variant '{variant}'") + + print(f" Loading SAM segmentation model ({variant}) from {checkpoint}") + sam = sam_model_registry[variant](checkpoint=str(checkpoint)) + self.segmenter_device = "cuda" if torch.cuda.is_available() else "cpu" + sam.to(device=self.segmenter_device) + + generator_kwargs = { + "points_per_side": self.segmentation_config.get("points_per_side", 16), + "pred_iou_thresh": self.segmentation_config.get("pred_iou_thresh", 0.86), + "stability_score_thresh": self.segmentation_config.get("stability_score_thresh", 0.92), + "crop_n_layers": self.segmentation_config.get("crop_n_layers", 1), + "crop_n_points_downscale_factor": self.segmentation_config.get( + "crop_n_points_downscale_factor", 2 + ), + "min_mask_region_area": self.segmentation_config.get("min_mask_region_area", 200), + } + + print( + " SAM segmentation ready " + f"(points_per_side={generator_kwargs['points_per_side']}, device={self.segmenter_device})" + ) + return SamAutomaticMaskGenerator(sam, **generator_kwargs) + + def analyze(self, image_path: str) -> dict[str, Any]: + """ + 分析MRI影像 + + Args: + image_path: 影像文件路径 + + Returns: + 影像特征字典 + """ + image_path_obj = Path(image_path) + + if not image_path_obj.exists(): + # 创建模拟数据用于演示 + return self._create_mock_analysis() + + # Step 1: 加载和预处理图像 + image = self._load_image(image_path_obj) + + # Step 2: 质量评估 + quality_score = self._assess_quality(image) + + # Step 3: 解剖结构分割 + vertebrae = self._segment_vertebrae(image) + discs = self._segment_discs(image) + + # Step 4: 病变检测 + abnormalities = self._detect_abnormalities(image, vertebrae, discs) + + # Step 5: 特征提取 + image_embedding = self._extract_features(image) + + return { + "vertebrae": vertebrae, + "discs": discs, + "abnormalities": abnormalities, + "image_quality": quality_score, + "image_embedding": image_embedding, + "image_path": str(image_path), + } + + def _load_image(self, image_path: Path): + """ + 加载影像 + + 支持格式: + - DICOM格式 (.dcm, .dicom) + - 常规图像格式 (PNG, JPG, JPEG, BMP, TIFF等) + + Args: + image_path: 影像文件路径 + + Returns: + numpy数组格式的灰度图像，失败时返回None + """ + if not image_path.exists(): + print(f" Warning: 图像文件不存在 {image_path}") + return None + + # 获取文件扩展名 + file_extension = image_path.suffix.lower() + + try: + # 尝试加载DICOM格式 + if file_extension in [".dcm", ".dicom"]: + return self._load_dicom_image(image_path) + + # 尝试加载常规图像格式 + return self._load_regular_image(image_path) + + except Exception as e: + print(f" Warning: 无法加载图像 {image_path}: {e}") + return None + + def _load_dicom_image(self, image_path: Path): + """ + 加载DICOM格式影像 + + Args: + image_path: DICOM文件路径 + + Returns: + numpy数组格式的灰度图像 + """ + try: + import pydicom + + # 读取DICOM文件 + dicom_data = pydicom.dcmread(str(image_path)) + + # 获取像素数据 + image_array = dicom_data.pixel_array + + # 应用DICOM的窗宽窗位（如果存在） + if hasattr(dicom_data, "WindowCenter") and hasattr(dicom_data, "WindowWidth"): + window_center = ( + dicom_data.WindowCenter + if isinstance(dicom_data.WindowCenter, (int, float)) + else dicom_data.WindowCenter[0] + ) + window_width = ( + dicom_data.WindowWidth + if isinstance(dicom_data.WindowWidth, (int, float)) + else dicom_data.WindowWidth[0] + ) + + # 应用窗宽窗位 + img_min = window_center - window_width / 2 + img_max = window_center + window_width / 2 + image_array = np.clip(image_array, img_min, img_max) + + # 归一化到0-255范围 + if image_array.max() > image_array.min(): + image_array = ( + (image_array - image_array.min()) + / (image_array.max() - image_array.min()) + * 255 + ) + else: + image_array = np.zeros_like(image_array) + + # 转换为uint8类型 + image_array = image_array.astype(np.uint8) + + return image_array + + except ImportError: + print(" Warning: pydicom未安装，无法加载DICOM文件") + print(" 请运行: pip install pydicom") + return None + except Exception as e: + print(f" Warning: 加载DICOM文件失败 {image_path}: {e}") + return None + + def _load_regular_image(self, image_path: Path): + """ + 加载常规图像格式 + + Args: + image_path: 图像文件路径 + + Returns: + numpy数组格式的灰度图像 + """ + try: + from PIL import Image + + # 使用PIL加载图像并转换为灰度 + image = Image.open(image_path) + + # 转换为灰度图像 + if image.mode != "L": + image = image.convert("L") + + # 转换为numpy数组 + return np.array(image) + + except Exception as e: + print(f" Warning: 加载图像文件失败 {image_path}: {e}") + return None + + def _assess_quality(self, image) -> float: + """ + 评估影像质量 + + 使用多个指标综合评估: + 1. 对比度 (Contrast): 标准差、动态范围与分位区间 + 2. 清晰度 (Sharpness): 拉普拉斯方差 + 梯度能量 + 3. 噪声水平 (Noise): 高频残差能量 + 4. 亮度分布 (Brightness): 直方图占用与亮度平衡 + + Returns: + 0-1之间的质量分数，越高表示质量越好 + """ + if image is None: + return 0.0 + + try: + img = self._prepare_quality_image(image) + if img.size == 0: + return 0.0 + + metrics = self._compute_quality_metrics(img) + return self._combine_quality_scores(metrics) + + except Exception as e: # pragma: no cover - 防御性回退 + print(f" Warning: 质量评估失败: {e}") + return 0.5 + + def _prepare_quality_image(self, image: Any) -> np.ndarray: + """将输入图像转换为[0,1]范围的灰度图""" + arr = np.asarray(image) + if arr.ndim == 3: + arr = arr.mean(axis=-1) + arr = arr.astype(np.float32, copy=False) + if arr.size == 0: + return arr + + max_val = float(arr.max()) + min_val = float(arr.min()) + if max_val == min_val == 0.0: + return arr + + if max_val > 1.0 or min_val < 0.0: + # 依据位深度估算归一化系数 + if max_val <= 255.0: + scale = 255.0 + elif max_val <= 1023.0: + scale = 1023.0 + elif max_val <= 4095.0: + scale = 4095.0 + else: + scale = max_val + arr = arr / max(scale, 1e-6) + if arr.min() < 0: + arr = arr - arr.min() + arr = arr / max(arr.max(), 1e-6) + + return np.clip(arr, 0.0, 1.0) + + def _compute_quality_metrics(self, img: np.ndarray) -> dict[str, float]: + """分别计算对比度、清晰度、噪声与亮度分数""" + metrics = { + "contrast": self._evaluate_contrast(img), + "sharpness": self._evaluate_sharpness(img), + "noise": self._evaluate_noise(img), + "brightness": self._evaluate_brightness(img), + } + return {k: float(np.clip(v, 0.0, 1.0)) for k, v in metrics.items()} + + def _evaluate_contrast(self, img: np.ndarray) -> float: + std_dev = float(np.std(img)) + dynamic_range = float(np.ptp(img)) + percentile_low, percentile_high = np.percentile(img, (5, 95)) + percentile_range = float(max(percentile_high - percentile_low, 0.0)) + + std_component = min(std_dev / 0.18, 1.0) + range_component = min(dynamic_range / 0.85, 1.0) + percentile_component = min(percentile_range / 0.75, 1.0) + + return float( + np.clip( + 0.5 * std_component + 0.3 * range_component + 0.2 * percentile_component, + 0.0, + 1.0, + ) + ) + + def _evaluate_sharpness(self, img: np.ndarray) -> float: + if img.size == 0: + return 0.0 + + lap_response = self._apply_kernel(img, self._LAPLACIAN_KERNEL) + lap_var = float(np.mean(lap_response**2)) + lap_component = min(lap_var / 0.012, 1.0) + + grad_x = self._apply_kernel(img, self._SOBEL_X) + grad_y = self._apply_kernel(img, self._SOBEL_Y) + gradient_energy = float(np.mean(np.hypot(grad_x, grad_y))) + gradient_component = min(gradient_energy / 0.35, 1.0) + + return float(np.clip(0.6 * lap_component + 0.4 * gradient_component, 0.0, 1.0)) + + def _evaluate_noise(self, img: np.ndarray) -> float: + if min(img.shape) < 2: + return 0.8 + + smoothed = self._apply_kernel(img, self._BLUR_KERNEL) + high_freq = img - smoothed + grad_x = self._apply_kernel(img, self._SOBEL_X) + grad_y = self._apply_kernel(img, self._SOBEL_Y) + grad_mag = np.hypot(grad_x, grad_y) + + hf_energy = float(np.mean(high_freq**2)) + smooth_mask = grad_mag < 0.05 + if np.any(smooth_mask): + smooth_noise = float(np.mean(np.abs(high_freq[smooth_mask]))) + else: + smooth_noise = float(np.mean(np.abs(high_freq))) + + energy_component = max(1.0 - hf_energy / 0.015, 0.0) + smooth_component = max(1.0 - smooth_noise / 0.06, 0.0) + + return float(np.clip(0.5 * energy_component + 0.5 * smooth_component, 0.0, 1.0)) + + def _evaluate_brightness(self, img: np.ndarray) -> float: + if img.size == 0: + return 0.0 + + mean_intensity = float(np.mean(img)) + histogram, _ = np.histogram(img, bins=32, range=(0.0, 1.0)) + if histogram.sum() == 0: + return 0.0 + + hist = histogram.astype(np.float32) / float(histogram.sum()) + clipping_ratio = float(hist[0] + hist[-1]) + + mid_balance = max(1.0 - abs(mean_intensity - 0.5) * 2.0, 0.0) + distribution_score = max(1.0 - clipping_ratio * 0.5, 0.0) + + brightness_score = mid_balance * 0.7 + distribution_score * 0.3 + return float(np.clip(brightness_score, 0.0, 1.0)) + + def _combine_quality_scores(self, metrics: dict[str, float]) -> float: + weights_cfg = ( + self.config.get("image_processing", {}).get("quality_weights", {}) + if isinstance(self.config, dict) + else {} + ) + weights: dict[str, float] = {} + total = 0.0 + for key, default_value in self._DEFAULT_QUALITY_WEIGHTS.items(): + weight = float(weights_cfg.get(key, default_value)) + if weight < 0: + weight = 0.0 + weights[key] = weight + total += weight + + if total <= 0: + weights = self._DEFAULT_QUALITY_WEIGHTS.copy() + total = sum(weights.values()) + + for key in weights: + weights[key] /= total + + score = 0.0 + for key, weight in weights.items(): + score += float(metrics.get(key, 0.0)) * weight + + noise_metric = float(np.clip(metrics.get("noise", 1.0), 0.0, 1.0)) + dampening = 0.55 + 0.45 * noise_metric # 高噪声时额外惩罚总体分数 + return float(np.clip(score * dampening, 0.0, 1.0)) + + def _apply_kernel(self, img: np.ndarray, kernel: np.ndarray) -> np.ndarray: + """使用滑动窗口卷积核，必要时回退至循环计算""" + pad_y = kernel.shape[0] // 2 + pad_x = kernel.shape[1] // 2 + padded = np.pad(img, ((pad_y, pad_y), (pad_x, pad_x)), mode="reflect") + + if sliding_window_view is not None: + windows = sliding_window_view(padded, kernel.shape) + response = np.tensordot(windows, kernel, axes=((2, 3), (0, 1))) + return response.astype(np.float32, copy=False) + + return self._apply_kernel_fallback(padded, kernel) + + @staticmethod + def _apply_kernel_fallback(padded: np.ndarray, kernel: np.ndarray) -> np.ndarray: + """在旧版NumPy上使用逐像素卷积，保证算法可用""" + k_h, k_w = kernel.shape + out_h = padded.shape[0] - k_h + 1 + out_w = padded.shape[1] - k_w + 1 + output = np.empty((out_h, out_w), dtype=np.float32) + for i in range(out_h): + for j in range(out_w): + patch = padded[i : i + k_h, j : j + k_w] + output[i, j] = float(np.sum(patch * kernel)) + return output + + def _segment_vertebrae(self, image) -> list[dict[str, Any]]: + """分割椎体""" + # TODO: 使用分割模型识别 L1-L5 椎体 + # Issue URL: https://github.com/intellistream/SAGE/issues/896 + + # 模拟输出 + vertebrae_names = ["L1", "L2", "L3", "L4", "L5"] + return [ + { + "name": name, + "position": {"x": 100, "y": 50 + i * 80}, + "size": {"width": 40, "height": 30}, + "features": { + "signal_intensity": 0.7 + i * 0.02, + "shape_regularity": 0.9, + }, + } + for i, name in enumerate(vertebrae_names) + ] + + def _segment_discs(self, image) -> list[dict[str, Any]]: + """分割椎间盘""" + if image is None: + return self._generate_heuristic_discs(image) + + if self.segmentation_enabled and self.segmentation_model is not None: + try: + rgb_image = self._prepare_segmentation_input(image) + mask_outputs = self._run_segmentation_model(rgb_image) + discs = self._build_discs_from_masks(mask_outputs, image) + if discs: + return discs + print( + " Warning: Segmentation produced no disc candidates, using heuristic fallback" + ) + except Exception as exc: + print(f" Warning: Disc segmentation failed ({exc}), using heuristic fallback") + + return self._generate_heuristic_discs(image) + + def _prepare_segmentation_input(self, image: np.ndarray) -> np.ndarray: + """将灰度图转换为分割模型可用的 RGB 图""" + array = np.array(image) + + if array.dtype != np.uint8: + min_val = float(array.min()) if array.size else 0.0 + max_val = float(array.max()) if array.size else 0.0 + if max_val - min_val > 1e-5: + array = ((array - min_val) / (max_val - min_val) * 255).astype(np.uint8) + else: + array = np.zeros_like(array, dtype=np.uint8) + + if array.ndim == 2: + array = np.stack([array] * 3, axis=-1) + elif array.ndim == 3 and array.shape[2] == 1: + array = np.repeat(array, 3, axis=2) + + return array + + def _run_segmentation_model(self, rgb_image: np.ndarray) -> list[dict[str, Any]]: + """运行分割模型""" + if hasattr(self.segmentation_model, "generate"): + return self.segmentation_model.generate(rgb_image) + if callable(self.segmentation_model): + return self.segmentation_model(rgb_image) + raise RuntimeError("Segmentation model does not expose a generate() method") + + def _build_discs_from_masks( + self, masks: list[dict[str, Any]], original_image: np.ndarray + ) -> list[dict[str, Any]]: + """根据分割掩码构建椎间盘信息""" + if not masks: + return [] + + height, width = original_image.shape[:2] + candidates = self._select_disc_masks(masks, (height, width)) + if not candidates: + return [] + + candidates.sort(key=lambda item: item["center"][1]) + discs: list[dict[str, Any]] = [] + + for idx, candidate in enumerate(candidates): + if idx >= len(self.disc_levels): + break + features, position = self._compute_disc_features(original_image, candidate) + discs.append( + { + "level": self.disc_levels[idx], + "position": position, + "features": features, + } + ) + + return discs + + def _select_disc_masks( + self, masks: list[dict[str, Any]], image_shape: tuple[int, int] + ) -> list[dict[str, Any]]: + """筛选椎间盘候选掩码""" + height, width = image_shape + filtered: list[dict[str, Any]] = [] + + for mask in masks: + segmentation = mask.get("segmentation") + if segmentation is None: + continue + + segmentation = np.asarray(segmentation, dtype=bool) + + bbox = mask.get("bbox") + if bbox is None: + bbox = self._mask_to_bbox(segmentation) + if bbox is None: + continue + + x, y, w, h = [int(value) for value in bbox] + if w <= 0 or h <= 0: + continue + + height_ratio = h / max(height, 1) + width_ratio = w / max(width, 1) + aspect_ratio = w / (h + 1e-5) + center = (x + w / 2.0, y + h / 2.0) + vertical_pos = center[1] / max(height, 1) + confidence = float(mask.get("predicted_iou", mask.get("stability_score", 0.0))) + stability = float(mask.get("stability_score", confidence)) + + if height_ratio < self.segmentation_thresholds["min_height_ratio"]: + continue + if height_ratio > self.segmentation_thresholds["max_height_ratio"]: + continue + if width_ratio < self.segmentation_thresholds["min_width_ratio"]: + continue + if aspect_ratio < self.segmentation_thresholds["min_aspect_ratio"]: + continue + if not ( + self.segmentation_thresholds["min_vertical_pos"] + <= vertical_pos + <= self.segmentation_thresholds["max_vertical_pos"] + ): + continue + if confidence < self.segmentation_thresholds["min_confidence"]: + continue + + filtered.append( + { + "segmentation": segmentation, + "bbox": (x, y, w, h), + "center": center, + "confidence": confidence, + "stability": stability, + "area": float(mask.get("area", float(segmentation.sum()))), + } + ) + + if not filtered: + return [] + + # 移除过于接近的候选，保证每个椎间盘只保留一个掩码 + min_sep = self.segmentation_thresholds.get("min_separation_ratio", 0.05) * height + deduped: list[dict[str, Any]] = [] + filtered.sort(key=lambda item: (-item["confidence"], item["center"][1])) + for candidate in filtered: + if any( + abs(candidate["center"][1] - existing["center"][1]) < min_sep + for existing in deduped + ): + continue + deduped.append(candidate) + + deduped.sort(key=lambda item: item["center"][1]) + return deduped + + def _mask_to_bbox(self, mask: np.ndarray) -> tuple[int, int, int, int] | None: + """根据掩码计算包围盒""" + if mask.ndim != 2 or not mask.any(): + return None + + rows = np.any(mask, axis=1) + cols = np.any(mask, axis=0) + y_indices = np.where(rows)[0] + x_indices = np.where(cols)[0] + if y_indices.size == 0 or x_indices.size == 0: + return None + + y_min, y_max = int(y_indices[0]), int(y_indices[-1]) + x_min, x_max = int(x_indices[0]), int(x_indices[-1]) + return x_min, y_min, x_max - x_min + 1, y_max - y_min + 1 + + def _compute_disc_features( + self, image: np.ndarray, mask_info: dict[str, Any] + ) -> tuple[dict[str, Any], dict[str, int]]: + """从掩码中提取椎间盘特征""" + x, y, w, h = mask_info["bbox"] + height, width = image.shape[:2] + crop = image[y : y + h, x : x + w] + segmentation = mask_info["segmentation"][y : y + h, x : x + w] + + if crop.ndim == 3: + crop_gray = crop[:, :, 0] + else: + crop_gray = crop + + if segmentation.any(): + disc_pixels = crop_gray[segmentation] + intensity = float(disc_pixels.mean() / 255.0) if disc_pixels.size else 0.0 + else: + intensity = float(crop_gray.mean() / 255.0) if crop_gray.size else 0.0 + + height_ratio = h / max(height, 1) + height_mm = round(height_ratio * self.pixel_spacing_mm, 2) + degeneration_score = max(0.0, 1.0 - height_mm / max(self.baseline_disc_height_mm, 1e-3)) + irregularity = 1.0 - mask_info["area"] / max(w * h, 1) + lateral_shift = self._compute_lateral_imbalance(segmentation) + + herniation = bool( + irregularity > self.segmentation_config.get("irregularity_threshold", 0.35) + or lateral_shift > self.segmentation_config.get("lateral_shift_threshold", 0.18) + or degeneration_score > 0.35 + ) + + features = { + "height": max(height_mm, 0.1), + "height_ratio": round(height_ratio, 4), + "signal_intensity": round(max(min(intensity, 1.0), 0.0), 3), + "degeneration_score": round(degeneration_score, 3), + "irregularity": round(max(irregularity, 0.0), 3), + "lateral_shift": round(lateral_shift, 3), + "herniation": herniation, + "mask_confidence": round(mask_info.get("confidence", 0.0), 3), + "stability": round(mask_info.get("stability", 0.0), 3), + } + + position = {"x": int(x + w / 2), "y": int(y + h / 2)} + return features, position + + def _compute_lateral_imbalance(self, mask: np.ndarray) -> float: + """计算左右侧面积差异，用于判断椎间盘偏移""" + if mask.size == 0: + return 0.0 + + column_profile = mask.sum(axis=0).astype(np.float32) + mid = column_profile.size // 2 + left = column_profile[:mid].sum() + right = column_profile[mid:].sum() + total = left + right + if total == 0: + return 0.0 + return float(abs(left - right) / total) + + def _generate_heuristic_discs(self, image: np.ndarray | None) -> list[dict[str, Any]]: + """基于几何启发式生成椎间盘信息（在缺少模型时使用）""" + height, width = image.shape[:2] if isinstance(image, np.ndarray) else (512, 512) + discs: list[dict[str, Any]] = [] + + for idx, level in enumerate(self.disc_levels): + y = int((0.2 + idx * 0.12) * height) + x = width // 2 + window = None + if isinstance(image, np.ndarray): + y0 = max(y - 4, 0) + y1 = min(y + 4, height) + window = image[y0:y1, :] + + if window is not None and window.size: + intensity = float(window.mean() / 255.0) + else: + intensity = max(0.0, 0.75 - idx * 0.08) + + height_value = max(self.baseline_disc_height_mm - idx * 0.45, 4.0) + discs.append( + { + "level": level, + "position": {"x": x, "y": y}, + "features": { + "height": round(height_value, 2), + "height_ratio": round( + height_value / max(self.baseline_disc_height_mm, 1e-3), 3 + ), + "signal_intensity": round(max(min(intensity, 1.0), 0.0), 3), + "herniation": idx >= 3, + "degeneration_score": round(idx * 0.1, 3), + "irregularity": 0.2 + idx * 0.05, + "lateral_shift": 0.05 * idx, + "mask_confidence": 0.0, + "stability": 0.0, + }, + } + ) + + return discs + + def _detect_abnormalities( + self, image, vertebrae: list[dict], discs: list[dict] + ) -> list[dict[str, Any]]: + """检测异常""" + abnormalities = [] + + # 检查椎间盘突出 + for disc in discs: + if disc["features"].get("herniation"): + abnormalities.append( + { + "type": "disc_herniation", + "location": disc["level"], + "severity": "moderate", + "description": f"{disc['level']} 椎间盘突出", + } + ) + + # 检查椎间盘退变 + for disc in discs: + if disc["features"]["height"] < 6.0: + abnormalities.append( + { + "type": "disc_degeneration", + "location": disc["level"], + "severity": ("mild" if disc["features"]["height"] > 5.0 else "moderate"), + "description": f"{disc['level']} 椎间盘退行性变", + } + ) + + # 检查信号强度异常 + for disc in discs: + if disc["features"]["signal_intensity"] < 0.6: + abnormalities.append( + { + "type": "signal_change", + "location": disc["level"], + "severity": "mild", + "description": f"{disc['level']} 椎间盘信号减低", + } + ) + + return abnormalities + + def _extract_features(self, image) -> np.ndarray | None: + """提取影像特征向量 + + 使用预训练模型 (CLIP/DINOv2) 提取影像特征 + 如果模型不可用，回退到模拟特征 + + Args: + image: 输入图像 (numpy array 或 PIL Image) + + Returns: + 特征向量 (numpy array) 或 None + """ + if image is None: + return None + + # 如果没有加载特征提取模型，使用模拟特征 + if self.feature_model is None or not TORCH_AVAILABLE: + print(" Using mock features (model not available)") + return self._create_mock_features() + + try: + # 转换图像为 PIL Image (如果是 numpy array) + if isinstance(image, np.ndarray): + # 确保是 uint8 类型 + if image.dtype != np.uint8: + image = ((image - image.min()) / (image.max() - image.min()) * 255).astype( + np.uint8 + ) + # 转换灰度图为 RGB + if len(image.shape) == 2: + image = np.stack([image] * 3, axis=-1) + pil_image = PILImage.fromarray(image) + else: + pil_image = image + + # 使用对应的模型提取特征 + if self.feature_method == "clip": + features = self._extract_clip_features(pil_image) + elif self.feature_method == "dinov2": + features = self._extract_dinov2_features(pil_image) + else: + print(f" Unknown feature method: {self.feature_method}") + features = self._create_mock_features() + + return features + + except Exception as e: + print(f" Warning: Feature extraction failed: {e}") + print(" Falling back to mock features") + return self._create_mock_features() + + def _extract_clip_features(self, image: "PILImage.Image") -> np.ndarray: + """使用 CLIP 提取特征""" + with torch.no_grad(): + # 预处理图像 + inputs = self.feature_processor(images=image, return_tensors="pt") + + # 移动到正确的设备 + if torch.cuda.is_available(): + inputs = {k: v.cuda() for k, v in inputs.items()} + + # 提取图像特征 + image_features = self.feature_model.get_image_features(**inputs) + + # 转换为 numpy array + features = image_features.cpu().numpy().squeeze() + + # 归一化 + features = features / np.linalg.norm(features) + + return features.astype(np.float32) + + def _extract_dinov2_features(self, image: "PILImage.Image") -> np.ndarray: + """使用 DINOv2 提取特征""" + with torch.no_grad(): + # 预处理图像 + inputs = self.feature_processor(images=image, return_tensors="pt") + + # 移动到正确的设备 + if torch.cuda.is_available(): + inputs = {k: v.cuda() for k, v in inputs.items()} + + # 提取特征 + outputs = self.feature_model(**inputs) + + # 使用 [CLS] token 的输出作为图像特征 + features = outputs.last_hidden_state[:, 0].cpu().numpy().squeeze() + + # 归一化 + features = features / np.linalg.norm(features) + + return features.astype(np.float32) + + def _create_mock_features(self) -> np.ndarray: + """创建模拟特征向量 (用于测试或回退)""" + # 获取配置的特征维度，默认 768 + dimension = ( + self.config.get("image_processing", {}) + .get("feature_extraction", {}) + .get("dimension", 768) + ) + return np.random.randn(dimension).astype(np.float32) + + def _create_mock_analysis(self) -> dict[str, Any]: + """创建模拟分析结果（用于演示）""" + return { + "vertebrae": [ + {"name": f"L{i}", "position": {"x": 100, "y": 50 + i * 80}} for i in range(1, 6) + ], + "discs": self._generate_heuristic_discs(None), + "abnormalities": [ + { + "type": "disc_herniation", + "location": "L4/L5", + "severity": "moderate", + "description": "L4/L5 椎间盘突出", + }, + { + "type": "disc_degeneration", + "location": "L5/S1", + "severity": "mild", + "description": "L5/S1 椎间盘退行性变", + }, + ], + "image_quality": 0.85, + "image_embedding": np.random.randn(768).astype(np.float32), + } + + +if __name__ == "__main__": + # 测试 + config = {"models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}} + + analyzer = ImageAnalyzer(config) + result = analyzer.analyze("test.jpg") + + print(f"椎体数量: {len(result['vertebrae'])}") + print(f"椎间盘数量: {len(result['discs'])}") + print(f"异常数量: {len(result['abnormalities'])}") diff --git a/sage-apps/src/sage/apps/medical_diagnosis/agents/report_generator.py b/sage-apps/src/sage/apps/medical_diagnosis/agents/report_generator.py new file mode 100644 index 0000000..34cd804 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/agents/report_generator.py @@ -0,0 +1,497 @@ +""" +报告生成Agent +负责生成结构化的诊断报告 +""" + +import contextlib +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +try: # Optional dependency - fall back to template reports if unavailable + from sage.common.components.sage_vllm import VLLMService + + _VLLM_COMPONENT_AVAILABLE = True +except Exception: # pragma: no cover - vLLM extras may be missing in CI + VLLMService = None # type: ignore + _VLLM_COMPONENT_AVAILABLE = False + + +@dataclass +class DiagnosisReport: + """诊断报告""" + + diagnoses: list[str] # 诊断列表 + confidence: float # 置信度 + findings: list[str] # 发现列表 + recommendations: list[str] # 建议列表 + similar_cases: list[dict[str, Any]] # 相似病例 + report: str # 完整报告 + timestamp: str # 时间戳 + quality_score: float = 0.0 # 质量评分 + + +class ReportGenerator: + """ + 诊断报告生成器 + + 功能: + 1. 整合影像分析结果 + 2. 参考相似病例 + 3. 生成结构化诊断报告 + 4. 提供治疗建议 + """ + + def __init__(self, config: dict): + """ + 初始化报告生成器 + + Args: + config: 配置字典 + """ + self.config = config + self.llm_service = None + self._llm_generate_options: dict[str, Any] = {} + self._setup_llm() + + def _setup_llm(self): + """设置LLM服务""" + print(f" Loading LLM: {self.config['models']['llm_model']}") + + models_cfg = self.config.get("models", {}) or {} + services_cfg = self.config.get("services", {}) or {} + vllm_cfg = dict(services_cfg.get("vllm", {}) or {}) + self._llm_generate_options = self._resolve_generation_options(vllm_cfg) + + if not vllm_cfg.get("enabled", True): + print(" Warning: vLLM service disabled via config, using template reports") + return + + llm_model = models_cfg.get("llm_model") + if not llm_model: + print(" Warning: Missing 'llm_model' config, using template reports") + return + + if not _VLLM_COMPONENT_AVAILABLE or VLLMService is None: + print(" Warning: sage_vllm component unavailable, using template reports") + return + + try: + service_config = self._build_vllm_config(llm_model, vllm_cfg, models_cfg) + self.llm_service = VLLMService(service_config) + self.llm_service.setup() + print(" ✓ vLLM service ready") + except Exception as exc: # pragma: no cover - runtime dependency issues + print(f" Warning: Failed to initialize vLLM service ({exc}), using template reports") + self.llm_service = None + + def generate( + self, + image_features: dict[str, Any], + patient_info: dict[str, Any] | None, + similar_cases: list[dict], + medical_knowledge: list[dict], + ) -> DiagnosisReport: + """ + 生成诊断报告 + + Args: + image_features: 影像特征 + patient_info: 患者信息 + similar_cases: 相似病例 + medical_knowledge: 医学知识 + + Returns: + DiagnosisReport: 诊断报告 + """ + # Step 1: 构建提示词 + prompt = self._build_prompt(image_features, patient_info, similar_cases, medical_knowledge) + + # Step 2: 调用LLM生成报告 + if self.llm_service is not None: + try: + report_text = self._generate_with_llm(prompt) + except Exception as exc: # pragma: no cover - network/runtime failures + print(f" Warning: vLLM generation failed ({exc}), falling back to template") + report_text = self._generate_template_report( + image_features, patient_info, similar_cases + ) + else: + report_text = self._generate_template_report( + image_features, patient_info, similar_cases + ) + + # Step 3: 提取诊断和建议 + diagnosis_summary, findings, recommendations = self._parse_report( + report_text, image_features + ) + + # 提取诊断列表 + diagnoses_list = [] + for abnorm in image_features.get("abnormalities", []): + if abnorm["type"] == "disc_herniation": + diagnoses_list.append(f"{abnorm['location']}椎间盘突出症") + elif abnorm["type"] == "disc_degeneration": + diagnoses_list.append(f"{abnorm['location']}椎间盘退行性变") + + if not diagnoses_list: + diagnoses_list = ["未见明显异常"] + + # Step 4: 计算置信度 + confidence = self._calculate_confidence(image_features, similar_cases) + + # 获取时间戳 + from datetime import datetime + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + return DiagnosisReport( + diagnoses=diagnoses_list, + confidence=confidence, + findings=findings, + recommendations=recommendations, + similar_cases=similar_cases, + report=report_text, + timestamp=timestamp, + quality_score=image_features.get("image_quality", 0.0), + ) + + def _build_prompt( + self, + image_features: dict, + patient_info: dict | None, + similar_cases: list[dict], + medical_knowledge: list[dict], + ) -> str: + """构建LLM提示词""" + prompt_parts = [ + "你是一位经验丰富的脊柱外科医生，正在为患者撰写腰椎MRI诊断报告。", + "", + "## 影像分析结果", + ] + + # 添加影像发现 + if image_features.get("abnormalities"): + prompt_parts.append("\n### 主要发现:") + for abnorm in image_features["abnormalities"]: + prompt_parts.append( + f"- {abnorm['location']}: {abnorm['description']} " + f"(严重程度: {abnorm['severity']})" + ) + + # 添加患者信息 + if patient_info: + prompt_parts.append("\n## 患者信息") + if "age" in patient_info: + prompt_parts.append(f"年龄: {patient_info['age']}岁") + if "gender" in patient_info: + prompt_parts.append(f"性别: {patient_info['gender']}") + if "symptoms" in patient_info: + prompt_parts.append(f"主诉: {patient_info['symptoms']}") + + # 添加相似病例参考 + if similar_cases: + prompt_parts.append("\n## 相似病例参考") + for i, case in enumerate(similar_cases[:3], 1): + prompt_parts.append(f"\n病例{i}:") + prompt_parts.append(f" 诊断: {case.get('diagnosis', 'N/A')}") + prompt_parts.append(f" 治疗: {case.get('treatment', 'N/A')}") + + # 添加医学知识 + if medical_knowledge: + prompt_parts.append("\n## 相关医学知识") + for knowledge in medical_knowledge: + prompt_parts.append(f"- {knowledge.get('content', '')}") + + prompt_parts.append("\n## 任务") + prompt_parts.append("请基于以上信息，撰写一份专业的诊断报告，包括:") + prompt_parts.append("1. 影像描述") + prompt_parts.append("2. 诊断结论") + prompt_parts.append("3. 治疗建议") + + return "\n".join(prompt_parts) + + def _generate_with_llm(self, prompt: str) -> str: + """使用LLM生成报告""" + if self.llm_service is None: + raise RuntimeError("LLM service not initialized") + + options = {k: v for k, v in self._llm_generate_options.items() if v is not None} + outputs = self.llm_service.generate(prompt, **options) + if not outputs: + raise RuntimeError("vLLM service returned no outputs") + + generations = outputs[0].get("generations") or [] + if not generations: + raise RuntimeError("vLLM service returned empty generations") + + text = generations[0].get("text", "") + return text.strip() + + def _resolve_generation_options(self, vllm_cfg: dict[str, Any]) -> dict[str, Any]: + """解析采样/生成参数供每次请求使用""" + + sampling_cfg = ( + vllm_cfg.get("sampling", {}) if isinstance(vllm_cfg.get("sampling"), dict) else {} + ) + + def _get_option(key: str, default: Any): + if sampling_cfg.get(key) is not None: + return sampling_cfg[key] + if vllm_cfg.get(key) is not None: + return vllm_cfg[key] + return default + + try: + max_tokens = int(_get_option("max_tokens", 768)) + except (TypeError, ValueError): + max_tokens = 768 + + try: + temperature = float(_get_option("temperature", 0.7)) + except (TypeError, ValueError): + temperature = 0.7 + + try: + top_p = float(_get_option("top_p", 0.95)) + except (TypeError, ValueError): + top_p = 0.95 + + return { + "max_tokens": max(1, max_tokens), + "temperature": max(0.0, temperature), + "top_p": min(max(top_p, 0.0), 1.0), + } + + def _build_vllm_config( + self, + llm_model: str, + vllm_cfg: dict[str, Any], + models_cfg: dict[str, Any], + ) -> dict[str, Any]: + """构建传递给 VLLMService 的配置字典""" + + engine_cfg = {} + engine_cfg.update( + vllm_cfg.get("engine", {}) if isinstance(vllm_cfg.get("engine"), dict) else {} + ) + engine_cfg.setdefault("dtype", vllm_cfg.get("dtype", "auto")) + engine_cfg.setdefault("tensor_parallel_size", int(vllm_cfg.get("tensor_parallel_size", 1))) + engine_cfg.setdefault( + "gpu_memory_utilization", float(vllm_cfg.get("gpu_memory_utilization", 0.9)) + ) + engine_cfg.setdefault("max_model_len", int(vllm_cfg.get("max_model_len", 4096))) + + sampling_cfg = {} + sampling_cfg.update( + vllm_cfg.get("sampling", {}) if isinstance(vllm_cfg.get("sampling"), dict) else {} + ) + for key, value in self._llm_generate_options.items(): + sampling_cfg.setdefault(key, value) + + config_dict: dict[str, Any] = { + "model_id": llm_model, + "auto_download": bool(vllm_cfg.get("auto_download", True)), + "auto_reload": bool(vllm_cfg.get("auto_reload", True)), + "engine": engine_cfg, + "sampling": sampling_cfg, + } + + embedding_model = vllm_cfg.get("embedding_model_id") or models_cfg.get("embedding_model") + if embedding_model: + config_dict["embedding_model_id"] = embedding_model + + return config_dict + + def __del__(self): # pragma: no cover - best-effort cleanup + if getattr(self, "llm_service", None) and hasattr(self.llm_service, "cleanup"): + with contextlib.suppress(Exception): + self.llm_service.cleanup() + + def _generate_template_report( + self, + image_features: dict, + patient_info: dict | None, + similar_cases: list[dict], + ) -> str: + """使用模板生成报告（演示用）""" + report_parts = [] + + # 报告头 + report_parts.append("=" * 60) + report_parts.append("腰椎MRI诊断报告") + report_parts.append("=" * 60) + report_parts.append(f"报告时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + report_parts.append("") + + # 患者信息 + if patient_info: + report_parts.append("【患者信息】") + if "age" in patient_info: + report_parts.append(f"年龄: {patient_info['age']}岁") + if "gender" in patient_info: + gender_cn = "男" if patient_info["gender"] == "male" else "女" + report_parts.append(f"性别: {gender_cn}") + if "symptoms" in patient_info: + report_parts.append(f"主诉: {patient_info['symptoms']}") + report_parts.append("") + + # 影像描述 + report_parts.append("【影像描述】") + report_parts.append("腰椎序列正常，可见L1-L5椎体及L1/L2-L5/S1椎间盘。") + report_parts.append("") + + # 主要发现 + report_parts.append("【主要发现】") + abnormalities = image_features.get("abnormalities", []) + + if abnormalities: + for abnorm in abnormalities: + severity_cn = { + "mild": "轻度", + "moderate": "中度", + "severe": "重度", + }.get(abnorm.get("severity", "mild"), "轻度") + + report_parts.append( + f"{abnorm['location']}: {abnorm['description']} ({severity_cn})" + ) + else: + report_parts.append("未见明显异常。") + + report_parts.append("") + + # 诊断结论 + report_parts.append("【诊断结论】") + diagnoses = [] + + for abnorm in abnormalities: + if abnorm["type"] == "disc_herniation": + diagnoses.append(f"{abnorm['location']}椎间盘突出症") + elif abnorm["type"] == "disc_degeneration": + diagnoses.append(f"{abnorm['location']}椎间盘退行性变") + + if diagnoses: + for i, diag in enumerate(diagnoses, 1): + report_parts.append(f"{i}. {diag}") + else: + report_parts.append("腰椎MRI未见明显异常。") + + report_parts.append("") + + # 治疗建议 + report_parts.append("【建议】") + + if any(a["type"] == "disc_herniation" for a in abnormalities): + report_parts.append("1. 建议休息，避免重体力劳动") + report_parts.append("2. 可考虑物理治疗和药物治疗") + report_parts.append("3. 如保守治疗无效，可考虑微创或手术治疗") + report_parts.append("4. 建议脊柱外科门诊随诊") + elif any(a["type"] == "disc_degeneration" for a in abnormalities): + report_parts.append("1. 注意腰部保护，避免久坐久站") + report_parts.append("2. 适当功能锻炼，增强腰背肌力量") + report_parts.append("3. 必要时可行保守治疗") + report_parts.append("4. 定期复查") + else: + report_parts.append("1. 保持良好生活习惯") + report_parts.append("2. 适当运动锻炼") + report_parts.append("3. 如有症状变化，及时就诊") + + report_parts.append("") + + # 相似病例参考 + if similar_cases: + report_parts.append("【参考病例】") + report_parts.append(f"检索到 {len(similar_cases)} 个相似病例供参考。") + + report_parts.append("") + report_parts.append("=" * 60) + report_parts.append("注: 本报告由AI辅助生成，仅供参考，最终诊断需由专业医师确认。") + report_parts.append("=" * 60) + + return "\n".join(report_parts) + + def _parse_report(self, report_text: str, image_features: dict) -> tuple: + """解析报告提取诊断和建议""" + # 从报告中提取诊断 + diagnoses = [] + findings = [] + recommendations = [] + + # 提取异常发现 + for abnorm in image_features.get("abnormalities", []): + findings.append(f"{abnorm['location']}: {abnorm['description']}") + + if abnorm["type"] == "disc_herniation": + diagnoses.append(f"{abnorm['location']}椎间盘突出症") + elif abnorm["type"] == "disc_degeneration": + diagnoses.append(f"{abnorm['location']}椎间盘退行性变") + + # 生成建议 + has_herniation = any( + a["type"] == "disc_herniation" for a in image_features.get("abnormalities", []) + ) + + if has_herniation: + recommendations = [ + "建议休息，避免重体力劳动", + "可考虑物理治疗和药物治疗", + "如保守治疗无效，可考虑微创或手术治疗", + ] + else: + recommendations = ["保持良好生活习惯", "适当运动锻炼", "定期复查"] + + diagnosis_summary = "; ".join(diagnoses) if diagnoses else "未见明显异常" + + return diagnosis_summary, findings, recommendations + + def _calculate_confidence(self, image_features: dict, similar_cases: list[dict]) -> float: + """计算诊断置信度""" + confidence = 0.5 # 基础置信度 + + # 影像质量影响 + quality = image_features.get("image_quality", 0.5) + confidence += quality * 0.2 + + # 相似病例数量影响 + if len(similar_cases) >= 5: + confidence += 0.2 + elif len(similar_cases) >= 3: + confidence += 0.1 + + # 异常明确性影响 + abnormalities = image_features.get("abnormalities", []) + if abnormalities: + # 有明确病变，置信度提升 + confidence += 0.1 + + return min(confidence, 1.0) + + +if __name__ == "__main__": + # 测试 + config = {"models": {"llm_model": "Qwen/Qwen2.5-7B-Instruct"}} + + generator = ReportGenerator(config) + + # 模拟数据 + image_features = { + "abnormalities": [ + { + "type": "disc_herniation", + "location": "L4/L5", + "severity": "moderate", + "description": "L4/L5 椎间盘突出", + } + ], + "image_quality": 0.85, + } + + result = generator.generate( + image_features=image_features, + patient_info={"age": 45, "gender": "male", "symptoms": "腰痛"}, + similar_cases=[], + medical_knowledge=[], + ) + + print(result.report) diff --git a/sage-apps/src/sage/apps/medical_diagnosis/config/agent_config.yaml b/sage-apps/src/sage/apps/medical_diagnosis/config/agent_config.yaml new file mode 100644 index 0000000..afc7475 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/config/agent_config.yaml @@ -0,0 +1,126 @@ +# 医疗诊断Agent配置 + +agent: + name: "LumbarSpineDiagnosticAgent" + version: "1.0.0" + description: "腰椎MRI诊断智能助手" + max_iterations: 5 + timeout: 300 # 秒 + +models: + # 视觉理解模型（用于影像分析） + vision_model: "Qwen/Qwen2-VL-7B-Instruct" + + # 语言模型（用于报告生成） + llm_model: "Qwen/Qwen2.5-7B-Instruct" + + # 嵌入模型（用于知识检索） + embedding_model: "BAAI/bge-large-zh-v1.5" + +services: + # VLLM服务配置 + vllm: + enabled: true + gpu_memory_utilization: 0.9 + tensor_parallel_size: 1 + max_model_len: 4096 + dtype: "auto" + + # Embedding服务配置 + embedding: + method: "hf" + cache_enabled: true + normalize: true + batch_size: 32 + + # 向量数据库配置 + vector_db: + backend: "sage_db" + collection_name: "lumbar_spine_cases" + dimension: 1024 + index_type: "HNSW" + metric_type: "cosine" + top_k: 5 + +image_processing: + # 图像预处理 + input_size: [512, 512] + normalization: true + augmentation: false + + # 分割模型 + segmentation: + enabled: false # 没有GPU或模型权重时可以暂时关闭 + model: "SAM" # Segment Anything Model + checkpoint_path: ".sage/models/sam_vit_h.pth" # SAM 权重路径 + sam_variant: "vit_h" # vit_b / vit_l / vit_h + points_per_side: 16 + pred_iou_thresh: 0.86 + stability_score_thresh: 0.92 + min_mask_region_area: 200 + pixel_spacing_mm: 180.0 # MRI 像素间距估计（mm） + baseline_disc_height_mm: 9.0 # 正常椎间盘高度（mm） + irregularity_threshold: 0.35 + lateral_shift_threshold: 0.18 + disc_levels: + - L1/L2 + - L2/L3 + - L3/L4 + - L4/L5 + - L5/S1 + thresholds: + min_height_ratio: 0.02 + max_height_ratio: 0.18 + min_aspect_ratio: 1.5 + min_width_ratio: 0.12 + min_vertical_pos: 0.08 + max_vertical_pos: 0.95 + min_confidence: 0.35 + min_separation_ratio: 0.05 + + # 特征提取 + feature_extraction: + method: "clip" # 特征提取方法: "clip" (推荐) 或 "dinov2" + dimension: 768 # 特征向量维度 (CLIP: 512, DINOv2: 768) + +diagnosis: + # 诊断阈值 + confidence_threshold: 0.7 + + # 病变检测 + detection: + disc_herniation_threshold: 0.8 + degeneration_threshold: 0.6 + + # 报告生成 + report: + include_similar_cases: true + max_similar_cases: 5 + include_confidence_score: true + language: "zh" + +knowledge_base: + # 知识来源 + sources: + - "medical_literature" + - "clinical_guidelines" + - "case_database" + + # 更新策略 + update_frequency: "weekly" + auto_update: false + +output: + # 输出格式 + format: "structured" # structured, json, pdf + save_path: "output/diagnoses/" + + # 包含内容 + include_image_annotations: true + include_similar_cases: true + include_references: true + +logging: + level: "INFO" + save_logs: true + log_path: "logs/medical_diagnosis/" diff --git a/sage-apps/src/sage/apps/medical_diagnosis/run_diagnosis.py b/sage-apps/src/sage/apps/medical_diagnosis/run_diagnosis.py new file mode 100644 index 0000000..9ebf9f7 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/run_diagnosis.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +""" +腰椎MRI医疗诊断Agent - 主运行脚本 + +功能: +1. 单个病例诊断 +2. 批量病例处理 +3. 交互式诊断会话 +""" + +import argparse +import subprocess +from pathlib import Path +from typing import Any + +from sage.apps.medical_diagnosis.agents import DiagnosticAgent + + +def check_and_setup_data(auto_setup: bool = False): + """检查数据是否存在，如果不存在则自动下载和准备""" + # 获取当前脚本所在目录 + current_dir = Path(__file__).parent + data_dir = current_dir / "data" / "processed" + setup_script = current_dir / "setup_data.sh" + + # 检查数据是否存在 + if data_dir.exists() and (data_dir / "train_index.json").exists(): + print(f"✓ 数据集已就绪: {data_dir}") + return True + + print(f"⚠️ 数据集未找到: {data_dir}") + print("") + + # 检查 setup_data.sh 是否存在 + if not setup_script.exists(): + print(f"❌ 数据设置脚本未找到: {setup_script}") + print("") + print("请手动准备数据或检查安装是否完整") + return False + + # 自动下载数据 + print(f"数据集设置脚本: {setup_script}") + print("") + print("🤖 自动下载并准备数据集...") + print("提示: 如果不想自动下载，请使用 Ctrl+C 取消") + print("") + print("开始自动设置数据集...") + print("=" * 70) + + try: + # 运行 setup_data.sh + subprocess.run( + ["bash", str(setup_script)], + cwd=str(current_dir), + check=True, + text=True, + capture_output=False, + ) + + print("=" * 70) + print("✅ 数据集设置完成！") + return True + + except subprocess.CalledProcessError: + print("=" * 70) + print("❌ 数据集设置失败") + print("") + print("您可以手动运行以下命令来设置数据:") + print(f" bash {setup_script}") + print("") + print("或者查看错误日志以获取更多信息") + return False + except Exception as e: + print(f"❌ 发生错误: {e}") + return False + + +def parse_args(): + """解析命令行参数""" + parser = argparse.ArgumentParser( + description="腰椎MRI医疗诊断Agent", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + # 诊断单个病例 + python run_diagnosis.py --image data/medical/test/case_001.jpg --age 45 --gender male --symptoms "腰痛伴左腿麻木" + + # 批量处理 + python run_diagnosis.py --batch data/medical/batch_cases/ --output output/diagnoses/ + + # 交互式模式 + python run_diagnosis.py --interactive + """, + ) + + # 单个病例诊断参数 + parser.add_argument("--image", "-i", type=str, help="MRI影像路径") + + parser.add_argument("--age", type=int, help="患者年龄") + + parser.add_argument( + "--gender", type=str, choices=["male", "female"], help="患者性别 (male/female)" + ) + + parser.add_argument("--symptoms", type=str, help="患者症状描述") + + # 批量处理参数 + parser.add_argument("--batch", type=str, help="批量处理目录") + + parser.add_argument( + "--output", + "-o", + type=str, + default="output/diagnoses/", + help="输出目录 (默认: output/diagnoses/)", + ) + + # 其他参数 + parser.add_argument( + "--config", + "-c", + type=str, + default="examples/medical_diagnosis/config/agent_config.yaml", + help="配置文件路径", + ) + + parser.add_argument("--interactive", action="store_true", help="交互式模式") + + parser.add_argument("--verbose", "-v", action="store_true", help="显示详细信息") + + parser.add_argument("--auto-setup", action="store_true", help="自动下载并设置数据（无需确认）") + + return parser.parse_args() + + +def diagnose_single_case( + agent: DiagnosticAgent, + image_path: str, + patient_info: dict[str, Any] | None = None, + verbose: bool = True, +): + """诊断单个病例""" + print(f"\n{'=' * 70}") + print("🏥 腰椎MRI诊断系统") + print(f"{'=' * 70}\n") + + # 执行诊断 + result = agent.diagnose(image_path=image_path, patient_info=patient_info, verbose=verbose) + + print(f"\n{'=' * 70}") + print("✅ 诊断完成") + print(f"{'=' * 70}") + # DiagnosisReport 使用 diagnoses (复数) 而不是 diagnosis + if hasattr(result, "diagnoses") and result.diagnoses: + print(f"诊断: {', '.join(result.diagnoses)}") + print(f"置信度: {result.confidence:.2%}") + + return result + + +def batch_diagnose(agent: DiagnosticAgent, batch_dir: str, output_dir: str): + """批量诊断""" + batch_path = Path(batch_dir) + + if not batch_path.exists(): + print(f"❌ 错误: 批量处理目录不存在: {batch_dir}") + return + + # 查找所有图像文件 + image_files = [] + for ext in ["*.jpg", "*.jpeg", "*.png", "*.dcm"]: + image_files.extend(batch_path.glob(ext)) + + if not image_files: + print("❌ 错误: 未找到图像文件") + return + + print("\n🏥 批量诊断模式") + print(f"{'=' * 70}") + print(f"📂 输入目录: {batch_dir}") + print(f"📁 输出目录: {output_dir}") + print(f"📊 病例数量: {len(image_files)}") + print(f"{'=' * 70}\n") + + # 构建病例列表 + cases = [{"image_path": str(img), "patient_info": {"case_id": img.stem}} for img in image_files] + + # 批量处理 + results = agent.batch_diagnose(cases=cases, output_dir=output_dir) + + print("\n📊 批量诊断统计:") + print(f" 总病例数: {len(results)}") + + # 统计诊断结果 + diagnoses_count = {} + for result in results: + for diag in result.diagnoses: + diagnoses_count[diag] = diagnoses_count.get(diag, 0) + 1 + + print("\n 诊断分布:") + for diag, count in sorted(diagnoses_count.items(), key=lambda x: -x[1]): + print(f" - {diag}: {count} 例") + + +def interactive_mode(agent: DiagnosticAgent): + """交互式诊断模式""" + print(f"\n{'=' * 70}") + print("🏥 腰椎MRI诊断系统 - 交互式模式") + print(f"{'=' * 70}") + print("\n输入 'exit' 或 'quit' 退出\n") + + while True: + print(f"\n{'─' * 70}") + + # 输入影像路径 + image_path = input("📄 请输入MRI影像路径: ").strip() + + if image_path.lower() in ["exit", "quit"]: + print("\n👋 再见！") + break + + if not image_path: + print("❌ 路径不能为空") + continue + + # 输入患者信息 + print("\n👤 患者信息（可选，直接回车跳过）:") + + age_input = input(" 年龄: ").strip() + age = int(age_input) if age_input.isdigit() else None + + gender_input = input(" 性别 (male/female): ").strip().lower() + gender = gender_input if gender_input in ["male", "female"] else None + + symptoms = input(" 症状: ").strip() + + # 构建患者信息 + patient_info = {} + if age: + patient_info["age"] = age + if gender: + patient_info["gender"] = gender + if symptoms: + patient_info["symptoms"] = symptoms + + # 执行诊断 + try: + agent.diagnose( + image_path=image_path, + patient_info=patient_info if patient_info else None, + verbose=True, + ) + except Exception as e: + print(f"\n❌ 诊断失败: {e}") + continue + + # 询问是否继续 + continue_input = input("\n是否继续诊断下一个病例? (y/n): ").strip().lower() + if continue_input != "y": + print("\n👋 再见！") + break + + +def main(): + """主函数""" + args = parse_args() + + # 检查并设置数据（如果需要） + print(f"\n{'=' * 70}") + print("📦 检查数据集状态...") + print(f"{'=' * 70}\n") + + if not check_and_setup_data(auto_setup=args.auto_setup): + print("\n⚠️ 警告: 数据集未就绪") + print("系统将使用模拟数据运行演示模式") + print("") + + # 初始化Agent + print("\n🚀 初始化诊断Agent...") + print(f"{'=' * 70}") + + config_path = args.config if Path(args.config).exists() else None + agent = DiagnosticAgent(config_path=config_path) + + print(f"{'=' * 70}\n") + + # 根据参数执行不同模式 + if args.interactive: + # 交互式模式 + interactive_mode(agent) + + elif args.batch: + # 批量处理模式 + batch_diagnose(agent=agent, batch_dir=args.batch, output_dir=args.output) + + elif args.image: + # 单个病例诊断 + patient_info = {} + if args.age: + patient_info["age"] = args.age + if args.gender: + patient_info["gender"] = args.gender + if args.symptoms: + patient_info["symptoms"] = args.symptoms + + diagnose_single_case( + agent=agent, + image_path=args.image, + patient_info=patient_info if patient_info else None, + verbose=args.verbose, + ) + + else: + # 演示模式（使用模拟数据） + print("\n💡 演示模式（使用模拟数据）\n") + print("提示: 使用 --help 查看完整使用说明\n") + + # 创建一个演示病例 + demo_image = "data/medical/demo/mri_case.jpg" + demo_patient = { + "age": 45, + "gender": "male", + "symptoms": "下背部疼痛，左腿麻木，持续2周", + } + + diagnose_single_case( + agent=agent, image_path=demo_image, patient_info=demo_patient, verbose=True + ) + + +if __name__ == "__main__": + main() diff --git a/sage-apps/src/sage/apps/medical_diagnosis/scripts/download_lumbar_dataset.py b/sage-apps/src/sage/apps/medical_diagnosis/scripts/download_lumbar_dataset.py new file mode 100755 index 0000000..936e734 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/scripts/download_lumbar_dataset.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""下载腰椎MRI数据集""" + +from pathlib import Path + +from datasets import Dataset, load_dataset +from huggingface_hub import snapshot_download + + +def download_lumbar_spine_dataset(): + """下载 UniDataPro/lumbar-spine-mri 数据集""" + + dataset_name = "UniDataPro/lumbar-spine-mri" + output_dir = Path("data/medical/lumbar-spine-mri") + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"📥 开始下载数据集: {dataset_name}") + print(f"📂 保存路径: {output_dir.absolute()}") + + try: + # 方式1: 使用 datasets 库加载 + print("\n🔄 方式1: 使用 datasets 库加载...") + dataset = load_dataset(dataset_name, split="train") + + # 确保是 Dataset 类型 + if not isinstance(dataset, Dataset): + raise TypeError(f"Expected Dataset, got {type(dataset)}") + + print("✅ 据集加载成功!") + print(f"📊 样本数量: {len(dataset)}") + print(f"📋 字段: {dataset.column_names}") + + # 保存数据集信息 + info_file = output_dir / "dataset_info.txt" + with open(info_file, "w", encoding="utf-8") as f: + f.write(f"Dataset: {dataset_name}\n") + f.write(f"Samples: {len(dataset)}\n") + f.write(f"Columns: {dataset.column_names}\n") + f.write("\nFirst sample:\n") + if len(dataset) > 0: + first_sample = dataset[0] # type: ignore[index] + if isinstance(first_sample, dict): + for key, value in first_sample.items(): + f.write(f" {key}: {type(value).__name__}\n") + + print(f"\n💾 数据集信息已保存到: {info_file}") + + # 保存数据集到本地 + cache_dir = output_dir / "cache" + dataset.save_to_disk(str(cache_dir)) + print(f"💾 数据集已缓存到: {cache_dir}") + + return dataset + + except Exception as e: + print(f"❌ 下载失败: {e}") + print("\n🔄 尝试方式2: 直接下载仓库...") + + try: + snapshot_download( + repo_id=dataset_name, + repo_type="dataset", + local_dir=str(output_dir / "raw"), + local_dir_use_symlinks=False, + ) + print(f"✅ 数据集仓库下载成功: {output_dir / 'raw'}") + except Exception as e2: + print(f"❌ 方式2也失败: {e2}") + raise + + +if __name__ == "__main__": + download_lumbar_spine_dataset() diff --git a/sage-apps/src/sage/apps/medical_diagnosis/scripts/explore_dataset.py b/sage-apps/src/sage/apps/medical_diagnosis/scripts/explore_dataset.py new file mode 100644 index 0000000..4286807 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/scripts/explore_dataset.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +数据集探索脚本 +用于查看腰椎MRI数据集的样本和统计信息 +""" + +import sys +from collections import Counter +from pathlib import Path + +import numpy as np +from datasets import Dataset, load_from_disk +from PIL import Image + +# 设置项目路径 +project_root = Path(__file__).parent.parent.parent.parent +sys.path.insert(0, str(project_root)) + + +def explore_dataset(): + """探索数据集""" + + dataset_path = project_root / "data" / "medical" / "lumbar-spine-mri" / "cache" + + if not dataset_path.exists(): + print(f"❌ 数据集不存在: {dataset_path}") + print("请先运行 scripts/download_lumbar_dataset.py 下载数据集") + return + + print("=" * 80) + print("📊 腰椎MRI数据集探索") + print("=" * 80) + + # 加载数据集 + print(f"\n📂 从 {dataset_path} 加载数据集...") + dataset = load_from_disk(str(dataset_path)) + + # 确保是 Dataset 类型 + if not isinstance(dataset, Dataset): + raise TypeError(f"Expected Dataset, got {type(dataset)}") + + # 基本信息 + print("\n📋 数据集基本信息:") + print(f" - 样本总数: {len(dataset)}") + print(f" - 字段: {dataset.column_names}") + print(f" - Features: {dataset.features}") + + # 标签分布 + labels = [sample["label"] for sample in dataset] # type: ignore[index] + label_counts = Counter(labels) + + print("\n🏷️ 标签分布:") + for label, count in sorted(label_counts.items()): + percentage = count / len(dataset) * 100 + print(f" - Label {label}: {count} samples ({percentage:.1f}%)") + + # 图像统计 + print("\n🖼️ 图像统计 (前10个样本):") + image_sizes = [] + + for i in range(min(10, len(dataset))): + sample = dataset[i] + img = sample["image"] + + if isinstance(img, Image.Image): + width, height = img.size + mode = img.mode + image_sizes.append((width, height)) + + if i < 5: # 只打印前5个 + print(f" - 样本 {i}: {width}x{height}, mode={mode}, label={sample['label']}") + + if image_sizes: + widths = [w for w, h in image_sizes] + heights = [h for w, h in image_sizes] + + print("\n📐 图像尺寸范围:") + print(f" - 宽度: {min(widths)} ~ {max(widths)} (平均: {np.mean(widths):.0f})") + print(f" - 高度: {min(heights)} ~ {max(heights)} (平均: {np.mean(heights):.0f})") + + # 保存一些样本 + output_dir = project_root / "examples" / "medical_diagnosis" / "data" / "samples" + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"\n💾 保存样本图像到: {output_dir}") + + # 每个标签保存一个样本 + saved_labels = set() + saved_count = 0 + + for i, sample in enumerate(dataset): # type: ignore[arg-type] + if not isinstance(sample, dict): + continue + label = sample["label"] + + if label not in saved_labels: + img = sample["image"] + if isinstance(img, Image.Image): + output_path = output_dir / f"sample_label{label}_{i}.jpg" + img.save(output_path) + print(f" ✓ 保存 label={label} 样本: {output_path.name}") + saved_labels.add(label) + saved_count += 1 + + if saved_count >= 5: # 最多保存5个样本 + break + + # 创建简单的统计报告 + report_path = output_dir / "dataset_report.txt" + with open(report_path, "w", encoding="utf-8") as f: + f.write("腰椎MRI数据集统计报告\n") + f.write("=" * 80 + "\n\n") + f.write(f"数据集路径: {dataset_path}\n") + f.write(f"样本总数: {len(dataset)}\n") + f.write(f"字段: {', '.join(dataset.column_names)}\n\n") + + f.write("标签分布:\n") + for label, count in sorted(label_counts.items()): + percentage = count / len(dataset) * 100 + f.write(f" Label {label}: {count} ({percentage:.1f}%)\n") + + if image_sizes: + f.write("\n图像尺寸范围:\n") + f.write(f" 宽度: {min(widths)} ~ {max(widths)} (平均: {np.mean(widths):.0f})\n") + f.write(f" 高度: {min(heights)} ~ {max(heights)} (平均: {np.mean(heights):.0f})\n") + + print(f"\n📊 统计报告已保存: {report_path}") + + print("\n" + "=" * 80) + print("✅ 数据集探索完成!") + print("=" * 80) + + +if __name__ == "__main__": + explore_dataset() diff --git a/sage-apps/src/sage/apps/medical_diagnosis/scripts/prepare_data.py b/sage-apps/src/sage/apps/medical_diagnosis/scripts/prepare_data.py new file mode 100644 index 0000000..75b81cf --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/scripts/prepare_data.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +""" +数据预处理脚本 +将下载的数据集转换为训练/测试集，并生成模拟的诊断报告 +""" + +import json +import random +import sys +from pathlib import Path + +from datasets import load_from_disk +from PIL import Image +from sklearn.model_selection import train_test_split + +# Note: This script assumes sage-apps package is installed +# Install with: cd sage-apps && pip install -e . + + +# 疾病类别映射 (模拟) +DISEASE_MAPPING = { + 0: {"name": "正常", "severity": "无", "description": "未见明显异常"}, + 1: { + "name": "轻度退行性变", + "severity": "轻度", + "description": "L4/L5椎间盘轻度退行性变", + }, + 2: { + "name": "椎间盘突出", + "severity": "中度", + "description": "L4/L5椎间盘向后突出,压迫硬膜囊", + }, + 3: { + "name": "多节段退行性变", + "severity": "中度", + "description": "L3/L4, L4/L5, L5/S1多节段退行性变", + }, + 4: { + "name": "椎管狭窄", + "severity": "重度", + "description": "L4/L5椎管狭窄,硬膜囊受压", + }, + 5: { + "name": "椎间盘脱出", + "severity": "重度", + "description": "L5/S1椎间盘脱出,游离到椎管内", + }, + 6: {"name": "骨质增生", "severity": "轻度", "description": "L3-L5椎体缘骨质增生"}, + 7: {"name": "椎体滑脱", "severity": "中度", "description": "L4椎体前移,I度滑脱"}, + 8: {"name": "韧带钙化", "severity": "轻度", "description": "后纵韧带钙化"}, +} + + +def generate_mock_report(label: int, patient_info: dict) -> str: + """生成模拟诊断报告""" + + disease_info = DISEASE_MAPPING.get(label, DISEASE_MAPPING[0]) + + age = patient_info.get("age", 45) + gender = patient_info.get("gender", "男") + + # 根据疾病生成症状 + if label == 0: + symptoms = "无明显不适" + findings = "腰椎MRI T2加权矢状位: 未见明显异常。各椎间盘信号正常，椎体排列整齐，椎管通畅。" + conclusion = "影像学未见明显异常。" + recommendations = "定期体检，保持良好的生活习惯。" + elif label in [1, 6, 8]: + symptoms = "偶尔腰部酸痛" + findings = ( + f"腰椎MRI T2加权矢状位: {disease_info['description']}。椎管尚通畅，未见明显神经根受压。" + ) + conclusion = f"{disease_info['name']}，程度{disease_info['severity']}。" + recommendations = ( + "适当休息，避免久坐久站。可进行腰背肌锻炼，如游泳、普拉提等。必要时物理治疗。" + ) + elif label in [2, 3]: + symptoms = "腰痛伴右下肢放射痛3周" + findings = f"腰椎MRI T2加权矢状位: {disease_info['description']}。相应节段椎管变窄，神经根可能受压。" + conclusion = f"{disease_info['name']}，程度{disease_info['severity']}。" + recommendations = "建议卧床休息2-3周，牵引治疗。口服非甾体抗炎药及神经营养药物。保守治疗无效时考虑手术治疗。" + elif label == 7: + symptoms = "腰部疼痛，活动受限" + findings = f"腰椎MRI T2加权矢状位: {disease_info['description']}。相应节段不稳定。" + conclusion = f"{disease_info['name']}，程度{disease_info['severity']}。" + recommendations = "避免重体力劳动，佩戴腰围保护。核心肌群训练。症状明显时考虑手术固定。" + else: # 重度 (4, 5) + symptoms = "腰痛伴双下肢麻木、无力2月" + findings = f"腰椎MRI T2加权矢状位: {disease_info['description']}。马尾神经受压。" + conclusion = f"{disease_info['name']}，程度{disease_info['severity']}。" + recommendations = ( + "建议尽早手术治疗(椎间盘摘除术或椎管减压术)，以解除神经压迫。术后康复训练。" + ) + + report = f"""患者信息: + 年龄: {age}岁 + 性别: {gender} + 主诉: {symptoms} + +影像描述: + {findings} + +主要发现: + - 病变节段: {"多节段" if label == 3 else "L4/L5或L5/S1"} + - 病变类型: {disease_info["name"]} + - 严重程度: {disease_info["severity"]} + +诊断结论: + {conclusion} + +治疗建议: + {recommendations} + +注: 本报告仅供参考，请结合临床症状和其他检查结果综合判断。 +""" + + return report + + +def prepare_dataset(): + """准备数据集""" + + dataset_path = project_root / "data" / "medical" / "lumbar-spine-mri" / "cache" + output_dir = project_root / "examples" / "medical_diagnosis" / "data" / "processed" + output_dir.mkdir(parents=True, exist_ok=True) + + print("=" * 80) + print("📊 数据预处理") + print("=" * 80) + + # 加载数据集 + print("\n📂 加载数据集...") + dataset = load_from_disk(str(dataset_path)) + print(f" ✓ 已加载 {len(dataset)} 个样本") + + # 准备数据 + print("\n🔄 准备数据...") + + samples = [] + for i, sample in enumerate(dataset): # type: ignore[arg-type] + if not isinstance(sample, dict): + continue + label = sample["label"] + image = sample["image"] + + # 生成患者信息 + age = random.randint(25, 75) + gender = random.choice(["男", "女"]) + + patient_info = { + "age": age, + "gender": gender, + "patient_id": f"P{i + 1:04d}", + } + + # 生成诊断报告 + report = generate_mock_report(label, patient_info) + + # 保存图像 + image_filename = f"case_{i + 1:04d}_label{label}.jpg" + image_path = output_dir / "images" / image_filename + image_path.parent.mkdir(exist_ok=True) + + if isinstance(image, Image.Image): + # 统一resize到512x512 + image_resized = image.resize((512, 512), Image.Resampling.LANCZOS) + image_resized.save(image_path, quality=95) + + # 保存报告 + report_filename = f"case_{i + 1:04d}_report.txt" + report_path = output_dir / "reports" / report_filename + report_path.parent.mkdir(exist_ok=True) + + with open(report_path, "w", encoding="utf-8") as f: + f.write(report) + + # 记录样本信息 + disease_info = DISEASE_MAPPING.get(label, DISEASE_MAPPING[0]) + samples.append( + { + "case_id": f"case_{i + 1:04d}", + "patient_id": patient_info["patient_id"], + "age": age, + "gender": gender, + "label": label, + "disease": disease_info["name"], + "severity": disease_info["severity"], + "image_path": str(image_path.relative_to(output_dir)), + "report_path": str(report_path.relative_to(output_dir)), + } + ) + + if (i + 1) % 20 == 0: + print(f" 处理进度: {i + 1}/{len(dataset)}") + + print(f" ✓ 已处理 {len(samples)} 个病例") + + # 划分训练/测试集 (80/20) + print("\n✂️ 划分训练/测试集...") + + train_samples, test_samples = train_test_split( + samples, test_size=0.2, random_state=42, stratify=[s["label"] for s in samples] + ) + + print(f" ✓ 训练集: {len(train_samples)} 样本") + print(f" ✓ 测试集: {len(test_samples)} 样本") + + # 保存索引文件 + print("\n💾 保存索引文件...") + + # 保存JSON格式 + with open(output_dir / "train_index.json", "w", encoding="utf-8") as f: + json.dump(train_samples, f, ensure_ascii=False, indent=2) + + with open(output_dir / "test_index.json", "w", encoding="utf-8") as f: + json.dump(test_samples, f, ensure_ascii=False, indent=2) + + with open(output_dir / "all_cases.json", "w", encoding="utf-8") as f: + json.dump(samples, f, ensure_ascii=False, indent=2) + + print(" ✓ train_index.json") + print(" ✓ test_index.json") + print(" ✓ all_cases.json") + + # 生成统计报告 + print("\n📊 生成统计报告...") + + stats = { + "total_samples": len(samples), + "train_samples": len(train_samples), + "test_samples": len(test_samples), + "label_distribution": {}, + "disease_distribution": {}, + "severity_distribution": {}, + "age_range": [min(s["age"] for s in samples), max(s["age"] for s in samples)], + "gender_distribution": { + "男": sum(1 for s in samples if s["gender"] == "男"), + "女": sum(1 for s in samples if s["gender"] == "女"), + }, + } + + for sample in samples: + label = sample["label"] + disease = sample["disease"] + severity = sample["severity"] + + stats["label_distribution"][label] = stats["label_distribution"].get(label, 0) + 1 + stats["disease_distribution"][disease] = stats["disease_distribution"].get(disease, 0) + 1 + stats["severity_distribution"][severity] = ( + stats["severity_distribution"].get(severity, 0) + 1 + ) + + with open(output_dir / "stats.json", "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + + # 生成可读的统计报告 + with open(output_dir / "README.txt", "w", encoding="utf-8") as f: + f.write("腰椎MRI数据集 - 处理后\n") + f.write("=" * 80 + "\n\n") + f.write(f"总样本数: {stats['total_samples']}\n") + f.write(f"训练集: {stats['train_samples']}\n") + f.write(f"测试集: {stats['test_samples']}\n\n") + + f.write("疾病分布:\n") + for disease, count in sorted(stats["disease_distribution"].items(), key=lambda x: -x[1]): + percentage = count / stats["total_samples"] * 100 + f.write(f" - {disease}: {count} ({percentage:.1f}%)\n") + + f.write("\n严重程度分布:\n") + for severity, count in stats["severity_distribution"].items(): + percentage = count / stats["total_samples"] * 100 + f.write(f" - {severity}: {count} ({percentage:.1f}%)\n") + + f.write(f"\n年龄范围: {stats['age_range'][0]} - {stats['age_range'][1]} 岁\n") + f.write("\n性别分布:\n") + for gender, count in stats["gender_distribution"].items(): + percentage = count / stats["total_samples"] * 100 + f.write(f" - {gender}: {count} ({percentage:.1f}%)\n") + + f.write("\n目录结构:\n") + f.write(" - images/: MRI影像文件 (512x512 JPG)\n") + f.write(" - reports/: 诊断报告文件 (TXT)\n") + f.write(" - train_index.json: 训练集索引\n") + f.write(" - test_index.json: 测试集索引\n") + f.write(" - all_cases.json: 所有病例索引\n") + f.write(" - stats.json: 统计信息 (JSON)\n") + f.write(" - README.txt: 本文件\n") + + print(" ✓ stats.json") + print(" ✓ README.txt") + + print("\n" + "=" * 80) + print("✅ 数据预处理完成!") + print(f"📁 输出目录: {output_dir}") + print("=" * 80) + + # 显示一些统计信息 + print("\n📊 数据集统计:") + print(f" - 总样本: {stats['total_samples']}") + print(f" - 训练集: {stats['train_samples']}") + print(f" - 测试集: {stats['test_samples']}") + print(f"\n🏥 疾病类型: {len(stats['disease_distribution'])} 种") + for disease, count in sorted(stats["disease_distribution"].items(), key=lambda x: -x[1])[:5]: + print(f" - {disease}: {count}") + + +if __name__ == "__main__": + prepare_dataset() diff --git a/sage-apps/src/sage/apps/medical_diagnosis/setup_data.sh b/sage-apps/src/sage/apps/medical_diagnosis/setup_data.sh new file mode 100755 index 0000000..3092690 --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/setup_data.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# 医疗诊断数据集自动下载和准备脚本 + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DATA_DIR="$SCRIPT_DIR/data/processed" + +echo "==========================================" +echo "医疗诊断数据集自动设置" +echo "==========================================" + +# 检查是否已经存在处理好的数据 +if [ -d "$DATA_DIR" ] && [ -f "$DATA_DIR/train_index.json" ]; then + echo "✓ 数据集已存在: $DATA_DIR" + echo "" + + # 显示数据集统计 + if [ -f "$DATA_DIR/stats.json" ]; then + echo "📊 数据集统计:" + python3 -c " +import json +with open('$DATA_DIR/stats.json', 'r') as f: + stats = json.load(f) + print(f\" 总样本数: {stats['total_samples']}\") + print(f\" 训练集: {stats['train_samples']}\") + print(f\" 测试集: {stats['test_samples']}\") + print(f\" 疾病类型: {len(stats['disease_distribution'])}\") +" + fi + + echo "" + echo "如需重新下载，请删除 data/ 目录后重新运行此脚本" + exit 0 +fi + +echo "📥 开始下载和准备数据集..." +echo "" + +# 1. 安装必要的依赖 +echo "1️⃣ 检查Python依赖..." +pip install -q huggingface_hub datasets pillow scikit-learn || { + echo "❌ 依赖安装失败" + exit 1 +} +echo " ✓ 依赖已安装" + +# 2. 下载数据集 +echo "" +echo "2️⃣ 下载腰椎MRI数据集..." +python3 "$PROJECT_ROOT/scripts/download_lumbar_dataset.py" || { + echo "❌ 数据集下载失败" + exit 1 +} +echo " ✓ 数据集下载完成" + +# 3. 预处理数据 +echo "" +echo "3️⃣ 预处理数据集..." +python3 "$SCRIPT_DIR/scripts/prepare_data.py" || { + echo "❌ 数据预处理失败" + exit 1 +} +echo " ✓ 数据预处理完成" + +# 4. 验证数据 +echo "" +echo "4️⃣ 验证数据集..." +if [ -f "$DATA_DIR/train_index.json" ] && [ -f "$DATA_DIR/test_index.json" ]; then + echo " ✓ 数据集验证成功" + + # 显示统计信息 + python3 -c " +import json +with open('$DATA_DIR/stats.json', 'r') as f: + stats = json.load(f) + print('') + print('📊 数据集准备完成!') + print('=' * 50) + print(f\"总样本数: {stats['total_samples']}\") + print(f\"训练集: {stats['train_samples']}\") + print(f\"测试集: {stats['test_samples']}\") + print(f\"疾病类型: {len(stats['disease_distribution'])}\") + print('=' * 50) +" +else + echo " ❌ 数据集验证失败" + exit 1 +fi + +echo "" +echo "✅ 数据集设置完成！" +echo "" +echo "可以开始使用:" +echo " python examples/medical_diagnosis/test_diagnosis.py --mode single" +echo " python examples/medical_diagnosis/run_diagnosis.py --interactive" +echo "" diff --git a/sage-apps/src/sage/apps/medical_diagnosis/tools/__init__.py b/sage-apps/src/sage/apps/medical_diagnosis/tools/__init__.py new file mode 100644 index 0000000..3efb2aa --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/tools/__init__.py @@ -0,0 +1,5 @@ +"""工具包""" + +from .knowledge_base import MedicalKnowledgeBase + +__all__ = ["MedicalKnowledgeBase"] diff --git a/sage-apps/src/sage/apps/medical_diagnosis/tools/knowledge_base.py b/sage-apps/src/sage/apps/medical_diagnosis/tools/knowledge_base.py new file mode 100644 index 0000000..67f301a --- /dev/null +++ b/sage-apps/src/sage/apps/medical_diagnosis/tools/knowledge_base.py @@ -0,0 +1,643 @@ +""" +医学知识库管理 +负责医学知识和病例的检索 +""" + +import json +from pathlib import Path +from typing import Any + +from sage.common.components.sage_embedding.service import EmbeddingService + +# SageVDB has been migrated to independent PyPI package +try: + from sagevdb import SageVDB + + SAGEDB_AVAILABLE = True +except ImportError: + SAGEDB_AVAILABLE = False + SageVDB = None # type: ignore + + +class MedicalKnowledgeBase: + """ + 医学知识库 + + 功能: + 1. 存储和检索医学知识 + 2. 存储和检索病例数据 + 3. 多模态检索（文本+影像） + + 配置选项 (config): + data_path: str - 处理后数据集的路径 + enable_dataset_knowledge: bool - 是否从数据集加载知识 (默认True) + enable_report_knowledge: bool - 是否从医学报告加载知识 (默认True) + enable_case_database: bool - 是否加载病例数据库 (默认True) + max_reports: int - 最大读取报告数量 (默认50) + verbose: bool - 是否输出详细日志 (默认True) + """ + + def __init__(self, config: dict): + """ + 初始化知识库 + + Args: + config: 配置字典，支持以下选项： + - data_path: 数据路径 + - enable_dataset_knowledge: 启用数据集知识加载 + - enable_report_knowledge: 启用报告知识加载 + - enable_case_database: 启用病例数据库加载 + - max_reports: 最大报告读取数量 + - verbose: 详细输出 + """ + self.config = config + self.embedding_service = None + self.vector_db = None + + # 配置选项 + self._verbose = config.get("verbose", True) + self._enable_dataset = config.get("enable_dataset_knowledge", True) + self._enable_reports = config.get("enable_report_knowledge", True) + self._enable_cases = config.get("enable_case_database", True) + self._max_reports = config.get("max_reports", 50) + + self._setup_services() + self._load_knowledge() + + def _log(self, message: str): + """输出日志信息（如果启用详细模式）""" + if self._verbose: + print(message) + + def _setup_services(self): + """设置服务""" + self._log(" Setting up knowledge base services...") + + # 获取embedding配置 + embedding_config = self.config.get("services", {}).get("embedding", {}) + models_config = self.config.get("models", {}) + + # 初始化 EmbeddingService + embedding_service_config = { + "method": embedding_config.get("method", "hf"), + "model": models_config.get("embedding_model", "BAAI/bge-large-zh-v1.5"), + "batch_size": embedding_config.get("batch_size", 32), + "normalize": embedding_config.get("normalize", True), + "cache_enabled": embedding_config.get("cache_enabled", False), + } + + self.embedding_service = EmbeddingService(embedding_service_config) + self.embedding_service.setup() + + # 初始化 SageVDB + # 获取embedding维度 + dimension = self.embedding_service.get_dimension() + # index_type = vector_db_config.get("index_type", "AUTO") + + # TODO: Update to use SageVDB directly after service migration + # self.vector_db = SageVDB(dimension=dimension) + self.vector_db = None # Temporarily disabled pending service migration + + print( + f" ✓ EmbeddingService initialized (dim={dimension}, method={embedding_service_config['method']})" + ) + # print(f" ✓ SageVDB initialized (dim={dimension})") + + def _load_knowledge(self): + """加载医学知识""" + self._log(" Loading medical knowledge...") + + # 从默认知识开始 + self.knowledge_base = self._get_default_knowledge() + self.case_database = [] + + # 从数据集加载知识 + if self._enable_dataset: + dataset_knowledge = self._load_knowledge_from_dataset() + if dataset_knowledge: + self.knowledge_base.extend(dataset_knowledge) + self._log(f" ✓ Loaded {len(dataset_knowledge)} knowledge entries from dataset") + + # 从医学文献/报告加载知识 + if self._enable_reports: + literature_knowledge = self._load_knowledge_from_reports() + if literature_knowledge: + self.knowledge_base.extend(literature_knowledge) + self._log(f" ✓ Loaded {len(literature_knowledge)} knowledge entries from reports") + + # 加载病例数据库 + if self._enable_cases: + cases = self._load_case_database() + if cases: + self.case_database = cases + self._log(f" ✓ Loaded {len(cases)} cases from database") + + self._log(f" Total knowledge base size: {len(self.knowledge_base)} entries") + + def _load_knowledge_from_dataset(self) -> list[dict[str, Any]]: + """从处理好的数据集加载医学知识""" + knowledge = [] + + # 获取数据集路径配置 + data_path = self.config.get("data_path") + if not data_path: + # 尝试默认路径 + current_file = Path(__file__) + default_path = current_file.parent.parent / "data" / "processed" + if default_path.exists(): + data_path = str(default_path) + else: + return knowledge + + data_dir = Path(data_path) + if not data_dir.exists(): + return knowledge + + # 加载统计信息，从中提取疾病知识 + stats_file = data_dir / "stats.json" + if stats_file.exists(): + try: + with open(stats_file, encoding="utf-8") as f: + stats = json.load(f) + + # 从疾病分布中提取知识 + disease_dist = stats.get("disease_distribution", {}) + for disease, count in disease_dist.items(): + if disease and disease != "正常": + knowledge.append( + { + "topic": disease, + "content": f"{disease}是常见的腰椎疾病，在数据集中有{count}个相关病例。", + "source": "dataset_statistics", + "case_count": count, + } + ) + except Exception as e: + print(f" Warning: Failed to load stats.json: {e}") + + return knowledge + + def _load_knowledge_from_reports(self) -> list[dict[str, Any]]: + """从医学报告中提取知识""" + knowledge = [] + + # 获取数据集路径配置 + data_path = self.config.get("data_path") + if not data_path: + # 尝试默认路径 + current_file = Path(__file__) + default_path = current_file.parent.parent / "data" / "processed" + if default_path.exists(): + data_path = str(default_path) + else: + return knowledge + + data_dir = Path(data_path) + reports_dir = data_dir / "reports" + + if not reports_dir.exists(): + return knowledge + + # 读取所有报告并提取知识 + report_files = list(reports_dir.glob("*.txt")) + + # 用于存储提取的独特知识点 + disease_knowledge = {} + + for report_file in report_files[: self._max_reports]: # 限制读取数量以避免过载 + try: + with open(report_file, encoding="utf-8") as f: + report_content = f.read() + + # 从报告中提取诊断结论和治疗建议 + lines = report_content.split("\n") + diagnosis = None + treatment = None + findings = None + + for i, line in enumerate(lines): + if "诊断结论:" in line and i + 1 < len(lines): + diagnosis = lines[i + 1].strip() + elif "治疗建议:" in line and i + 1 < len(lines): + treatment = lines[i + 1].strip() + elif "主要发现:" in line: + # 提取接下来几行的发现 + findings_lines = [] + for j in range(i + 1, min(i + 5, len(lines))): + if lines[j].strip().startswith("-"): + findings_lines.append(lines[j].strip()) + findings = "\n".join(findings_lines) if findings_lines else None + + # 如果提取到有效信息，添加到知识库 + if diagnosis and diagnosis not in disease_knowledge: + # 从诊断中提取疾病名称 + disease_name = diagnosis.split("，")[0] if "，" in diagnosis else diagnosis + + if disease_name and len(disease_name) > 2: + disease_knowledge[disease_name] = { + "topic": disease_name, + "content": diagnosis, + "treatment": treatment if treatment else "请咨询专业医生", + "findings": findings if findings else "影像学检查显示相应变化", + "source": "medical_reports", + } + + except Exception: + # 忽略单个文件的错误，继续处理其他文件 + continue + + # 将提取的知识转换为列表 + knowledge = list(disease_knowledge.values()) + + return knowledge + + def _load_case_database(self) -> list[dict[str, Any]]: + """从数据集加载病例数据库""" + cases = [] + + # 获取数据集路径配置 + data_path = self.config.get("data_path") + if not data_path: + # 尝试默认路径 + current_file = Path(__file__) + default_path = current_file.parent.parent / "data" / "processed" + if default_path.exists(): + data_path = str(default_path) + else: + return cases + + data_dir = Path(data_path) + + # 尝试加载所有病例索引 + all_cases_file = data_dir / "all_cases.json" + if all_cases_file.exists(): + try: + with open(all_cases_file, encoding="utf-8") as f: + cases_data = json.load(f) + + # 转换为统一格式 + for case in cases_data: + cases.append( + { + "case_id": case.get("case_id", ""), + "age": case.get("age", 0), + "gender": case.get("gender", "unknown"), + "diagnosis": case.get("disease", ""), + "severity": case.get("severity", ""), + "image_path": case.get("image_path", ""), + "report_path": case.get("report_path", ""), + } + ) + + except Exception as e: + print(f" Warning: Failed to load case database: {e}") + + return cases + + def _get_default_knowledge(self) -> list[dict[str, str]]: + """获取默认医学知识""" + return [ + { + "topic": "腰椎间盘突出症", + "content": """ + 腰椎间盘突出症是指椎间盘的纤维环破裂，髓核组织从破裂处突出于后方或椎管内， + 导致相邻脊神经根遭受刺激或压迫，从而产生腰部疼痛、下肢麻木疼痛等一系列临床症状。 + 好发部位：L4/L5、L5/S1。 + """, + "diagnosis_criteria": "MRI T2加权像显示椎间盘后突，硬膜囊受压变形", + "treatment": "保守治疗包括卧床休息、物理治疗、药物治疗；严重者可考虑手术", + }, + { + "topic": "腰椎退行性变", + "content": """ + 腰椎退行性变是指随年龄增长，腰椎间盘、椎体及小关节发生的退行性改变。 + 主要表现为椎间盘高度降低、信号减低、骨质增生等。 + """, + "diagnosis_criteria": "MRI显示椎间盘信号减低、高度降低、椎体边缘骨赘形成", + "treatment": "以保守治疗为主，加强腰背肌锻炼，避免久坐久站", + }, + { + "topic": "椎管狭窄", + "content": """ + 腰椎管狭窄症是指因各种原因导致椎管容积减小，压迫硬膜囊、马尾神经或神经根， + 引起相应神经功能障碍的一组综合征。 + """, + "diagnosis_criteria": "MRI显示椎管矢状径<12mm，硬膜囊明显受压", + "treatment": "轻度可保守治疗，重度需手术减压", + }, + ] + + def retrieve_similar_cases( + self, query: str, image_features: dict[str, Any], top_k: int = 5 + ) -> list[dict[str, Any]]: + """ + 检索相似病例 + + Args: + query: 查询文本 + image_features: 影像特征 + top_k: 返回数量 + + Returns: + 相似病例列表 + """ + # 如果向量数据库为空，使用关键词匹配或模拟病例 + if self.vector_db.stats()["size"] == 0: + # 如果有加载的病例数据库，使用它；否则返回模拟病例 + if self.case_database: + # 简单的关键词匹配（实际应该使用向量检索） + matched_cases = [] + query_keywords = query.lower().split() + + for case in self.case_database: + # 计算匹配分数 + score = 0.0 + diagnosis = case.get("diagnosis", "").lower() + + for keyword in query_keywords: + if keyword in diagnosis: + score += 1.0 + + if score > 0: + matched_cases.append( + { + "case_id": case.get("case_id", ""), + "age": case.get("age", 0), + "gender": case.get("gender", "unknown"), + "diagnosis": case.get("diagnosis", ""), + "severity": case.get("severity", ""), + "similarity_score": min(score / len(query_keywords), 1.0), + } + ) + + # 按相似度排序 + matched_cases.sort(key=lambda x: x["similarity_score"], reverse=True) + + # 如果找到匹配的病例，返回它们 + if matched_cases: + return matched_cases[:top_k] + + # 没有加载的数据或没有匹配，返回模拟病例 + return self._get_mock_cases()[:top_k] + + # 使用 EmbeddingService 生成查询向量 + result = self.embedding_service.embed(query) + query_vector = result["vectors"][0] + + # 使用 SageDB 进行向量检索 + search_results = self.vector_db.search(query_vector, k=top_k, include_metadata=True) + + # 转换为病例格式 + cases = [] + for res in search_results: + metadata = res.get("metadata", {}) + cases.append( + { + "case_id": metadata.get("case_id", f"CASE_{res['id']:03d}"), + "age": int(metadata.get("age", 0)) if metadata.get("age") else 0, + "gender": metadata.get("gender", "unknown"), + "diagnosis": metadata.get("diagnosis", ""), + "symptoms": metadata.get("symptoms", ""), + "treatment": metadata.get("treatment", ""), + "outcome": metadata.get("outcome", ""), + "similarity_score": float(res["score"]), + } + ) + + return cases + + def retrieve_knowledge(self, query: str, top_k: int = 3) -> list[dict[str, Any]]: + """ + 检索医学知识 + + Args: + query: 查询文本 + top_k: 返回数量 + + Returns: + 医学知识列表 + """ + # 使用简单的关键词匹配（可以后续改进为向量检索） + results = [] + + for knowledge in self.knowledge_base: + topic = knowledge.get("topic", "") + content = knowledge.get("content", "") + + # Check if query contains topic or topic contains query (for Chinese text) + # Also check if query matches content + if ( + topic in query + or query in topic + or any(char in topic for char in query if len(char.strip()) > 0) + or topic in content + or query in content + ): + results.append(knowledge) + + # 如果没有匹配结果，使用向量相似度检索 + if not results and self.embedding_service: + # 为知识库条目生成embedding并检索 + knowledge_texts = [k["topic"] + " " + k["content"] for k in self.knowledge_base] + knowledge_embeddings = self.embedding_service.embed(knowledge_texts) + + # 为查询生成embedding + query_embedding = self.embedding_service.embed(query) + query_vec = query_embedding["vectors"][0] + + # 计算相似度并排序 + import numpy as np + + similarities = [] + for i, emb_vec in enumerate(knowledge_embeddings["vectors"]): + similarity = float(np.dot(query_vec, emb_vec)) + similarities.append((i, similarity)) + + # 按相似度排序 + similarities.sort(key=lambda x: x[1], reverse=True) + + # 返回top_k个最相似的知识 + for idx, _ in similarities[:top_k]: + results.append(self.knowledge_base[idx]) + + return results[:top_k] + + def _get_mock_cases(self) -> list[dict[str, Any]]: + """获取模拟病例""" + return [ + { + "case_id": "CASE_001", + "age": 48, + "gender": "male", + "diagnosis": "L4/L5椎间盘突出症", + "symptoms": "下背部疼痛，左腿麻木", + "treatment": "保守治疗3个月后症状缓解", + "outcome": "良好", + "similarity_score": 0.92, + }, + { + "case_id": "CASE_002", + "age": 52, + "gender": "male", + "diagnosis": "L5/S1椎间盘突出症伴椎管狭窄", + "symptoms": "腰痛伴双下肢麻木", + "treatment": "手术治疗（椎间盘切除+椎管减压）", + "outcome": "症状明显改善", + "similarity_score": 0.88, + }, + { + "case_id": "CASE_003", + "age": 43, + "gender": "female", + "diagnosis": "L4/L5椎间盘突出症", + "symptoms": "右下肢放射痛", + "treatment": "物理治疗+药物治疗", + "outcome": "部分缓解", + "similarity_score": 0.85, + }, + { + "case_id": "CASE_004", + "age": 60, + "gender": "male", + "diagnosis": "腰椎退行性变，L3/L4、L4/L5椎间盘突出", + "symptoms": "慢性腰痛", + "treatment": "长期康复训练", + "outcome": "稳定", + "similarity_score": 0.82, + }, + { + "case_id": "CASE_005", + "age": 38, + "gender": "female", + "diagnosis": "L5/S1椎间盘突出症", + "symptoms": "急性腰痛，左腿麻木", + "treatment": "卧床休息+止痛药", + "outcome": "2周后好转", + "similarity_score": 0.80, + }, + ] + + def add_case(self, case_data: dict[str, Any]): + """添加新病例到知识库""" + # 构建病例文本描述用于embedding + case_text = f"{case_data.get('diagnosis', '')} {case_data.get('symptoms', '')}" + + # 生成文本embedding + result = self.embedding_service.embed(case_text) + case_vector = result["vectors"][0] + + # 准备metadata + metadata = { + "case_id": str(case_data.get("case_id", "")), + "age": str(case_data.get("age", "")), + "gender": str(case_data.get("gender", "")), + "diagnosis": str(case_data.get("diagnosis", "")), + "symptoms": str(case_data.get("symptoms", "")), + "treatment": str(case_data.get("treatment", "")), + "outcome": str(case_data.get("outcome", "")), + } + + # 存入向量数据库 + self.vector_db.add(case_vector, metadata) + + # 同时添加到本地缓存 + self.case_database.append(case_data) + + def update_knowledge(self, knowledge_data: dict[str, Any]) -> dict[str, str]: + """ + 更新医学知识 + + Args: + knowledge_data: 知识数据字典，必须包含 'topic' 字段 + + Returns: + 操作结果字典，包含 'action' (added/updated) 和 'topic' 字段 + + Raises: + ValueError: 如果 knowledge_data 缺少必需的 'topic' 字段 + """ + # 验证必需字段 + if not knowledge_data or "topic" not in knowledge_data: + raise ValueError("knowledge_data must contain 'topic' field") + + topic = knowledge_data["topic"] + + # 检查是否已存在相同主题的知识 + existing_index = None + for i, knowledge in enumerate(self.knowledge_base): + if knowledge.get("topic") == topic: + existing_index = i + break + + if existing_index is not None: + # 更新现有知识 + self.knowledge_base[existing_index] = knowledge_data + return {"action": "updated", "topic": topic} + else: + # 添加新知识 + self.knowledge_base.append(knowledge_data) + return {"action": "added", "topic": topic} + + def cleanup(self): + """清理资源""" + if self.embedding_service and hasattr(self.embedding_service, "cleanup"): + self.embedding_service.cleanup() + + def __del__(self): + """析构时清理资源""" + self.cleanup() + + +if __name__ == "__main__": + # 测试 + config = {"services": {"vector_db": {"collection_name": "lumbar_cases", "top_k": 5}}} + + kb = MedicalKnowledgeBase(config) + + print("=" * 80) + print("测试知识库功能") + print("=" * 80) + + # 测试检索 + print(f"\n1. 初始知识库包含 {len(kb.knowledge_base)} 个知识条目") + print(f" 主题: {[k['topic'] for k in kb.knowledge_base]}") + + # 测试添加新知识 + print("\n2. 测试添加新知识") + new_knowledge = { + "topic": "椎体压缩性骨折", + "content": "椎体压缩性骨折是指椎体在外力作用下发生压缩性变形", + "diagnosis_criteria": "X线或CT显示椎体高度降低超过20%", + "treatment": "急性期卧床休息，必要时行椎体成形术", + } + result = kb.update_knowledge(new_knowledge) + print(f" 结果: {result['action']} - {result['topic']}") + print(f" 知识库现有 {len(kb.knowledge_base)} 个条目") + + # 测试更新现有知识 + print("\n3. 测试更新现有知识") + updated_knowledge = { + "topic": "腰椎间盘突出症", + "content": "更新后的内容：腰椎间盘突出症是最常见的腰椎疾病之一", + "diagnosis_criteria": "更新后的诊断标准", + "treatment": "更新后的治疗方案", + "additional_info": "新增字段：这是补充信息", + } + result = kb.update_knowledge(updated_knowledge) + print(f" 结果: {result['action']} - {result['topic']}") + print(f" 知识库仍有 {len(kb.knowledge_base)} 个条目 (未增加)") + + # 验证更新是否成功 + disc_knowledge = [k for k in kb.knowledge_base if k["topic"] == "腰椎间盘突出症"] + print(f" '腰椎间盘突出症' 条目数: {len(disc_knowledge)}") + if disc_knowledge: + print(f" 更新后的内容预览: {disc_knowledge[0]['content'][:50]}...") + + # 测试相似病例检索 + print("\n4. 测试相似病例检索") + cases = kb.retrieve_similar_cases(query="腰痛伴下肢麻木", image_features={}, top_k=3) + print(f" 检索到 {len(cases)} 个相似病例:") + for case in cases: + print(f" - {case['case_id']}: {case['diagnosis']} (相似度: {case['similarity_score']})") + + print("\n" + "=" * 80) diff --git a/sage-apps/src/sage/apps/py.typed b/sage-apps/src/sage/apps/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/sage-apps/src/sage/apps/smart_home/README.md b/sage-apps/src/sage/apps/smart_home/README.md new file mode 100644 index 0000000..35a5b60 --- /dev/null +++ b/sage-apps/src/sage/apps/smart_home/README.md @@ -0,0 +1,235 @@ +# Smart Home System + +A distributed IoT-based smart home automation system built on SAGE framework demonstrating +interconnectivity and coordination capabilities across multiple devices. + +## Features + +- **IoT Device Network**: Simulates various smart home devices (robots, sensors, appliances) +- **Automated Workflows**: Coordinates complex multi-device workflows using SAGE operators +- **Environment Monitoring**: Real-time sensor data collection and response +- **Event-Driven Architecture**: Devices communicate through SAGE stream processing +- **Device Coordination**: Manages dependencies and sequencing with MapFunction chains + +## Quick Start + +```python +from sage.apps.smart_home import run_smart_home_demo + +# Run single laundry automation cycle +run_smart_home_demo() + +# Run multiple cycles +run_smart_home_demo(num_cycles=3) + +# Verbose mode with event logging +run_smart_home_demo(num_cycles=2, verbose=True) +``` + +## Command Line Usage + +```bash +# Run default (1 cycle) +python -m sage.apps.smart_home.pipeline + +# Run multiple cycles +python -m sage.apps.smart_home.pipeline --cycles 3 + +# Verbose mode +python -m sage.apps.smart_home.pipeline --verbose +``` + +## Pipeline Architecture + +The system uses SAGE stream processing operators: + +``` +LaundryWorkflowSource (BatchFunction) + ↓ +DeviceExecutor (MapFunction) + ↓ +EnvironmentMonitor (MapFunction) + ↓ +WorkflowProgressSink (SinkFunction) + ↓ +EventLogSink (SinkFunction) [optional] +``` + +### Operators + +- **LaundryWorkflowSource**: Generates workflow tasks (check env → collect → wash → transfer → dry → + store) +- **DeviceExecutor**: Executes tasks on simulated IoT devices (robot, washer, dryer, sensors) +- **EnvironmentMonitor**: Monitors and validates environmental conditions +- **WorkflowProgressSink**: Tracks and reports workflow progress +- **EventLogSink**: Logs all device events for auditing + +## Devices + +### Robot (🤖) + +- **Purpose**: Move items between locations +- **Commands**: `move_laundry` +- **Events**: `task_completed` + +### Washer (🧺) + +- **Purpose**: Wash laundry +- **Commands**: `start_wash` +- **Events**: `wash_completed` + +### Dryer (💨) + +- **Purpose**: Dry laundry +- **Commands**: `start_dry` +- **Events**: `dry_completed` + +### Humidity Sensor (📊) + +- **Purpose**: Monitor humidity levels +- **Commands**: `read` +- **Events**: `reading` +- **Range**: 30-80% + +### Motion Sensor (👁️) + +- **Purpose**: Detect motion +- **Commands**: `detect` +- **Events**: `motion_detected`, `no_motion` + +## Automated Workflows + +### Laundry Automation + +Complete end-to-end laundry automation: + +1. **Environment Check**: Monitor humidity and conditions +1. **Collection**: Robot picks up laundry from basket +1. **Washing**: Washer runs complete wash cycle +1. **Transfer**: Robot moves laundry to dryer +1. **Drying**: Dryer runs complete dry cycle +1. **Storage**: Robot moves laundry to drying rack + +```python +home = SmartHomeSystem() +home.laundry_automation_workflow() +``` + +### Custom Workflows + +Create your own automation: + +```python +# Send custom commands +home.send_command("robot_001", "move_laundry", { + "from": "basket", + "to": "washer" +}) + +home.send_command("washer_001", "start_wash") +``` + +## Architecture + +### Event-Driven Communication + +Devices communicate through events: + +``` +Sensor Reading → Event → Automation Rule → Device Command → Action +``` + +### Device Coordination + +The system coordinates multiple devices: + +``` +Robot → Washer → Robot → Dryer → Robot + | | | | | + v v v v v +Event → Event → Event → Event → Event +``` + +## Example Output + +``` +====================================================================== +Smart Home Laundry Automation Workflow +====================================================================== + +[Step 1] Checking environmental conditions... + 📊 Humidity: 45.3% + +[Step 2] Robot collecting laundry... + 🤖 Robot robot_001: Moving laundry from basket to washer + +[Step 3] Washing laundry... + 🧺 Washer washer_001: Starting wash cycle + ✓ Washer washer_001: Wash cycle completed + +[Step 4] Robot moving laundry to dryer... + 🤖 Robot robot_001: Moving laundry from washer to dryer + +[Step 5] Drying laundry... + 💨 Dryer dryer_001: Starting dry cycle + ✓ Dryer dryer_001: Dry cycle completed + +[Step 6] Robot moving laundry to drying rack... + 🤖 Robot robot_001: Moving laundry from dryer to drying_rack + +====================================================================== +✓ Laundry automation workflow completed successfully! +====================================================================== +``` + +## Use Cases + +- **Home Automation**: Automated household tasks +- **IoT Coordination**: Multi-device workflows +- **Smart Building**: Building management systems +- **Industrial IoT**: Factory automation +- **Energy Management**: Optimize resource usage + +## Testing + +Run the example script: + +```bash +python examples/apps/run_smart_home.py +``` + +Run with specific workflow: + +```bash +python examples/apps/run_smart_home.py --workflow laundry +python examples/apps/run_smart_home.py --workflow monitoring +``` + +## Configuration + +Devices can be configured with custom parameters: + +```python +# Custom cycle times +washer = Washer("washer_001") +washer.cycle_time = 5 # seconds + +# Custom thresholds +humidity_sensor = HumiditySensor("humid_001") +``` + +## Dependencies + +- Python 3.8+ +- Standard library only +- Optional: SAGE framework for distributed processing + +## Future Enhancements + +- Real IoT device integration (MQTT, CoAP) +- Machine learning for predictive automation +- Voice control integration +- Mobile app interface +- Energy optimization algorithms +- Multi-home coordination +- Cloud integration for remote control diff --git a/sage-apps/src/sage/apps/smart_home/__init__.py b/sage-apps/src/sage/apps/smart_home/__init__.py new file mode 100644 index 0000000..669209b --- /dev/null +++ b/sage-apps/src/sage/apps/smart_home/__init__.py @@ -0,0 +1,20 @@ +""" +Smart Home System + +A distributed IoT-based smart home automation system demonstrating SAGE's +interconnectivity capabilities. Manages coordinated workflows between devices +like robots, sensors, washers, and dryers. + +Built on SAGE framework using stream processing operators. + +Features: +- IoT device network simulation +- Automated workflows (e.g., laundry automation) +- Environment monitoring and response +- Device coordination and communication +- Event-driven automation +""" + +from .pipeline import run_smart_home_demo + +__all__ = ["run_smart_home_demo"] diff --git a/sage-apps/src/sage/apps/smart_home/operators.py b/sage-apps/src/sage/apps/smart_home/operators.py new file mode 100644 index 0000000..fbacc33 --- /dev/null +++ b/sage-apps/src/sage/apps/smart_home/operators.py @@ -0,0 +1,263 @@ +""" +SAGE Operators for Smart Home System + +Implements IoT device operators for smart home automation. +""" + +from __future__ import annotations + +import random +import time +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from typing import Any + +from sage.common.core import BatchFunction, MapFunction, SinkFunction + + +class DeviceType(Enum): + """IoT device types.""" + + ROBOT = "robot" + WASHER = "washer" + DRYER = "dryer" + HUMIDITY_SENSOR = "humidity_sensor" + MOTION_SENSOR = "motion_sensor" + + +class TaskStatus(Enum): + """Task execution status.""" + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class DeviceEvent: + """Event emitted by a device.""" + + event_id: int + device_id: str + device_type: DeviceType + event_type: str + timestamp: str + data: dict[str, Any] + + +class LaundryWorkflowSource(BatchFunction): + """ + BatchFunction source that generates laundry workflow tasks. + + Emits a sequence of tasks for the automated laundry workflow. + """ + + def __init__(self, num_cycles: int = 1, **kwargs) -> None: + super().__init__(**kwargs) + self.num_cycles = num_cycles + self.current_cycle = 0 + self.step = 0 + + # Define workflow steps + self.steps = [ + { + "task": "check_environment", + "device": "humid_sensor_001", + "description": "Check humidity and environment", + }, + { + "task": "collect_laundry", + "device": "robot_001", + "description": "Robot collects laundry from basket", + "params": {"from": "basket", "to": "washer"}, + }, + { + "task": "wash", + "device": "washer_001", + "description": "Wash laundry", + "duration": 2.0, + }, + { + "task": "move_to_dryer", + "device": "robot_001", + "description": "Robot moves laundry to dryer", + "params": {"from": "washer", "to": "dryer"}, + }, + { + "task": "dry", + "device": "dryer_001", + "description": "Dry laundry", + "duration": 1.5, + }, + { + "task": "move_to_rack", + "device": "robot_001", + "description": "Robot moves laundry to drying rack", + "params": {"from": "dryer", "to": "drying_rack"}, + }, + ] + + def execute(self) -> dict[str, Any] | None: + """Generate next workflow task.""" + if self.current_cycle >= self.num_cycles: + return None + + if self.step >= len(self.steps): + self.current_cycle += 1 + self.step = 0 + if self.current_cycle >= self.num_cycles: + return None + + task = self.steps[self.step].copy() + task["cycle"] = self.current_cycle + 1 + task["step"] = self.step + 1 + task["total_steps"] = len(self.steps) + + self.step += 1 + return task + + +class DeviceExecutor(MapFunction): + """ + MapFunction that simulates device task execution. + """ + + # Humidity sensor constants + MIN_HUMIDITY = 30.0 + MAX_HUMIDITY = 80.0 + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.event_counter = 0 + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Execute device task.""" + device_id = data.get("device", "unknown") + task = data.get("task", "unknown") + description = data.get("description", "") + duration = data.get("duration", 0.5) + + # Log task start + icon = { + "robot_001": "🤖", + "washer_001": "🧺", + "dryer_001": "💨", + "humid_sensor_001": "📊", + }.get(device_id, "📱") + + self.logger.info(f"{icon} {device_id}: {description}") + + # Simulate task execution + time.sleep(duration) + + # Generate result + self.event_counter += 1 + result = data.copy() + result["status"] = TaskStatus.COMPLETED.value + result["completed_at"] = datetime.now().isoformat() + result["event_id"] = self.event_counter + + # Add device-specific results + if "humid_sensor" in device_id: + result["humidity"] = random.uniform(self.MIN_HUMIDITY, self.MAX_HUMIDITY) + elif task in ["wash", "dry"]: + result["success"] = True + + return result + + +class EnvironmentMonitor(MapFunction): + """ + MapFunction that monitors environmental conditions. + """ + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + """Monitor and log environmental conditions.""" + if data.get("task") == "check_environment": + humidity = data.get("humidity", 0) + self.logger.info(f" 📊 Current humidity: {humidity:.1f}%") + + # Check if conditions are suitable + if 40 <= humidity <= 70: + data["environment_status"] = "optimal" + elif humidity < 40: + data["environment_status"] = "too_dry" + else: + data["environment_status"] = "too_humid" + + return data + + +class WorkflowProgressSink(SinkFunction): + """ + SinkFunction that tracks and displays workflow progress. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.tasks_completed = 0 + self.start_time = time.time() + self.total_steps = 0 + + def execute(self, data: dict[str, Any]) -> None: + """Track task completion.""" + self.tasks_completed += 1 + step = data.get("step", 0) + total = data.get("total_steps", 0) + cycle = data.get("cycle", 1) + + if total > 0: + self.total_steps = total + + # Log progress + if step == total: + self.logger.info(f"✓ Cycle {cycle} completed ({step}/{total} steps)") + + def close(self) -> None: + """Display final summary.""" + elapsed = time.time() - self.start_time + + print("\n" + "=" * 70) + print("✅ Smart Home Workflow Summary") + print("=" * 70) + print(f" Total tasks completed: {self.tasks_completed}") + print(f" Total time: {elapsed:.2f}s") + print(f" Average time per task: {elapsed / max(self.tasks_completed, 1):.2f}s") + print("=" * 70 + "\n") + + +class EventLogSink(SinkFunction): + """ + SinkFunction that logs all device events. + """ + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.events: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> None: + """Log device event.""" + self.events.append(data) + + def close(self) -> None: + """Display event log summary.""" + print("\n📝 Event Log Summary:") + print(f" Total events: {len(self.events)}") + + # Group by device + by_device: dict[str, int] = {} + for event in self.events: + device = event.get("device", "unknown") + by_device[device] = by_device.get(device, 0) + 1 + + print(" Events by device:") + for device, count in sorted(by_device.items()): + icon = { + "robot_001": "🤖", + "washer_001": "🧺", + "dryer_001": "💨", + "humid_sensor_001": "📊", + }.get(device, "📱") + print(f" {icon} {device}: {count} events") diff --git a/sage-apps/src/sage/apps/smart_home/pipeline.py b/sage-apps/src/sage/apps/smart_home/pipeline.py new file mode 100644 index 0000000..2be35b9 --- /dev/null +++ b/sage-apps/src/sage/apps/smart_home/pipeline.py @@ -0,0 +1,131 @@ +""" +Smart Home Pipeline + +Main pipeline implementation using SAGE operators for smart home automation. +""" + +from __future__ import annotations + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + +from .operators import ( + DeviceExecutor, + EnvironmentMonitor, + EventLogSink, + LaundryWorkflowSource, + WorkflowProgressSink, +) + + +def run_smart_home_demo(num_cycles: int = 1, verbose: bool = False) -> None: + """ + Run the smart home laundry automation demonstration. + + Args: + num_cycles: Number of laundry cycles to run + verbose: Enable verbose logging + + Example: + >>> from sage.apps.smart_home import run_smart_home_demo + >>> run_smart_home_demo(num_cycles=1) + """ + print("\n" + "🏠" * 35) + print(" " * 15 + "SAGE Smart Home Automation Demo") + print("🏠" * 35) + print() + print("=" * 70) + print("📋 Configuration") + print("=" * 70) + print(f" Laundry cycles: {num_cycles}") + print(f" Verbose logging: {verbose}") + print("=" * 70) + print() + + print("=" * 70) + print("🔄 Automated Laundry Workflow") + print("=" * 70) + print(" Steps:") + print(" 1. Check environmental conditions (humidity)") + print(" 2. Robot collects laundry from basket") + print(" 3. Washer runs wash cycle") + print(" 4. Robot moves laundry to dryer") + print(" 5. Dryer runs dry cycle") + print(" 6. Robot moves laundry to drying rack") + print("=" * 70) + print() + + # Create SAGE environment + env = LocalEnvironment("smart_home") + + # Build pipeline: + # 1. LaundryWorkflowSource: Generate workflow tasks + # 2. DeviceExecutor: Execute tasks on simulated devices + # 3. EnvironmentMonitor: Monitor environmental conditions + # 4. WorkflowProgressSink: Track progress + # 5. EventLogSink: Log all events + pipeline = ( + env.from_batch( + LaundryWorkflowSource, + num_cycles=num_cycles, + ) + .map(DeviceExecutor) + .map(EnvironmentMonitor) + .sink(WorkflowProgressSink) + ) + + if verbose: + pipeline.sink(EventLogSink) + + # Execute pipeline + print("🚀 Starting smart home automation...") + print() + env.submit(autostop=True) + + print("\n🏠" * 35) + print(" " * 20 + "Demo completed!") + print("🏠" * 35 + "\n") + + +def main(): + """Main entry point for command-line usage.""" + import argparse + + parser = argparse.ArgumentParser( + description="SAGE Smart Home System - IoT device automation demo", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Run single laundry cycle + python -m sage.apps.smart_home.pipeline + + # Run multiple cycles + python -m sage.apps.smart_home.pipeline --cycles 3 + + # Verbose mode + python -m sage.apps.smart_home.pipeline --verbose + """, + ) + + parser.add_argument( + "--cycles", + "-c", + type=int, + default=1, + help="Number of laundry cycles to run (default: 1)", + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") + + args = parser.parse_args() + + # Disable global console debug unless verbose + if not args.verbose: + CustomLogger.disable_global_console_debug() + + # Run demo + run_smart_home_demo(num_cycles=args.cycles, verbose=args.verbose) + + +if __name__ == "__main__": + main() diff --git a/sage-apps/src/sage/apps/video/__init__.py b/sage-apps/src/sage/apps/video/__init__.py new file mode 100644 index 0000000..9801443 --- /dev/null +++ b/sage-apps/src/sage/apps/video/__init__.py @@ -0,0 +1,7 @@ +"""Video intelligence demos for the SAGE examples suite.""" + +from . import operators + +__all__ = [ + "operators", +] diff --git a/sage-apps/src/sage/apps/video/config/default_config.yaml b/sage-apps/src/sage/apps/video/config/default_config.yaml new file mode 100644 index 0000000..648f9bc --- /dev/null +++ b/sage-apps/src/sage/apps/video/config/default_config.yaml @@ -0,0 +1,63 @@ +# Default configuration for the SAGE video intelligence pipeline +# Copy this file and tweak the parameters for your own experiments. + +video_path: # Must be provided via --video argument or config override +# Process one frame every N frames. Increase to downsample for long videos. +sample_every_n_frames: 3 +# Uncomment to shorten runs for quick debugging +# max_frames: 180 +frame_resize: 336 + +analysis: + # CLIP templates that steer zero-shot scene understanding + clip_templates: + - "Outdoor cycling activity" + - "Relaxing in a park" + - "Discussing in an office" + - "Working out in a gym" + - "Night-time city scene" + - "Casual indoor conversation" + - "Sports match in progress" + - "Crowded street" + - "Car driving on a road" + - "Animal present in frame" + clip_top_k: 4 + classifier_top_k: 5 + min_event_confidence: 0.25 + brightness_delta_threshold: 35.0 + +window_summary: + window_seconds: 8.0 + stride_seconds: 4.0 + max_frames_per_window: 120 + summarizer_model: "sshleifer/distilbart-cnn-12-6" + max_summary_tokens: 96 + +output: + timeline_path: "artifacts/video_timeline.jsonl" + summary_path: "artifacts/video_summary.json" + event_stats_path: "artifacts/video_event_stats.json" + preview_every: 12 + +integrations: + enable_sage_db: true + enable_sage_flow: true + enable_neuromem: true + + sage_db: + service_name: "video_scene_db" + dimension: 512 + index_type: "AUTO" + neighbor_k: 3 + + sage_flow: + service_name: "video_vector_flow" + dim: 512 + dtype: "Float32" + auto_flush: 1 + + neuromem: + service_name: "video_memory_service" + collection_name: "demo_collection" + topk: 3 + with_metadata: true diff --git a/sage-apps/src/sage/apps/video/operators/__init__.py b/sage-apps/src/sage/apps/video/operators/__init__.py new file mode 100644 index 0000000..ea420f2 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/__init__.py @@ -0,0 +1,25 @@ +"""Operator collection for the video intelligence demo.""" + +from .analytics import FrameEventEmitter, SlidingWindowSummaryEmitter, TemporalAnomalyDetector +from .formatters import FrameLightweightFormatter +from .integrations import SageMiddlewareIntegrator, SummaryMemoryAugmentor +from .perception import FrameObjectClassifier, SceneConceptExtractor +from .preprocessing import FramePreprocessor +from .sinks import EventStatsSink, SummarySink, TimelineSink +from .sources import VideoFrameSource + +__all__ = [ + "VideoFrameSource", + "FramePreprocessor", + "SceneConceptExtractor", + "FrameObjectClassifier", + "TemporalAnomalyDetector", + "FrameEventEmitter", + "SlidingWindowSummaryEmitter", + "FrameLightweightFormatter", + "SageMiddlewareIntegrator", + "SummaryMemoryAugmentor", + "TimelineSink", + "SummarySink", + "EventStatsSink", +] diff --git a/sage-apps/src/sage/apps/video/operators/analytics.py b/sage-apps/src/sage/apps/video/operators/analytics.py new file mode 100644 index 0000000..1fa7a59 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/analytics.py @@ -0,0 +1,241 @@ +"""Analytics operators for the video intelligence demo.""" + +from __future__ import annotations + +from collections import Counter, deque +from collections.abc import Iterable +from typing import Any + +import torch + +from sage.common.core import FlatMapFunction, MapFunction +from sage.common.utils.logging.custom_logger import CustomLogger + +try: + from transformers import pipeline as hf_pipeline +except ImportError: # pragma: no cover - optional dependency + hf_pipeline = None + + +class TemporalAnomalyDetector(MapFunction): + """Detects lighting shifts and abrupt scene switches.""" + + def __init__(self, brightness_delta_threshold: float = 35.0) -> None: + super().__init__() + self.threshold = max(1.0, brightness_delta_threshold) + self.prev_brightness: float | None = None + self.prev_scene: str | None = None + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + brightness = data.get("brightness") + primary_scene = data.get("primary_scene") + + anomalies: list[dict[str, Any]] = [] + if brightness is not None and self.prev_brightness is not None: + delta = abs(brightness - self.prev_brightness) + if delta >= self.threshold: + anomalies.append( + { + "type": "lighting_shift", + "delta": float(delta), + } + ) + self.prev_brightness = brightness + + if primary_scene and self.prev_scene and primary_scene != self.prev_scene: + anomalies.append( + { + "type": "scene_transition", + "from": self.prev_scene, + "to": primary_scene, + } + ) + if primary_scene: + self.prev_scene = primary_scene + + data["anomalies"] = anomalies + return data + + +class FrameEventEmitter(FlatMapFunction): + """Emits structured events for downstream keyed aggregation.""" + + def __init__(self, min_confidence: float = 0.25) -> None: + super().__init__() + self.min_confidence = min_confidence + + def execute(self, data: dict[str, Any]) -> Iterable[dict[str, Any]]: + frame_id = data.get("frame_id") + timestamp = data.get("timestamp") + events: list[dict[str, Any]] = [] + + for concept in data.get("scene_concepts", []): + if concept["score"] < self.min_confidence: + continue + events.append( + { + "event_type": "scene_concept", + "label": concept["label"], + "score": concept["score"], + "frame_id": frame_id, + "timestamp": timestamp, + } + ) + + for prediction in data.get("object_predictions", []): + if prediction["score"] < self.min_confidence: + continue + events.append( + { + "event_type": "object", + "label": prediction["label"], + "score": prediction["score"], + "frame_id": frame_id, + "timestamp": timestamp, + } + ) + + for anomaly in data.get("anomalies", []): + events.append( + { + "event_type": anomaly.get("type", "anomaly"), + "frame_id": frame_id, + "timestamp": timestamp, + **{k: v for k, v in anomaly.items() if k != "type"}, + } + ) + + return events + + +class SlidingWindowSummaryEmitter(FlatMapFunction): + """Produces natural-language summaries for sliding windows of frames.""" + + def __init__( + self, + window_seconds: float, + stride_seconds: float, + max_frames: int, + summarizer_model: str | None, + max_summary_tokens: int, + ) -> None: + super().__init__() + self.window_seconds = max(1.0, float(window_seconds)) + self.stride_seconds = max(0.5, float(stride_seconds)) + self.max_frames = max(10, int(max_frames)) + self.max_summary_tokens = max(32, int(max_summary_tokens)) + self.window: deque[dict[str, Any]] = deque() + self.last_emit_timestamp: float | None = None + self.summary_index = 0 + + self.summarizer = None + if summarizer_model: + if hf_pipeline is None: + logger = CustomLogger( + outputs=[("console", "INFO")], name="SlidingWindowSummaryEmitter" + ) + logger.warning( + "transformers is not installed; summariser '%s' cannot be loaded. Falling back to template summaries.", + summarizer_model, + ) + else: + try: + device = 0 if torch.cuda.is_available() else -1 + self.summarizer = hf_pipeline( + "summarization", + model=summarizer_model, + device=device, + ) + except Exception as exc: # pragma: no cover - optional model + logger = CustomLogger( + outputs=[("console", "INFO")], + name="SlidingWindowSummaryEmitter", + ) + logger.warning( + "Failed to load summariser '%s': %s. Falling back to template summaries.", + summarizer_model, + exc, + ) + self.summarizer = None + + def execute(self, data: dict[str, Any]) -> Iterable[dict[str, Any]]: + record = { + "frame_id": data.get("frame_id"), + "timestamp": data.get("timestamp"), + "primary_scene": data.get("primary_scene"), + "scene_concepts": data.get("scene_concepts", []), + "object_predictions": data.get("object_predictions", []), + "anomalies": data.get("anomalies", []), + } + self.window.append(record) + + while self.window and ( + record["timestamp"] - self.window[0]["timestamp"] > self.window_seconds + or len(self.window) > self.max_frames + ): + self.window.popleft() + + should_emit = ( + self.last_emit_timestamp is None + or (record["timestamp"] - self.last_emit_timestamp) >= self.stride_seconds + ) + + if not should_emit or not self.window: + return [] + + self.last_emit_timestamp = record["timestamp"] + self.summary_index += 1 + return [self._build_summary()] + + def _build_summary(self) -> dict[str, Any]: + start_ts = self.window[0]["timestamp"] + end_ts = self.window[-1]["timestamp"] + + all_concepts = Counter() + all_objects = Counter() + anomalies = Counter() + captions: list[str] = [] + + for entry in self.window: + for concept in entry.get("scene_concepts", []): + all_concepts[concept["label"]] += concept["score"] + for obj in entry.get("object_predictions", []): + all_objects[obj["label"]] += obj["score"] + for anomaly in entry.get("anomalies", []): + anomalies[anomaly.get("type", "anomaly")] += 1 + if entry.get("primary_scene"): + captions.append(entry["primary_scene"]) + + def _top(counter: Counter, limit: int = 5) -> list[str]: + return [label for label, _ in counter.most_common(limit)] + + composed_text = " " + " ".join(captions) + if self.summarizer and composed_text.strip(): + try: + raw_summary = self.summarizer( + composed_text, + max_length=self.max_summary_tokens, + min_length=max(20, self.max_summary_tokens // 3), + do_sample=False, + )[0]["summary_text"].strip() + except Exception: # pragma: no cover - fallback path + raw_summary = self._fallback_summary(_top(all_concepts, 3), _top(all_objects, 5)) + else: + raw_summary = self._fallback_summary(_top(all_concepts, 3), _top(all_objects, 5)) + + return { + "summary_id": self.summary_index, + "start_timestamp": float(start_ts), + "end_timestamp": float(end_ts), + "duration_seconds": float(end_ts - start_ts), + "top_scene_concepts": _top(all_concepts, 5), + "top_objects": _top(all_objects, 5), + "anomaly_counts": dict(anomalies), + "generated_summary": raw_summary, + } + + @staticmethod + def _fallback_summary(concepts: list[str], objects: list[str]) -> str: + concept_text = ", ".join(concepts) if concepts else "diverse scenes" + object_text = ", ".join(objects) if objects else "generic objects" + return f"Window highlights {concept_text}. Frequent objects include {object_text}." diff --git a/sage-apps/src/sage/apps/video/operators/formatters.py b/sage-apps/src/sage/apps/video/operators/formatters.py new file mode 100644 index 0000000..4a68f20 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/formatters.py @@ -0,0 +1,19 @@ +"""Formatting utilities for the video intelligence demo.""" + +from __future__ import annotations + +from typing import Any + +from sage.common.core import MapFunction + + +class FrameLightweightFormatter(MapFunction): + """Drops heavy artefacts and adds convenience fields.""" + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + data.pop("frame", None) + data.pop("pil_image", None) + data.pop("resized_image", None) + data["top_scene_labels"] = [entry["label"] for entry in data.get("scene_concepts", [])] + data["top_object_labels"] = [entry["label"] for entry in data.get("object_predictions", [])] + return data diff --git a/sage-apps/src/sage/apps/video/operators/integrations.py b/sage-apps/src/sage/apps/video/operators/integrations.py new file mode 100644 index 0000000..cd642ba --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/integrations.py @@ -0,0 +1,178 @@ +"""Service integration operators for the video intelligence demo.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from sage.common.core import MapFunction + + +class SageMiddlewareIntegrator(MapFunction): + """Publishes frame embeddings to SageDB and SageFlow services.""" + + def __init__( + self, + enable_db: bool = True, + db_service_name: str = "video_scene_db", + db_neighbor_k: int = 3, + enable_flow: bool = True, + flow_service_name: str = "video_vector_flow", + flow_auto_flush: int = 1, + ) -> None: + super().__init__() + self.enable_db = enable_db and bool(db_service_name) + self.db_service_name = db_service_name + self.db_neighbor_k = max(0, int(db_neighbor_k)) + self.enable_flow = enable_flow and bool(flow_service_name) + self.flow_service_name = flow_service_name + self.flow_auto_flush = max(1, int(flow_auto_flush)) + self._flow_since_flush = 0 + + def _format_neighbors( + self, neighbors: list[dict[str, Any]], entry_id: Any + ) -> list[dict[str, Any]]: + formatted: list[dict[str, Any]] = [] + for item in neighbors: + if item.get("id") == entry_id: + continue + meta = item.get("metadata", {}) or {} + formatted.append( + { + "score": round(float(item.get("score", 0.0)), 4), + "frame_id": meta.get("frame_id"), + "timestamp": meta.get("timestamp"), + "primary_scene": meta.get("primary_scene"), + } + ) + return formatted[: self.db_neighbor_k] + + def _flush_flow(self) -> None: + if not self.enable_flow or self._flow_since_flush <= 0: + return + try: + self.call_service(self.flow_service_name, method="run") + except Exception as exc: # pragma: no cover - runtime safety + self.logger.warning("SageFlow flush failed: %s", exc) + finally: + self._flow_since_flush = 0 + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + embedding = data.get("clip_image_embedding") + if embedding is None: + return data + + vector = np.asarray(embedding, dtype=np.float32) + vector_list = vector.tolist() + integrations = data.setdefault("integrations", {}) + + if self.enable_db: + metadata = { + "frame_id": str(data.get("frame_id")), + "timestamp": f"{float(data.get('timestamp', 0.0)):.3f}", + "primary_scene": data.get("primary_scene", ""), + } + try: + entry_id = self.call_service( + self.db_service_name, + vector_list, + metadata=metadata, + method="add", + ) + integrations["sage_db_entry_id"] = entry_id + + if self.db_neighbor_k > 0: + neighbors = self.call_service( + self.db_service_name, + vector_list, + method="search", + k=self.db_neighbor_k + 1, + include_metadata=True, + ) + integrations["sage_db_similar"] = self._format_neighbors( + neighbors or [], entry_id + ) + except Exception as exc: # pragma: no cover - service resilience + self.logger.warning("SageDB service call failed: %s", exc) + + if self.enable_flow: + try: + uid = int(data.get("frame_id", 0)) + self.call_service( + self.flow_service_name, + uid, + vector_list, + method="push", + ) + self._flow_since_flush += 1 + if self._flow_since_flush >= self.flow_auto_flush: + self._flush_flow() + except Exception as exc: # pragma: no cover - service resilience + self.logger.warning("SageFlow service call failed: %s", exc) + + data.pop("clip_image_embedding", None) + return data + + def __del__(self): # pragma: no cover - best-effort flush + try: + self._flush_flow() + except Exception: + pass + + +class SummaryMemoryAugmentor(MapFunction): + """Enriches summaries with NeuroMem memory retrieval results.""" + + def __init__( + self, + enable: bool = True, + service_name: str = "video_memory_service", + top_k: int = 3, + collection_name: str | None = None, + with_metadata: bool = True, + ) -> None: + super().__init__() + self.enable = enable and bool(service_name) + self.service_name = service_name + self.top_k = max(1, int(top_k)) + self.collection_name = collection_name + self.with_metadata = with_metadata + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + if not self.enable: + return data + + query = data.get("generated_summary") or " ".join(data.get("top_scene_concepts", [])) + if not query: + return data + + try: + results = self.call_service( + self.service_name, + query, + method="retrieve", + topk=self.top_k, + collection_name=self.collection_name, + with_metadata=self.with_metadata, + ) + except Exception as exc: # pragma: no cover - service resilience + self.logger.warning("NeuroMem retrieval failed: %s", exc) + return data + + if results: + formatted: list[dict[str, Any]] = [] + for item in results: + if isinstance(item, dict): + formatted.append( + { + "text": item.get("text") or item.get("history_query"), + "answer": item.get("answer") or item.get("metadata", {}).get("answer"), + "source": item.get("source_collection"), + } + ) + else: + formatted.append({"text": str(item), "source": None}) + data["memory_recall"] = formatted + + return data diff --git a/sage-apps/src/sage/apps/video/operators/perception.py b/sage-apps/src/sage/apps/video/operators/perception.py new file mode 100644 index 0000000..6611047 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/perception.py @@ -0,0 +1,190 @@ +"""Perception operators for the video intelligence demo.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as func +from PIL import Image +from torchvision.models import MobileNet_V3_Large_Weights, mobilenet_v3_large + +from sage.common.core import MapFunction + +try: + from transformers import CLIPModel, CLIPProcessor +except ImportError as exc: # pragma: no cover - surface a friendly error + raise RuntimeError( + "transformers is required for SceneConceptExtractor. Install it via `pip install transformers`" + ) from exc + + +class SceneConceptExtractor(MapFunction): + """Zero-shot scene understanding using CLIP templates.""" + + def __init__( + self, + templates: Iterable[str], + top_k: int = 3, + device: str | None = None, + ) -> None: + super().__init__() + self.templates = list(templates) + if not self.templates: + raise ValueError("CLIP templates list cannot be empty") + + self.top_k = max(1, top_k) + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + + # Use smaller CLIP model to reduce memory usage + # Options ranked by size (smallest to largest): + # - openai/clip-vit-base-patch32: ~150MB, 86M params (BEST for memory) + # For even lower memory, we can use CPU and half precision + model_name = "openai/clip-vit-base-patch32" + + self.logger.info(f"Loading CLIP model: {model_name} on {self.device}") + + # Load with memory optimization + # Note: In test/CI environments without internet or with network restrictions, + # model loading may fail. We gracefully degrade to passthrough mode. + try: + # Use dtype instead of torch_dtype (torch_dtype is deprecated) + self.model = CLIPModel.from_pretrained( + model_name, + dtype=torch.float16 if self.device == "cuda" else torch.float32, + low_cpu_mem_usage=True, + ) + self.model = self.model.to(self.device) # type: ignore[assignment] + self.processor = CLIPProcessor.from_pretrained(model_name) + self.logger.info("CLIP model loaded successfully") + self.model_available = True + except Exception as e: + self.logger.warning( + f"Failed to load CLIP model (network/cache issue): {e}. " + "Operating in passthrough mode - scene concepts will not be extracted." + ) + self.model = None + self.processor = None + self.model_available = False + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + pil_image_raw = data.get("pil_image") + if pil_image_raw is None: + return data + if not isinstance(pil_image_raw, Image.Image): + return data + pil_image: Image.Image = pil_image_raw + + # If model failed to load, passthrough without scene concepts + if not self.model_available or not self.processor or not self.model: + data["scene_concepts"] = [] + data["scene_vector"] = None + return data + + inputs = self.processor( + text=self.templates, + images=pil_image, + return_tensors="pt", + padding=True, + ) + inputs = {k: v.to(self.device) for k, v in inputs.items()} + + with torch.no_grad(): + outputs = self.model(**inputs) + scores = outputs.logits_per_image.softmax(dim=1)[0] + + if hasattr(outputs, "image_embeds") and outputs.image_embeds is not None: + image_features = outputs.image_embeds + else: + image_features = self.model.get_image_features( + pixel_values=inputs.get("pixel_values") + ) + + image_features = func.normalize(image_features, dim=-1) + data["clip_image_embedding"] = ( + image_features[0].detach().cpu().numpy().astype(np.float32) + ) + + top_k = min(self.top_k, scores.shape[-1]) + top_scores, top_indices = torch.topk(scores, top_k) + concepts: list[dict[str, Any]] = [] + for score, idx in zip(top_scores.tolist(), top_indices.tolist()): + concepts.append( + { + "label": self.templates[idx], + "score": float(score), + } + ) + + data["scene_concepts"] = concepts + data["primary_scene"] = concepts[0]["label"] if concepts else "Unknown" + data["scene_confidence"] = float(concepts[0]["score"]) if concepts else 0.0 + return data + + +class FrameObjectClassifier(MapFunction): + """Image classification via MobileNetV3 over ImageNet classes.""" + + def __init__(self, top_k: int = 5, device: str | None = None) -> None: + super().__init__() + self.top_k = max(1, top_k) + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + + self.logger.info(f"Loading MobileNetV3 model on {self.device}") + + try: + weights = MobileNet_V3_Large_Weights.DEFAULT + self.model = mobilenet_v3_large(weights=weights).to(self.device) + self.model.eval() + + # Enable half precision for GPU to save memory + if self.device == "cuda": + self.model = self.model.half() + + self.preprocess = weights.transforms() + self.categories = weights.meta["categories"] + self.logger.info("MobileNetV3 model loaded successfully") + self.model_available = True + except Exception as e: + self.logger.warning( + f"Failed to load MobileNetV3 model: {e}. " + "Operating in passthrough mode - objects will not be detected." + ) + self.model = None + self.preprocess = None + self.categories = None + self.model_available = False + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + pil_image_raw = data.get("pil_image") + if pil_image_raw is None: + return data + if not isinstance(pil_image_raw, Image.Image): + return data + pil_image: Image.Image = pil_image_raw + + # If model failed to load, passthrough without object detection + if not self.model_available or not self.preprocess or not self.model or not self.categories: + data["detected_objects"] = [] + return data + + tensor = self.preprocess(pil_image).unsqueeze(0).to(self.device) + with torch.no_grad(): + logits = self.model(tensor) + probs = torch.nn.functional.softmax(logits, dim=1)[0] + + k = min(self.top_k, probs.shape[0]) + top_scores, top_indices = torch.topk(probs, k) + predictions: list[dict[str, Any]] = [] + for score, idx in zip(top_scores.tolist(), top_indices.tolist()): + predictions.append( + { + "label": self.categories[idx], + "score": float(score), + } + ) + + data["object_predictions"] = predictions + return data diff --git a/sage-apps/src/sage/apps/video/operators/preprocessing.py b/sage-apps/src/sage/apps/video/operators/preprocessing.py new file mode 100644 index 0000000..f02ff61 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/preprocessing.py @@ -0,0 +1,34 @@ +"""Preprocessing operators for the video intelligence demo.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +from PIL import Image + +from sage.common.core import MapFunction + + +class FramePreprocessor(MapFunction): + """Converts numpy frames to PIL, resizes, and extracts basic metrics.""" + + def __init__(self, target_size: int = 336) -> None: + super().__init__() + self.target_size = target_size + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + frame = data.get("frame") + if frame is None: + return data + + pil_image = Image.fromarray(frame) + data["pil_image"] = pil_image + data["brightness"] = float(np.mean(frame)) + + if self.target_size: + data["resized_image"] = pil_image.resize((self.target_size, self.target_size)) + else: + data["resized_image"] = pil_image + + return data diff --git a/sage-apps/src/sage/apps/video/operators/sinks.py b/sage-apps/src/sage/apps/video/operators/sinks.py new file mode 100644 index 0000000..b024b30 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/sinks.py @@ -0,0 +1,125 @@ +"""Sink operators for the video intelligence demo.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +from sage.common.core import SinkFunction + + +class TimelineSink(SinkFunction): + """Persists per-frame insights as JSONL.""" + + def __init__(self, output_path: str, preview_every: int = 10) -> None: + super().__init__() + self.output_path = Path(output_path) + self.output_path.parent.mkdir(parents=True, exist_ok=True) + self.preview_every = max(1, preview_every) + self.count = 0 + self.file = self.output_path.open("w", encoding="utf-8") + + def execute(self, data: dict[str, Any]) -> None: + safe_data = { + key: value + for key, value in data.items() + if key not in {"frame", "pil_image", "resized_image"} + } + self.file.write(json.dumps(safe_data, ensure_ascii=False) + "\n") + self.count += 1 + + # Enhanced console output for better visibility + if self.count % self.preview_every == 0: + frame_id = safe_data.get("frame_id", "?") + timestamp = safe_data.get("timestamp_seconds", 0) + scene = safe_data.get("primary_scene", "Unknown scene") + objects = ", ".join(safe_data.get("top_object_labels", [])[:3]) + brightness = safe_data.get("brightness", 0) + + print(f"\n{'=' * 70}") + print(f"📹 Frame {frame_id} @ {timestamp:.2f}s") + print(f" Scene: {scene}") + print(f" Objects: {objects or 'none detected'}") + print(f" Brightness: {brightness:.1f}") + + # Show scene scores if available + scene_scores = safe_data.get("scene_scores", {}) + if scene_scores: + top_scenes = sorted(scene_scores.items(), key=lambda x: x[1], reverse=True)[:3] + print(f" Top scenes: {', '.join(f'{s}({v:.2f})' for s, v in top_scenes)}") + + print(f"{'=' * 70}") + + def __del__(self) -> None: # pragma: no cover - defensive cleanup + if hasattr(self, "file") and self.file and not self.file.closed: + self.file.close() + + +class SummarySink(SinkFunction): + """Collects sliding-window summaries and writes a compact JSON report.""" + + def __init__(self, summary_path: str) -> None: + super().__init__() + self.summary_path = Path(summary_path) + self.summary_path.parent.mkdir(parents=True, exist_ok=True) + self.records: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> None: + self.records.append(data) + + # Print summary to console + window_id = len(self.records) + summary_text = data.get("summary", "") + start_time = data.get("window_start_seconds", 0) + end_time = data.get("window_end_seconds", 0) + + print(f"\n🎬 Summary Window #{window_id} ({start_time:.1f}s - {end_time:.1f}s)") + print(f" {summary_text[:200]}{'...' if len(summary_text) > 200 else ''}") + + # Show memory recall if available + memory_recall = data.get("memory_recall", []) + if memory_recall: + print(f" 💭 Memory recalls: {len(memory_recall)} similar moments found") + + def __del__(self) -> None: # pragma: no cover - defensive cleanup + if self.records: + payload = { + "window_count": len(self.records), + "summaries": self.records, + } + with self.summary_path.open("w", encoding="utf-8") as fh: + json.dump(payload, fh, ensure_ascii=False, indent=2) + + +class EventStatsSink(SinkFunction): + """Aggregates event statistics and persists them on teardown.""" + + def __init__(self, stats_path: str, log_every: int = 25) -> None: + super().__init__() + self.stats_path = Path(stats_path) + self.stats_path.parent.mkdir(parents=True, exist_ok=True) + self.counter: Counter[str] = Counter() + self.total_events = 0 + self.log_every = max(1, log_every) + + def execute(self, event: dict[str, Any]) -> None: + event_type = event.get("event_type", "unknown") + self.counter[event_type] += 1 + self.total_events += 1 + + if self.total_events % self.log_every == 0: + top_events = self.counter.most_common(3) + top_str = ", ".join(f"{t}({c})" for t, c in top_events) + print(f"\n📊 Events processed: {self.total_events}") + print(f" Top events: {top_str}") + + def __del__(self) -> None: # pragma: no cover - defensive cleanup + if self.total_events: + payload = { + "total_events": self.total_events, + "event_counts": dict(self.counter), + } + with self.stats_path.open("w", encoding="utf-8") as fh: + json.dump(payload, fh, ensure_ascii=False, indent=2) diff --git a/sage-apps/src/sage/apps/video/operators/sources.py b/sage-apps/src/sage/apps/video/operators/sources.py new file mode 100644 index 0000000..c573422 --- /dev/null +++ b/sage-apps/src/sage/apps/video/operators/sources.py @@ -0,0 +1,73 @@ +"""Source operators for the video intelligence demo.""" + +from __future__ import annotations + +import os +from typing import Any + +import cv2 + +from sage.common.core import BatchFunction + + +class VideoFrameSource(BatchFunction): + """Reads frames from a video file as a BatchFunction source.""" + + def __init__( + self, + video_path: str, + sample_every_n_frames: int = 3, + max_frames: int | None = None, + ) -> None: + super().__init__() + if not os.path.exists(video_path): + raise FileNotFoundError(f"Video file not found: {video_path}") + + self.video_path = video_path + self.sample_every = max(1, int(sample_every_n_frames)) + self.max_frames = max_frames if max_frames and max_frames > 0 else None + self.cap = cv2.VideoCapture(video_path) + if not self.cap.isOpened(): + raise RuntimeError(f"Failed to open video: {video_path}") + + fps = self.cap.get(cv2.CAP_PROP_FPS) or 0.0 + self.fps = float(fps if fps > 0 else 30.0) + self.frame_cursor = 0 + self.frames_emitted = 0 + # logger is provided by BaseFunction via ctx + + def execute(self) -> dict[str, Any] | None: + if not self.cap or not self.cap.isOpened(): + return None + + while True: + ret, frame = self.cap.read() + if not ret: + self.cap.release() + return None + + current_index = self.frame_cursor + self.frame_cursor += 1 + + if (current_index % self.sample_every) != 0: + continue # skip until we hit the sampling cadence + + if self.max_frames and self.frames_emitted >= self.max_frames: + self.cap.release() + return None + + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + timestamp = current_index / self.fps + self.frames_emitted += 1 + + return { + "frame_id": int(current_index), + "timestamp": float(timestamp), + "fps": self.fps, + "frame": frame_rgb, + "video_path": self.video_path, + } + + def __del__(self) -> None: # pragma: no cover - defensive cleanup + if hasattr(self, "cap") and self.cap and self.cap.isOpened(): + self.cap.release() diff --git a/sage-apps/src/sage/apps/video/video_intelligence_pipeline.py b/sage-apps/src/sage/apps/video/video_intelligence_pipeline.py new file mode 100644 index 0000000..b2a904b --- /dev/null +++ b/sage-apps/src/sage/apps/video/video_intelligence_pipeline.py @@ -0,0 +1,587 @@ +"""Advanced video intelligence demo built on SAGE. + +This example upgrades the original notebook-only demo with a fully scripted +pipeline that showcases SAGE's declarative operators (Batch, Map, FlatMap, +KeyBy, Sink) and combines multiple AI capabilities: + +* Frame sampling and preprocessing +* Zero-shot scene understanding with CLIP +* Image classification with MobileNetV3 +* Lightweight temporal anomaly detection +* Sliding-window summarisation backed by an optional HuggingFace summariser +* Structured event stream with keyed aggregation for observability + +@test:skip - Requires HuggingFace model downloads (CLIP, MobileNetV3). +CI environments may have network restrictions preventing model downloads. +Run with: python -m sage.apps.video.video_intelligence_pipeline +Or use: python examples/apps/run_video_intelligence.py +""" + +from __future__ import annotations + +import argparse +import os +import urllib.request +from pathlib import Path +from typing import Any + +import yaml + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# Helper function to get logger +def get_logger(name: str) -> CustomLogger: + """Get a CustomLogger instance with the given name.""" + return CustomLogger(outputs=[("console", "INFO")], name=name) + + +try: # Optional middleware components - sage_mem (migrated from neuromem) + # Note: NeuroMemVDBService has been refactored in the new architecture + # Using registry-based approach instead + from sage.middleware.components.sage_mem.neuromem.services.neuromem_service_factory import ( + NeuromemServiceFactory, + ) + from sage.middleware.components.sage_mem.neuromem.services.registry import MemoryServiceRegistry + + # Legacy compatibility - may need updating based on new service API + NeuroMemVDBService = None # Deprecated - use registry instead +except ImportError: # pragma: no cover - optional dependency + MemoryServiceRegistry = None # type: ignore[assignment] + NeuromemServiceFactory = None # type: ignore[assignment] + NeuroMemVDBService = None # type: ignore[assignment] + +try: + from sage.middleware.components.sage_db.python.micro_service import SageDBService +except ImportError: # pragma: no cover - optional dependency + SageDBService = None # type: ignore[assignment] + +try: + from sage.middleware.components.sage_flow.python.micro_service import SageFlowService +except ImportError: # pragma: no cover - optional dependency + SageFlowService = None # type: ignore[assignment] + +from sage.apps.video.operators import ( + EventStatsSink, # noqa: E402 + FrameEventEmitter, + FrameLightweightFormatter, + FrameObjectClassifier, + FramePreprocessor, + SageMiddlewareIntegrator, + SceneConceptExtractor, + SlidingWindowSummaryEmitter, + SummaryMemoryAugmentor, + SummarySink, + TemporalAnomalyDetector, + TimelineSink, + VideoFrameSource, +) + +# Default config path +DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent / "config" / "default_config.yaml" + + +def load_config(config_path: str | None) -> dict[str, Any]: + """Load YAML configuration for the demo.""" + + # If user provided a config path, use it + if config_path: + path = Path(config_path) + if not path.exists(): + raise FileNotFoundError(f"Config file not found: {path}") + + with path.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise ValueError(f"Config file {path} must define a mapping at the top level") + print(f"[INFO] Loaded config from: {path}") + return data + + # Use default config + if DEFAULT_CONFIG_PATH.exists(): + with DEFAULT_CONFIG_PATH.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + if not isinstance(data, dict): + raise ValueError( + f"Config file {DEFAULT_CONFIG_PATH} must define a mapping at the top level" + ) + print(f"[INFO] Loaded config from: {DEFAULT_CONFIG_PATH}") + return data + + # If no config file found, return default configuration + print("[WARNING] No config file found") + print("[INFO] Using built-in default configuration") + return { + "video_path": None, + "output_dir": "output/video_intelligence", + "max_frames": None, + "sample_every_n_frames": 3, + "frame_resize": 336, + "analysis": { + "clip_top_k": 4, + "classifier_top_k": 5, + "min_event_confidence": 0.25, + }, + "models": { + "clip": "openai/clip-vit-base-patch32", + "mobilenet": "google/mobilenet_v3_small_100_224", + }, + } + + +def download_test_video(output_path: str) -> bool: + """Download a small test video for automated testing. + + Args: + output_path: Path where the video should be saved + + Returns: + True if download succeeded, False otherwise + """ + # Use a small sample video from a public source + # This is a ~1MB sample video suitable for testing + test_video_url = "https://sample-videos.com/video321/mp4/240/big_buck_bunny_240p_1mb.mp4" + + logger = CustomLogger("video_downloader") + + try: + logger.info(f"Downloading test video from {test_video_url}") + logger.info(f"Saving to {output_path}") + + # Create parent directory if it doesn't exist + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + + # Download the file + urllib.request.urlretrieve(test_video_url, output_path) + + # Verify the file exists and has content + if os.path.exists(output_path) and os.path.getsize(output_path) > 0: + logger.info( + f"Successfully downloaded test video ({os.path.getsize(output_path)} bytes)" + ) + return True + else: + logger.error("Downloaded file is empty or doesn't exist") + return False + + except Exception as e: + logger.error(f"Failed to download test video: {e}") + return False + + +def ensure_video_exists(video_path: str, auto_download: bool = True) -> str: + """Ensure a video file exists, downloading a test video if needed. + + Args: + video_path: Configured video path + auto_download: Whether to automatically download a test video if file doesn't exist + + Returns: + Path to the video file + + Raises: + FileNotFoundError: If video doesn't exist and auto_download is False or download failed + """ + logger = CustomLogger("video_validator") + + # If the configured path exists, use it + if os.path.exists(video_path): + logger.info(f"Using existing video file: {video_path}") + return video_path + + # If auto-download is disabled, raise an error + if not auto_download: + raise FileNotFoundError( + f"Video file '{video_path}' does not exist. Provide --video or update the config." + ) + + # Try to download a test video + logger.warning(f"Video file '{video_path}' not found. Attempting to download test video...") + + # Determine download path + if video_path == "./video_demo.mp4": + # Default case: download to current directory or sage.apps.video location + download_path = str(Path(__file__).parent / "video_demo.mp4") + else: + # Use the configured path + download_path = video_path + + if download_test_video(download_path): + logger.info(f"Test video ready at: {download_path}") + return download_path + else: + raise FileNotFoundError( + f"Video file '{video_path}' does not exist and automatic download failed. " + f"Please provide a video file using --video or update the config." + ) + + +# --------------------------------------------------------------------------- +# Pipeline assembly +# --------------------------------------------------------------------------- + + +def build_pipeline(env: LocalEnvironment, config: dict[str, Any]) -> None: + video_path = config["video_path"] + sample_every = config["sample_every_n_frames"] + max_frames = config.get("max_frames") + frame_resize = config.get("frame_resize") + + analysis_cfg = config.get("analysis", {}) + window_cfg = config.get("window_summary", {}) + output_cfg = config.get("output", {}) + integrations_cfg = config.get("integrations", {}) + + logger = get_logger("video_intelligence_pipeline") + + # ------------------------------------------------------------------ + # Service registrations (SageDB, SageFlow, NeuroMem) + # ------------------------------------------------------------------ + db_cfg = integrations_cfg.get("sage_db", {}) + db_service_name = db_cfg.get("service_name", "video_scene_db") + db_dimension = int(db_cfg.get("dimension", 512)) + enable_db = bool(integrations_cfg.get("enable_sage_db", False)) + if enable_db and SageDBService is None: + logger.warning( + "SageDBService is unavailable (module missing). Disabling SageDB integration." + ) + enable_db = False + + flow_cfg = integrations_cfg.get("sage_flow", {}) + flow_service_name = flow_cfg.get("service_name", "video_vector_flow") + flow_dim = int(flow_cfg.get("dim", 512)) + flow_dtype = flow_cfg.get("dtype", "Float32") + enable_flow = bool(integrations_cfg.get("enable_sage_flow", False)) + if enable_flow and SageFlowService is None: + logger.warning( + "SageFlowService is unavailable (module missing). Disabling SageFlow integration." + ) + enable_flow = False + + memory_cfg = integrations_cfg.get("neuromem", {}) + memory_service_name = memory_cfg.get("service_name", "video_memory_service") + memory_collection = memory_cfg.get("collection_name", "demo_collection") + enable_neuromem = bool(integrations_cfg.get("enable_neuromem", False)) + + # Disable NeuroMem in test mode (requires pre-created collection) + if os.environ.get("SAGE_EXAMPLES_MODE") == "test": + if enable_neuromem: + logger.info("Test mode: Disabling NeuroMem (requires collection setup)") + enable_neuromem = False + + if enable_neuromem and NeuroMemVDBService is None: + logger.warning( + "NeuroMemVDBService is unavailable (module missing). Disabling NeuroMem integration." + ) + enable_neuromem = False + + if enable_db: + if not SageDBService: + raise RuntimeError("SageDBService not available") + try: + env.register_service( + db_service_name, + SageDBService, + dimension=db_dimension, + index_type=db_cfg.get("index_type", "AUTO"), + ) + logger.info( + "Registered SageDB service '%s' (dim=%d)", + db_service_name, + db_dimension, + ) + except Exception as exc: # pragma: no cover - runtime resilience + logger.error("Failed to register SageDB service: %s", exc) + enable_db = False + + if enable_flow: + if not SageFlowService: + raise RuntimeError("SageFlowService not available") + try: + env.register_service( + flow_service_name, + SageFlowService, + dim=flow_dim, + dtype=flow_dtype, + ) + logger.info( + "Registered SageFlow service '%s' (dim=%d, dtype=%s)", + flow_service_name, + flow_dim, + flow_dtype, + ) + except Exception as exc: # pragma: no cover - runtime resilience + logger.error("Failed to register SageFlow service: %s", exc) + enable_flow = False + + if enable_neuromem: + if not NeuroMemVDBService: + raise RuntimeError("NeuroMemVDBService not available") + try: + env.register_service( + memory_service_name, + NeuroMemVDBService, + collection_name=memory_collection, + ) + logger.info( + "Registered NeuroMem service '%s' (collection=%s)", + memory_service_name, + memory_collection, + ) + except Exception as exc: # pragma: no cover - runtime resilience + logger.error("Failed to register NeuroMem service: %s", exc) + enable_neuromem = False + + source_stream = env.from_source( + VideoFrameSource, + video_path=video_path, + sample_every_n_frames=sample_every, + max_frames=max_frames, + ) + + annotated_stream = ( + source_stream.map(FramePreprocessor, target_size=frame_resize) + .map( + SceneConceptExtractor, + templates=analysis_cfg.get("clip_templates", []), + top_k=analysis_cfg.get("clip_top_k", 4), + ) + .map( + FrameObjectClassifier, + top_k=analysis_cfg.get("classifier_top_k", 5), + ) + .map( + TemporalAnomalyDetector, + brightness_delta_threshold=analysis_cfg.get("brightness_delta_threshold", 35.0), + ) + .map( + SageMiddlewareIntegrator, + enable_db=enable_db, + db_service_name=db_service_name, + db_neighbor_k=int(db_cfg.get("neighbor_k", 3)), + enable_flow=enable_flow, + flow_service_name=flow_service_name, + flow_auto_flush=int(flow_cfg.get("auto_flush", 1)), + ) + .map(FrameLightweightFormatter) + ) + + annotated_stream.sink( + TimelineSink, + output_path=output_cfg.get("timeline_path"), + preview_every=output_cfg.get("preview_every", 12), + ) + + events_stream = annotated_stream.flatmap( + FrameEventEmitter, + min_confidence=analysis_cfg.get("min_event_confidence", 0.22), + ) + + events_stream.sink( + EventStatsSink, + stats_path=output_cfg.get("event_stats_path"), + log_every=30, + ) + + summary_stream = annotated_stream.flatmap( + SlidingWindowSummaryEmitter, + window_seconds=window_cfg.get("window_seconds", 8.0), + stride_seconds=window_cfg.get("stride_seconds", 4.0), + max_frames=window_cfg.get("max_frames_per_window", 120), + summarizer_model=window_cfg.get("summarizer_model"), + max_summary_tokens=window_cfg.get("max_summary_tokens", 90), + ) + + summary_stream = summary_stream.map( + SummaryMemoryAugmentor, + enable=enable_neuromem, + service_name=memory_service_name, + top_k=int(memory_cfg.get("topk", 3)), + collection_name=memory_collection, + with_metadata=bool(memory_cfg.get("with_metadata", True)), + ) + + summary_stream.sink( + SummarySink, + summary_path=output_cfg.get("summary_path"), + ) + + +# --------------------------------------------------------------------------- +# CLI wiring +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the advanced SAGE video intelligence pipeline" + ) + parser.add_argument( + "--video", + dest="video_path", + type=str, + help="Path to a local video file (overrides config)", + ) + parser.add_argument( + "--config", + dest="config_path", + type=str, + help="Optional YAML config overriding defaults", + ) + parser.add_argument( + "--output-dir", + dest="output_dir", + type=str, + help="Optional base directory for generated artefacts", + ) + parser.add_argument( + "--max-frames", + dest="max_frames", + type=int, + help="Limit the number of frames processed (for quick demos)", + ) + return parser.parse_args() + + +def apply_runtime_overrides(config: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: + if args.video_path: + config["video_path"] = args.video_path + if args.max_frames is not None: + config["max_frames"] = args.max_frames + + output_dir = args.output_dir + if output_dir: + out_cfg = config.setdefault("output", {}) + base = Path(output_dir) + out_cfg["timeline_path"] = str(base / "timeline.jsonl") + out_cfg["summary_path"] = str(base / "summary.json") + out_cfg["event_stats_path"] = str(base / "event_stats.json") + + return config + + +def download_test_video_to_temp() -> str | None: + """Download a small test video for CI/testing purposes. + + Uses a public domain short video clip suitable for testing. + Returns the path to the downloaded video file, or None if download fails. + """ + import urllib.request + from pathlib import Path + + # Use a small public domain video (approx 1-2MB) + # Sample videos from Pexels (free to use) + test_video_url = ( + "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_1MB.mp4" + ) + + # Download to a temp location + video_dir = Path("/tmp/sage_test_videos") + video_dir.mkdir(exist_ok=True) + video_path = video_dir / "test_video.mp4" + + if video_path.exists(): + print(f"[INFO] Test video already exists at {video_path}") + return str(video_path) + + try: + print(f"[INFO] Downloading test video from {test_video_url}") + urllib.request.urlretrieve(test_video_url, str(video_path)) + print(f"[INFO] Test video downloaded to {video_path}") + return str(video_path) + except Exception as e: + print(f"[WARNING] Failed to download test video: {e}") + return None + + +def main() -> None: + args = parse_args() + config = load_config(args.config_path) + config = apply_runtime_overrides(config, args) + + video_path = config.get("video_path") + + # If no video file provided, try to download a test video + if not video_path or not os.path.exists(video_path): + print("\n" + "=" * 80) + print("📥 No video file provided - attempting to download test video...") + print("=" * 80) + test_video = download_test_video_to_temp() + if test_video: + video_path = test_video + config["video_path"] = video_path + print(f"✅ Test video downloaded: {test_video}") + + # In test mode, limit frames for faster testing + if os.environ.get("SAGE_EXAMPLES_MODE") == "test" and "max_frames" not in config: + config["max_frames"] = 30 + print("[INFO] Test mode: Limiting to 30 frames for faster testing") + else: + print("\n" + "=" * 80) + print("❌ Error: Could not download test video") + print("=" * 80) + print("\nPlease provide a video file using one of these methods:") + print(" 1. Command line: --video path/to/video.mp4") + print(" 2. Config file: Set 'video_path' in the config YAML") + print("\nExample:") + print(" python examples/apps/run_video_intelligence.py --video my_video.mp4") + print("=" * 80 + "\n") + raise FileNotFoundError( + "Could not download test video and no video file provided. " + "Please provide --video or update the config." + ) + print("=" * 80 + "\n") + + # Double check video file exists + if not video_path or not os.path.exists(video_path): + raise FileNotFoundError(f"Video file '{video_path}' does not exist.") + + env = LocalEnvironment("video_intelligence_demo") + build_pipeline(env, config) + + logger = get_logger("video_intelligence_demo") + logger.info("Starting pipeline on video '%s'", video_path) + + print("\n" + "=" * 80) + print("🎥 SAGE Video Intelligence Pipeline") + print("=" * 80) + print(f"📁 Video: {video_path}") + print(f"📊 Config: {config.get('video_path', 'default')}") + print("⚙️ Services: ", end="") + services = [] + if config.get("integrations", {}).get("enable_sage_db"): + services.append("SageDB") + if config.get("integrations", {}).get("enable_sage_flow"): + services.append("SageFlow") + if ( + config.get("integrations", {}).get("enable_neuromem") + and os.environ.get("SAGE_EXAMPLES_MODE") != "test" + ): + services.append("NeuroMem") + print(", ".join(services) if services else "None") + print("=" * 80 + "\n") + + # Submit and wait for pipeline to complete + env.submit(autostop=True) + + print("\n" + "=" * 80) + print("✅ Pipeline execution completed!") + + # Show output file locations + output_cfg = config.get("output", {}) + if output_cfg: + print("\n📂 Output files generated:") + if output_cfg.get("timeline_path"): + print(f" • Timeline: {output_cfg['timeline_path']}") + if output_cfg.get("summary_path"): + print(f" • Summary: {output_cfg['summary_path']}") + if output_cfg.get("event_stats_path"): + print(f" • Events: {output_cfg['event_stats_path']}") + print("=" * 80 + "\n") + + logger.info("Pipeline execution completed") + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/sage-apps/src/sage/apps/work_report_generator/README.md b/sage-apps/src/sage/apps/work_report_generator/README.md new file mode 100644 index 0000000..dc3f6a6 --- /dev/null +++ b/sage-apps/src/sage/apps/work_report_generator/README.md @@ -0,0 +1,286 @@ +# Work Report Generator + +基于 SAGE 框架的周报/日报自动生成器。 + +## 功能特性 + +- **GitHub 数据获取**: 自动获取指定仓库和分支的 commits 和 Pull Requests +- **子模块支持**: 默认收集 SAGE 主仓库及所有子模块的贡献 +- **贡献者统计**: 按贡献者聚合统计数据 +- **AI 摘要生成**: 使用 LLM 生成智能工作总结 +- **多种输出格式**: 支持 Markdown、JSON、Console 输出 +- **日记集成**: 可选集成个人工作日志 +- **CI/CD 集成**: 支持自动化定时生成（每周一 17:00 北京时间） + +## 快速开始 + +### 安装 + +```bash +# 安装 sage-apps +pip install -e packages/sage-apps +``` + +### 基本使用 + +```python +from sage.apps.work_report_generator import run_work_report_pipeline + +# 生成默认报告（所有 SAGE 仓库，main-dev 分支，过去7天） +run_work_report_pipeline() + +# 自定义配置 +run_work_report_pipeline( + repos=["intellistream/SAGE", "intellistream/sageLLM"], + branch="main-dev", # 指定分支 + days=14, + output_format="markdown", + output_path="reports/biweekly_report.md", + language="zh", # 中文报告 + include_submodules=False, # 仅指定仓库，不包含子模块 +) +``` + +### 命令行使用 + +```bash +# 基本使用（收集所有 SAGE 仓库 main-dev 分支的贡献） +python -m sage.apps.work_report_generator.pipeline + +# 仅主仓库，不包含子模块 +python -m sage.apps.work_report_generator.pipeline --no-submodules + +# 指定分支（如 main 而非 main-dev） +python -m sage.apps.work_report_generator.pipeline --branch main + +# 指定仓库和时间范围 +python -m sage.apps.work_report_generator.pipeline \ + --repos intellistream/SAGE,intellistream/sageLLM \ + --days 14 + +# JSON 格式输出 +python -m sage.apps.work_report_generator.pipeline \ + --format json \ + --output reports/weekly.json + +# 英文报告，不使用 LLM +python -m sage.apps.work_report_generator.pipeline \ + --language en \ + --no-llm +``` + +### 示例脚本 + +```bash +python examples/apps/run_work_report.py --help +python examples/apps/run_work_report.py --days 7 +``` + +## 默认仓库列表 + +当不指定 `repos` 参数时，pipeline 默认收集以下所有仓库的贡献： + +| 仓库 | 说明 | +| ------------------------ | ------------------ | +| `intellistream/SAGE` | 主仓库 | +| `intellistream/SAGE-Pub` | 文档 (docs-public) | +| `intellistream/sageData` | Benchmark 数据 | +| `intellistream/sageLLM` | LLM 调度模块 | +| `intellistream/LibAMM` | 近似矩阵乘法库 | +| `intellistream/sageVDB` | 数据库中间件 | +| `intellistream/sageFlow` | 流处理引擎 | +| `intellistream/neuromem` | 内存管理模块 | +| `intellistream/sageTSDB` | 时序数据库 | + +## 配置选项 + +| 参数 | 类型 | 默认值 | 说明 | +| -------------------- | --------- | ---------- | ---------------------------------------- | +| `repos` | list[str] | None | GitHub 仓库列表，None 表示所有 SAGE 仓库 | +| `branch` | str | "main-dev" | 目标分支名称 | +| `days` | int | 7 | 统计天数 | +| `output_format` | str | "markdown" | 输出格式 (console/markdown/json) | +| `output_path` | str | None | 输出文件路径 | +| `diary_path` | str | None | 日记文件/目录路径 | +| `language` | str | "zh" | 报告语言 (zh/en) | +| `github_token` | str | None | GitHub Token (可用环境变量) | +| `use_llm` | bool | True | 是否使用 LLM 生成摘要 | +| `include_submodules` | bool | True | 是否包含子模块仓库 | + +## 环境变量 + +```bash +# GitHub Token (必需，用于 API 访问) +export GITHUB_TOKEN=ghp_xxxxxxxxxxxx + +# 或使用 GIT_TOKEN +export GIT_TOKEN=ghp_xxxxxxxxxxxx +``` + +## 架构设计 + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Work Report Pipeline │ +├─────────────────────────────────────────────────────────────────┤ +│ GitHubDataSource ──┬──► ContributorAggregator │ +│ (main-dev branch) │ │ │ +│ - SAGE │ ▼ │ +│ - sageLLM │ LLMReportGenerator │ +│ - sageVDB │ │ │ +│ - sageFlow │ ▼ │ +│ - neuromem │ ReportSink │ +│ - sageTSDB │ (md/json/console) │ +│ - SAGE-docs │ │ +│ - SAGE-data │ │ +│ - libamm │ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +│ DiaryEntrySource ──┘ ▼ │ │ LLMReportGenerator (Optional) │ │ │ │ │ ▼ │ │ ReportSink │ │ (Console / +Markdown / JSON) │ └─────────────────────────────────────────────────────────────────┘ + +```` + +### 核心组件 + +| 组件 | 类型 | 功能 | +|------|------|------| +| `GitHubDataSource` | BatchFunction | 从 GitHub 获取 commits 和 PRs | +| `DiaryEntrySource` | BatchFunction | 加载本地日记文件 | +| `ContributorAggregator` | MapFunction | 按贡献者聚合数据 | +| `LLMReportGenerator` | MapFunction | 使用 LLM 生成摘要 | +| `ReportSink` | SinkFunction | 输出最终报告 | + +## 输出示例 + +### Markdown 格式 + +```markdown +# Weekly Work Report + +**Period:** 2024-01-08 ~ 2024-01-15 +**Repositories:** intellistream/SAGE + +## Summary + +- Total Commits: 42 +- Total PRs: 8 (6 merged) + +## Contributors + +### developer1 + +- **Commits:** 15 +- **PRs:** 3 (merged: 2) +- **Lines Changed:** +1500 / -300 + +**AI Summary:** +> developer1 本周主要完成了周报生成器的核心功能实现... +```` + +### JSON 格式 + +```json +{ + "start_date": "2024-01-08", + "end_date": "2024-01-15", + "repos": ["intellistream/SAGE"], + "summary": { + "total_commits": 42, + "total_prs": 8, + "total_merged_prs": 6 + }, + "contributors": [ + { + "username": "developer1", + "stats": { + "commits": 15, + "prs": 3, + "merged_prs": 2 + }, + "llm_summary": "..." + } + ] +} +``` + +## CI/CD 集成 + +### GitHub Actions (Weekly Schedule) + +参见 `.github/workflows/weekly-report.yml`: + +```yaml +name: Weekly Report Generator +on: + schedule: + - cron: '0 9 * * 1' # 每周一 9:00 UTC + workflow_dispatch: + +jobs: + generate-report: + runs-on: [self-hosted, A6000] + steps: + - uses: actions/checkout@v4 + - name: Generate Report + run: | + python -m sage.apps.work_report_generator.pipeline \ + --repos intellistream/SAGE \ + --days 7 \ + --format markdown \ + --output reports/weekly_$(date +%Y%m%d).md + env: + GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} +``` + +## 日记格式 + +### JSON 格式 + +```json +[ + { + "date": "2024-01-15", + "author": "developer1", + "content": "今天完成了周报生成器的开发...", + "tags": ["development", "feature"], + "category": "work" + } +] +``` + +### Markdown 格式 + +文件名格式: `YYYY-MM-DD.md` + +```markdown +# 2024-01-15 工作日志 + +今天主要完成了以下工作： +- 实现了 GitHub 数据获取模块 +- 添加了 LLM 摘要生成功能 +``` + +## 开发 + +### 运行测试 + +```bash +pytest packages/sage-apps/tests/unit/test_work_report_generator/ -v +``` + +### 代码结构 + +``` +packages/sage-apps/src/sage/apps/work_report_generator/ +├── __init__.py # 模块入口 +├── models.py # 数据模型 +├── operators.py # SAGE 算子 +├── pipeline.py # 管道实现 +└── README.md # 文档 +``` + +## 许可证 + +Apache License 2.0 diff --git a/sage-apps/src/sage/apps/work_report_generator/__init__.py b/sage-apps/src/sage/apps/work_report_generator/__init__.py new file mode 100644 index 0000000..ebab68f --- /dev/null +++ b/sage-apps/src/sage/apps/work_report_generator/__init__.py @@ -0,0 +1,20 @@ +""" +Work Report Generator - Weekly/Daily report generator based on SAGE framework. + +This application generates work reports by combining: +- GitHub commit and PR data from specified repositories +- Optional diary entries for personal notes +- LLM-powered summarization and formatting + +Example: + >>> from sage.apps.work_report_generator import run_work_report_pipeline + >>> run_work_report_pipeline( + ... repos=["intellistream/SAGE"], + ... days=7, + ... output_format="markdown" + ... ) +""" + +from .pipeline import run_work_report_pipeline + +__all__ = ["run_work_report_pipeline"] diff --git a/sage-apps/src/sage/apps/work_report_generator/models.py b/sage-apps/src/sage/apps/work_report_generator/models.py new file mode 100644 index 0000000..2868ecb --- /dev/null +++ b/sage-apps/src/sage/apps/work_report_generator/models.py @@ -0,0 +1,292 @@ +""" +Data models for Work Report Generator. + +Contains dataclasses for commits, PRs, diary entries, and reports. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any + + +class ReportPeriod(Enum): + """Report period types.""" + + WEEKLY = "weekly" + MONTHLY = "monthly" + QUARTERLY = "quarterly" + YEARLY = "yearly" + + @property + def display_name(self) -> str: + """Get display name for the period.""" + names = { + ReportPeriod.WEEKLY: "周报", + ReportPeriod.MONTHLY: "月报", + ReportPeriod.QUARTERLY: "季报", + ReportPeriod.YEARLY: "年报", + } + return names.get(self, self.value) + + @property + def english_name(self) -> str: + """Get English name for the period.""" + names = { + ReportPeriod.WEEKLY: "Weekly Report", + ReportPeriod.MONTHLY: "Monthly Report", + ReportPeriod.QUARTERLY: "Quarterly Report", + ReportPeriod.YEARLY: "Yearly Report", + } + return names.get(self, self.value.title()) + + +@dataclass +class GitHubCommit: + """Represents a GitHub commit.""" + + sha: str + message: str + author: str + author_email: str + committed_date: str + repo: str + url: str + additions: int = 0 + deletions: int = 0 + changed_files: int = 0 + # For Copilot commits: the user who assigned/owns the work + assigned_by: str | None = None + + @classmethod + def from_graphql(cls, data: dict[str, Any], repo: str) -> GitHubCommit: + """Create from GraphQL response.""" + author = data.get("author", {}) or {} + user = author.get("user", {}) or {} + author_login = user.get("login", author.get("name", "Unknown")) + + # Try to find the assigning user for Copilot commits + assigned_by = None + if cls._is_copilot_author(author_login): + assigned_by = cls._extract_assigned_by(data) + + return cls( + sha=data.get("oid", ""), + message=data.get("message", "").split("\n")[0], # First line only + author=author_login, + author_email=author.get("email", ""), + committed_date=data.get("committedDate", ""), + repo=repo, + url=data.get("url", ""), + additions=data.get("additions", 0), + deletions=data.get("deletions", 0), + changed_files=data.get("changedFiles", 0), + assigned_by=assigned_by, + ) + + @staticmethod + def _is_copilot_author(author: str) -> bool: + """Check if the author is Copilot.""" + copilot_names = {"copilot", "copilot-swe-agent", "github-copilot"} + return author.lower() in copilot_names + + @staticmethod + def _extract_assigned_by(data: dict[str, Any]) -> str | None: + """Extract the user who assigned/owns the Copilot work.""" + # Try from associated PR author or assignees + associated_prs = data.get("associatedPullRequests", {}) or {} + pr_nodes = associated_prs.get("nodes", []) + + if pr_nodes: + pr = pr_nodes[0] + # First try assignees + assignees = pr.get("assignees", {}) or {} + assignee_nodes = assignees.get("nodes", []) + if assignee_nodes: + return assignee_nodes[0].get("login") + + # Then try PR author + pr_author = pr.get("author", {}) or {} + pr_author_login = pr_author.get("login") + if pr_author_login and not GitHubCommit._is_copilot_author(pr_author_login): + return pr_author_login + + # Try from committer (different from author) + committer = data.get("committer", {}) or {} + committer_user = committer.get("user", {}) or {} + committer_login = committer_user.get("login") + if committer_login and not GitHubCommit._is_copilot_author(committer_login): + return committer_login + + return None + + +@dataclass +class GitHubPullRequest: + """Represents a GitHub Pull Request.""" + + number: int + title: str + author: str + state: str # OPEN, CLOSED, MERGED + created_at: str + merged_at: str | None + closed_at: str | None + repo: str + url: str + additions: int = 0 + deletions: int = 0 + changed_files: int = 0 + labels: list[str] = field(default_factory=list) + reviewers: list[str] = field(default_factory=list) + # For Copilot PRs: the user who assigned/owns the work + assigned_by: str | None = None + + @classmethod + def from_graphql(cls, data: dict[str, Any], repo: str) -> GitHubPullRequest: + """Create from GraphQL response.""" + author_data = data.get("author", {}) or {} + author_login = author_data.get("login", "Unknown") + + labels = [ + label.get("name", "") for label in (data.get("labels", {}) or {}).get("nodes", []) + ] + reviewers = [ + reviewer.get("login", "") + for reviewer in (data.get("reviewRequests", {}) or {}).get("nodes", []) + if reviewer.get("requestedReviewer") + ] + + # Extract assignees for Copilot attribution + assigned_by = None + if cls._is_copilot_author(author_login): + assigned_by = cls._extract_assigned_by(data, author_login) + + return cls( + number=data.get("number", 0), + title=data.get("title", ""), + author=author_login, + state=data.get("state", "UNKNOWN"), + created_at=data.get("createdAt", ""), + merged_at=data.get("mergedAt"), + closed_at=data.get("closedAt"), + repo=repo, + url=data.get("url", ""), + additions=data.get("additions", 0), + deletions=data.get("deletions", 0), + changed_files=data.get("changedFiles", 0), + labels=labels, + reviewers=reviewers, + assigned_by=assigned_by, + ) + + @staticmethod + def _is_copilot_author(author: str) -> bool: + """Check if the author is Copilot.""" + copilot_names = {"copilot", "copilot-swe-agent", "github-copilot"} + return author.lower() in copilot_names + + @staticmethod + def _extract_assigned_by(data: dict[str, Any], author_login: str) -> str | None: + """Extract the user who assigned/owns the Copilot PR.""" + # Try from assignees first + assignees = data.get("assignees", {}) or {} + assignee_nodes = assignees.get("nodes", []) + for assignee in assignee_nodes: + login = assignee.get("login") + if login and not GitHubPullRequest._is_copilot_author(login): + return login + + # Try from reviewers + review_requests = data.get("reviewRequests", {}) or {} + for req in review_requests.get("nodes", []): + reviewer = req.get("requestedReviewer", {}) or {} + login = reviewer.get("login") + if login and not GitHubPullRequest._is_copilot_author(login): + return login + + return None + + +@dataclass +class DiaryEntry: + """Represents a diary/note entry.""" + + date: str + author: str + content: str + tags: list[str] = field(default_factory=list) + category: str = "general" + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DiaryEntry: + """Create from dictionary.""" + return cls( + date=data.get("date", ""), + author=data.get("author", "Unknown"), + content=data.get("content", ""), + tags=data.get("tags", []), + category=data.get("category", "general"), + ) + + +@dataclass +class ContributorSummary: + """Summary of contributions for a single contributor.""" + + username: str + commits: list[GitHubCommit] = field(default_factory=list) + pull_requests: list[GitHubPullRequest] = field(default_factory=list) + diary_entries: list[DiaryEntry] = field(default_factory=list) + + # Aggregated stats + total_commits: int = 0 + total_prs: int = 0 + merged_prs: int = 0 + total_additions: int = 0 + total_deletions: int = 0 + total_changed_files: int = 0 + + def calculate_stats(self) -> None: + """Calculate aggregated statistics.""" + self.total_commits = len(self.commits) + self.total_prs = len(self.pull_requests) + self.merged_prs = sum(1 for pr in self.pull_requests if pr.state == "MERGED") + self.total_additions = sum(c.additions for c in self.commits) + sum( + pr.additions for pr in self.pull_requests + ) + self.total_deletions = sum(c.deletions for c in self.commits) + sum( + pr.deletions for pr in self.pull_requests + ) + self.total_changed_files = sum(c.changed_files for c in self.commits) + + +@dataclass +class PeriodReport: + """Report for a specific time period (weekly/monthly/quarterly/yearly).""" + + start_date: str + end_date: str + repos: list[str] + period: ReportPeriod = ReportPeriod.WEEKLY + contributors: list[ContributorSummary] = field(default_factory=list) + generated_at: str = "" + llm_summary: str = "" + + # Overall stats + total_commits: int = 0 + total_prs: int = 0 + total_merged_prs: int = 0 + + def calculate_overall_stats(self) -> None: + """Calculate overall statistics.""" + self.total_commits = sum(c.total_commits for c in self.contributors) + self.total_prs = sum(c.total_prs for c in self.contributors) + self.total_merged_prs = sum(c.merged_prs for c in self.contributors) + self.generated_at = datetime.now().isoformat() + + +# Alias for backward compatibility +WeeklyReport = PeriodReport diff --git a/sage-apps/src/sage/apps/work_report_generator/operators.py b/sage-apps/src/sage/apps/work_report_generator/operators.py new file mode 100644 index 0000000..831419c --- /dev/null +++ b/sage-apps/src/sage/apps/work_report_generator/operators.py @@ -0,0 +1,1507 @@ +""" +SAGE Operators for Work Report Generator. + +Implements source, map, and sink operators for the work report pipeline. +""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import requests + +from sage.common.core import BatchFunction, MapFunction, SinkFunction + +from .models import ( + ContributorSummary, + DiaryEntry, + GitHubCommit, + GitHubPullRequest, + ReportPeriod, + WeeklyReport, +) + + +def calculate_period_dates( + period: ReportPeriod, reference_date: datetime | None = None +) -> tuple[datetime, datetime]: + """ + Calculate start and end dates for a given report period. + + Args: + period: The report period type (weekly, monthly, quarterly, yearly). + reference_date: The reference date to calculate from. Defaults to now. + + Returns: + Tuple of (start_date, end_date) for the period. + """ + if reference_date is None: + reference_date = datetime.now() + + if period == ReportPeriod.WEEKLY: + # Last 7 days + end_date = reference_date + start_date = end_date - timedelta(days=7) + elif period == ReportPeriod.MONTHLY: + # Last calendar month or last 30 days + end_date = reference_date + # Go to first day of current month, then back one day to get last month + first_of_month = reference_date.replace(day=1) + start_date = (first_of_month - timedelta(days=1)).replace(day=1) + end_date = first_of_month - timedelta(days=1) + # If we're past the 7th, use last 30 days instead for more recent data + if reference_date.day > 7: + end_date = reference_date + start_date = end_date - timedelta(days=30) + elif period == ReportPeriod.QUARTERLY: + # Last quarter (3 months / ~90 days) + end_date = reference_date + start_date = end_date - timedelta(days=90) + elif period == ReportPeriod.YEARLY: + # Last year (365 days) + end_date = reference_date + start_date = end_date - timedelta(days=365) + else: + # Default to weekly + end_date = reference_date + start_date = end_date - timedelta(days=7) + + return start_date, end_date + + +def get_period_days(period: ReportPeriod) -> int: + """Get the number of days for a report period.""" + days_map = { + ReportPeriod.WEEKLY: 7, + ReportPeriod.MONTHLY: 30, + ReportPeriod.QUARTERLY: 90, + ReportPeriod.YEARLY: 365, + } + return days_map.get(period, 7) + + +class GitHubDataSource(BatchFunction): + """ + BatchFunction source that fetches commits and PRs from GitHub. + + Uses GitHub GraphQL API to fetch contribution data for specified repositories. + Supports fetching from specific branches (e.g., main-dev) and submodule repos. + """ + + # SAGE main repository and all submodule repositories + SAGE_REPOS = [ + "intellistream/SAGE", + "intellistream/SAGE-Pub", # docs-public submodule + "intellistream/sageData", # benchmark data submodule + "intellistream/sageLLM", # sageLLM submodule + "intellistream/LibAMM", # libamm submodule + "intellistream/sageVDB", # sageVDB submodule + "intellistream/sageFlow", # sageFlow submodule + "intellistream/neuromem", # neuromem submodule + "intellistream/sageTSDB", # sageTSDB submodule + ] + + def __init__( + self, + repos: list[str] | None = None, + days: int | None = None, + period: ReportPeriod | str = ReportPeriod.WEEKLY, + branch: str = "main-dev", + github_token: str | None = None, + include_submodules: bool = True, + **kwargs, + ) -> None: + """ + Initialize GitHub data source. + + Args: + repos: List of repositories in "owner/repo" format. + If None, defaults to SAGE_REPOS (main + submodules). + days: Number of days to look back for contributions (overrides period). + period: Report period type (weekly/monthly/quarterly/yearly). + branch: Branch name to fetch commits from. Default "main-dev". + github_token: GitHub personal access token. If not provided, + will try GITHUB_TOKEN or GIT_TOKEN environment variables. + include_submodules: If True and repos is None, include all SAGE submodules. + """ + super().__init__(**kwargs) + + # Use SAGE repos by default if none specified + if repos is None: + self.repos = self.SAGE_REPOS if include_submodules else ["intellistream/SAGE"] + else: + self.repos = repos + + # Handle period parameter + if isinstance(period, str): + period = ReportPeriod(period.lower()) + self.period = period + + # Days can override period + if days is not None: + self.days = days + else: + self.days = get_period_days(period) + + self.branch = branch + self.github_token = github_token or os.environ.get( + "GITHUB_TOKEN", os.environ.get("GIT_TOKEN", "") + ) + self.graphql_url = "https://api.github.com/graphql" + self.headers = { + "Authorization": f"Bearer {self.github_token}", + "Content-Type": "application/json", + } + + # Data storage + self.commits: list[GitHubCommit] = [] + self.pull_requests: list[GitHubPullRequest] = [] + self.current_index = 0 + self.fetched = False + + # Calculate date range based on period or days + self.start_date, self.end_date = calculate_period_dates(period) + + def _execute_graphql( + self, query: str, variables: dict[str, Any] | None = None + ) -> dict[str, Any]: + """Execute a GraphQL query.""" + payload = {"query": query} + if variables: + payload["variables"] = variables + + try: + response = requests.post( + self.graphql_url, + json=payload, + headers=self.headers, + timeout=30, + ) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + self.logger.error(f"GraphQL request failed: {e}") + return {"errors": [str(e)]} + + def _fetch_commits(self, owner: str, repo: str) -> list[GitHubCommit]: + """Fetch commits for a repository within the date range from specific branch.""" + commits = [] + since = self.start_date.isoformat() + + # Query that supports fetching from a specific branch (ref) + # Also fetches associated PR info to determine who assigned Copilot + query = """ + query($owner: String!, $repo: String!, $branch: String!, $since: GitTimestamp!, $after: String) { + repository(owner: $owner, name: $repo) { + ref(qualifiedName: $branch) { + target { + ... on Commit { + history(since: $since, first: 100, after: $after) { + pageInfo { + hasNextPage + endCursor + } + nodes { + oid + message + committedDate + url + additions + deletions + changedFiles + author { + name + email + user { + login + } + } + committer { + name + email + user { + login + } + } + associatedPullRequests(first: 1) { + nodes { + author { + login + } + assignees(first: 5) { + nodes { + login + } + } + } + } + } + } + } + } + } + } + } + """ + + after = None + repo_full = f"{owner}/{repo}" + + while True: + variables = { + "owner": owner, + "repo": repo, + "branch": f"refs/heads/{self.branch}", + "since": since, + "after": after, + } + result = self._execute_graphql(query, variables) + + if "errors" in result: + self.logger.error( + f"Error fetching commits from {repo_full}@{self.branch}: {result['errors']}" + ) + # Try fallback to default branch + fallback_commits = self._fetch_commits_default_branch(owner, repo) + return fallback_commits + + data = result.get("data", {}).get("repository", {}).get("ref", {}) + + if not data: + self.logger.warning( + f"Branch '{self.branch}' not found in {repo_full}. Trying default branch..." + ) + return self._fetch_commits_default_branch(owner, repo) + + history = data.get("target", {}).get("history", {}) + nodes = history.get("nodes", []) + + for node in nodes: + commit = GitHubCommit.from_graphql(node, repo_full) + commits.append(commit) + + page_info = history.get("pageInfo", {}) + if not page_info.get("hasNextPage"): + break + after = page_info.get("endCursor") + time.sleep(0.2) # Rate limiting + + return commits + + def _fetch_commits_default_branch(self, owner: str, repo: str) -> list[GitHubCommit]: + """Fallback: Fetch commits from the default branch if specified branch not found.""" + commits = [] + since = self.start_date.isoformat() + + query = """ + query($owner: String!, $repo: String!, $since: GitTimestamp!, $after: String) { + repository(owner: $owner, name: $repo) { + defaultBranchRef { + name + target { + ... on Commit { + history(since: $since, first: 100, after: $after) { + pageInfo { + hasNextPage + endCursor + } + nodes { + oid + message + committedDate + url + additions + deletions + changedFiles + author { + name + email + user { + login + } + } + } + } + } + } + } + } + } + """ + + after = None + repo_full = f"{owner}/{repo}" + branch_name = None + + while True: + variables = { + "owner": owner, + "repo": repo, + "since": since, + "after": after, + } + result = self._execute_graphql(query, variables) + + if "errors" in result: + self.logger.error(f"Error fetching commits: {result['errors']}") + break + + data = result.get("data", {}).get("repository", {}).get("defaultBranchRef", {}) + + if not branch_name and data: + branch_name = data.get("name", "unknown") + self.logger.info(f" Using default branch: {branch_name}") + + history = data.get("target", {}).get("history", {}) + nodes = history.get("nodes", []) + + for node in nodes: + commit = GitHubCommit.from_graphql(node, repo_full) + commits.append(commit) + + page_info = history.get("pageInfo", {}) + if not page_info.get("hasNextPage"): + break + after = page_info.get("endCursor") + time.sleep(0.2) # Rate limiting + + return commits + + def _fetch_pull_requests(self, owner: str, repo: str) -> list[GitHubPullRequest]: + """Fetch pull requests for a repository within the date range.""" + pull_requests = [] + + query = """ + query($owner: String!, $repo: String!, $after: String) { + repository(owner: $owner, name: $repo) { + pullRequests(first: 100, after: $after, orderBy: {field: CREATED_AT, direction: DESC}) { + pageInfo { + hasNextPage + endCursor + } + nodes { + number + title + state + createdAt + mergedAt + closedAt + url + additions + deletions + changedFiles + author { + login + } + assignees(first: 5) { + nodes { + login + } + } + labels(first: 10) { + nodes { + name + } + } + reviewRequests(first: 10) { + nodes { + requestedReviewer { + ... on User { + login + } + } + } + } + } + } + } + } + """ + + after = None + repo_full = f"{owner}/{repo}" + + while True: + variables = { + "owner": owner, + "repo": repo, + "after": after, + } + result = self._execute_graphql(query, variables) + + if "errors" in result: + self.logger.error(f"Error fetching PRs: {result['errors']}") + break + + data = result.get("data", {}).get("repository", {}).get("pullRequests", {}) + + nodes = data.get("nodes", []) + for node in nodes: + # Filter by date range + created_at = node.get("createdAt", "") + if created_at: + created_date = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if created_date.replace(tzinfo=None) < self.start_date: + # PRs are sorted by created_at DESC, so we can stop + break + + pr = GitHubPullRequest.from_graphql(node, repo_full) + pull_requests.append(pr) + + page_info = data.get("pageInfo", {}) + if not page_info.get("hasNextPage"): + break + + # Check if we've gone past our date range + if nodes: + last_created = nodes[-1].get("createdAt", "") + if last_created: + last_date = datetime.fromisoformat(last_created.replace("Z", "+00:00")) + if last_date.replace(tzinfo=None) < self.start_date: + break + + after = page_info.get("endCursor") + time.sleep(0.2) # Rate limiting + + return pull_requests + + def _fetch_all_data(self) -> None: + """Fetch all commits and PRs from all repositories.""" + if not self.github_token: + self.logger.warning("No GitHub token provided. Using mock data.") + self._use_mock_data() + return + + self.logger.info(f"Target branch: {self.branch}") + self.logger.info(f"Repositories to fetch: {len(self.repos)}") + + for repo in self.repos: + parts = repo.split("/") + if len(parts) != 2: + self.logger.warning(f"Invalid repo format: {repo}") + continue + + owner, repo_name = parts + self.logger.info(f"Fetching data from {repo} @ {self.branch}...") + + # Fetch commits from specified branch + commits = self._fetch_commits(owner, repo_name) + self.commits.extend(commits) + self.logger.info(f" Found {len(commits)} commits") + + # Fetch PRs (PRs are not branch-specific in the same way) + prs = self._fetch_pull_requests(owner, repo_name) + self.pull_requests.extend(prs) + self.logger.info(f" Found {len(prs)} pull requests") + + def _use_mock_data(self) -> None: + """Generate mock data for testing.""" + mock_commits = [ + { + "oid": "abc123", + "message": "feat: Add weekly report generator", + "committedDate": datetime.now().isoformat(), + "url": "https://github.com/intellistream/SAGE/commit/abc123", + "additions": 500, + "deletions": 100, + "changedFiles": 10, + "author": { + "name": "Developer One", + "email": "dev1@example.com", + "user": {"login": "dev1"}, + }, + }, + { + "oid": "def456", + "message": "fix: Resolve pipeline issue", + "committedDate": (datetime.now() - timedelta(days=1)).isoformat(), + "url": "https://github.com/intellistream/SAGE/commit/def456", + "additions": 50, + "deletions": 30, + "changedFiles": 3, + "author": { + "name": "Developer Two", + "email": "dev2@example.com", + "user": {"login": "dev2"}, + }, + }, + ] + + mock_prs = [ + { + "number": 100, + "title": "feat: Weekly report generator implementation", + "state": "MERGED", + "createdAt": datetime.now().isoformat(), + "mergedAt": datetime.now().isoformat(), + "closedAt": None, + "url": "https://github.com/intellistream/SAGE/pull/100", + "additions": 1000, + "deletions": 200, + "changedFiles": 15, + "author": {"login": "dev1"}, + "labels": {"nodes": [{"name": "feature"}]}, + "reviewRequests": {"nodes": []}, + }, + ] + + for commit_data in mock_commits: + self.commits.append(GitHubCommit.from_graphql(commit_data, "intellistream/SAGE")) + + for pr_data in mock_prs: + self.pull_requests.append(GitHubPullRequest.from_graphql(pr_data, "intellistream/SAGE")) + + def execute(self) -> dict[str, Any] | None: + """Execute batch function to emit data packets.""" + if not self.fetched: + self.logger.info(f"Fetching GitHub data for past {self.days} days...") + self._fetch_all_data() + self.fetched = True + self.logger.info(f"Fetched {len(self.commits)} commits, {len(self.pull_requests)} PRs") + + # Emit commits first, then PRs + total_items = len(self.commits) + len(self.pull_requests) + if self.current_index < len(self.commits): + commit = self.commits[self.current_index] + self.current_index += 1 + return { + "type": "commit", + "data": commit, + "repo": commit.repo, + "author": commit.author, + } + elif self.current_index < total_items: + pr_index = self.current_index - len(self.commits) + pr = self.pull_requests[pr_index] + self.current_index += 1 + return { + "type": "pull_request", + "data": pr, + "repo": pr.repo, + "author": pr.author, + } + + return None + + +class DiaryEntrySource(BatchFunction): + """ + BatchFunction source that loads diary entries from files. + + Supports JSON and Markdown formats. + """ + + def __init__( + self, + diary_path: str | Path | None = None, + days: int | None = None, + period: ReportPeriod | str = ReportPeriod.WEEKLY, + **kwargs, + ) -> None: + """ + Initialize diary entry source. + + Args: + diary_path: Path to diary directory or file. + days: Number of days to look back (overrides period). + period: Report period type (weekly/monthly/quarterly/yearly). + """ + super().__init__(**kwargs) + self.diary_path = Path(diary_path) if diary_path else None + + # Handle period parameter + if isinstance(period, str): + period = ReportPeriod(period.lower()) + self.period = period + + # Days can override period + if days is not None: + self.days = days + else: + self.days = get_period_days(period) + + self.entries: list[DiaryEntry] = [] + self.current_index = 0 + self.fetched = False + + # Calculate date range based on period + self.start_date, self.end_date = calculate_period_dates(period) + + def _load_entries(self) -> None: + """Load diary entries from the specified path.""" + if not self.diary_path or not self.diary_path.exists(): + self.logger.info("No diary path specified or path doesn't exist") + return + + if self.diary_path.is_file(): + self._load_file(self.diary_path) + elif self.diary_path.is_dir(): + # Load all JSON and MD files + for file_path in self.diary_path.glob("**/*.json"): + self._load_file(file_path) + for file_path in self.diary_path.glob("**/*.md"): + self._load_markdown_file(file_path) + + def _load_file(self, file_path: Path) -> None: + """Load entries from a JSON file.""" + try: + with open(file_path, encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, list): + for item in data: + entry = DiaryEntry.from_dict(item) + if self._is_in_date_range(entry.date): + self.entries.append(entry) + elif isinstance(data, dict): + entry = DiaryEntry.from_dict(data) + if self._is_in_date_range(entry.date): + self.entries.append(entry) + + except Exception as e: + self.logger.warning(f"Error loading {file_path}: {e}") + + def _load_markdown_file(self, file_path: Path) -> None: + """Load a single markdown file as a diary entry.""" + try: + content = file_path.read_text(encoding="utf-8") + + # Extract date from filename (e.g., 2024-01-15.md) + date_str = file_path.stem + try: + datetime.strptime(date_str, "%Y-%m-%d") + except ValueError: + date_str = datetime.now().strftime("%Y-%m-%d") + + if self._is_in_date_range(date_str): + entry = DiaryEntry( + date=date_str, + author="Unknown", + content=content, + category="diary", + ) + self.entries.append(entry) + + except Exception as e: + self.logger.warning(f"Error loading markdown {file_path}: {e}") + + def _is_in_date_range(self, date_str: str) -> bool: + """Check if a date string is within the configured range.""" + try: + date = datetime.strptime(date_str[:10], "%Y-%m-%d") + return self.start_date <= date <= self.end_date + except ValueError: + return True # Include if can't parse + + def execute(self) -> dict[str, Any] | None: + """Execute batch function to emit diary entries.""" + if not self.fetched: + self._load_entries() + self.fetched = True + self.logger.info(f"Loaded {len(self.entries)} diary entries") + + if self.current_index < len(self.entries): + entry = self.entries[self.current_index] + self.current_index += 1 + return { + "type": "diary_entry", + "data": entry, + "author": entry.author, + } + + return None + + +class ContributorAggregator(MapFunction): + """ + MapFunction that aggregates data by contributor. + + Collects commits, PRs, and diary entries for each contributor. + """ + + # Class-level storage for aggregation across all instances + _contributors: dict[str, ContributorSummary] = {} + + # Username aliases: map alternative names to canonical names + # Key: lowercase alias, Value: canonical name + USERNAME_ALIASES: dict[str, str] = { + # ShuhaoZhangTony variants + "shuhao zhang": "ShuhaoZhangTony", + "shuhaozhangtony": "ShuhaoZhangTony", + "shuhaozhang": "ShuhaoZhangTony", + # Copilot variants - these will be reassigned to their assigners + "copilot": "Copilot", + "copilot-swe-agent": "Copilot", + "github-copilot": "Copilot", + } + + # Copilot usernames (lowercase) + COPILOT_USERS: set[str] = {"copilot", "copilot-swe-agent", "github-copilot"} + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + + @classmethod + def normalize_username(cls, username: str) -> str: + """Normalize username to canonical form.""" + if not username: + return "Unknown" + # Check if lowercase version matches any alias + lower_name = username.lower().strip() + return cls.USERNAME_ALIASES.get(lower_name, username) + + @classmethod + def is_copilot_user(cls, username: str) -> bool: + """Check if the username is a Copilot variant.""" + return username.lower().strip() in cls.COPILOT_USERS + + def _resolve_author( + self, raw_author: str, item: GitHubCommit | GitHubPullRequest | DiaryEntry | None + ) -> tuple[str, bool]: + """ + Resolve the final author for contribution attribution. + + Returns: + tuple: (final_author, is_copilot_assisted) + """ + normalized = self.normalize_username(raw_author) + + # Check if this is a Copilot commit with an assigned user + if isinstance(item, GitHubCommit) and self.is_copilot_user(raw_author): + if item.assigned_by: + # Attribute to the assigning user, but mark as Copilot-assisted + final_author = self.normalize_username(item.assigned_by) + return final_author, True + + # Check if this is a Copilot PR with an assigned user + if isinstance(item, GitHubPullRequest) and self.is_copilot_user(raw_author): + if item.assigned_by: + # Attribute to the assigning user, but mark as Copilot-assisted + final_author = self.normalize_username(item.assigned_by) + return final_author, True + + return normalized, False + + def execute(self, data: dict[str, Any]) -> dict[str, Any] | None: + """Aggregate data by contributor.""" + # Skip non-dict data (e.g., StopSignal) + if not isinstance(data, dict): + return None + + raw_author = data.get("author", "Unknown") + data_type = data.get("type") + item = data.get("data") + + # Resolve author (handle Copilot attribution) + author, is_copilot_assisted = self._resolve_author(raw_author, item) + + if author not in ContributorAggregator._contributors: + ContributorAggregator._contributors[author] = ContributorSummary(username=author) + + contributor = ContributorAggregator._contributors[author] + + if data_type == "commit" and isinstance(item, GitHubCommit): + contributor.commits.append(item) + if is_copilot_assisted: + # Track Copilot-assisted commits separately + if not hasattr(contributor, "copilot_assisted_commits"): + contributor.copilot_assisted_commits = 0 + contributor.copilot_assisted_commits += 1 + elif data_type == "pull_request" and isinstance(item, GitHubPullRequest): + contributor.pull_requests.append(item) + if is_copilot_assisted: + # Track Copilot-assisted PRs separately + if not hasattr(contributor, "copilot_assisted_prs"): + contributor.copilot_assisted_prs = 0 + contributor.copilot_assisted_prs += 1 + elif data_type == "diary_entry" and isinstance(item, DiaryEntry): + contributor.diary_entries.append(item) + + # Return the updated contributor summary + contributor.calculate_stats() + return { + "type": "contributor_update", + "author": author, + "contributor": contributor, + } + + @classmethod + def get_all_contributors(cls) -> dict[str, ContributorSummary]: + """Get all aggregated contributors.""" + return cls._contributors + + @classmethod + def reset(cls) -> None: + """Reset the aggregator state.""" + cls._contributors = {} + + +class LLMReportGenerator(MapFunction): + """ + MapFunction that generates LLM-powered summaries for contributors. + + Uses UnifiedInferenceClient to generate natural language summaries. + """ + + def __init__( + self, + language: str = "zh", + **kwargs, + ) -> None: + """ + Initialize LLM report generator. + + Args: + language: Output language ("zh" for Chinese, "en" for English). + """ + super().__init__(**kwargs) + self.language = language + self._client = None + self._processed_authors: set[str] = set() + + def _get_client(self): + """Lazy initialization of LLM client.""" + if self._client is None: + try: + from sage.llm import UnifiedInferenceClient + + self._client = UnifiedInferenceClient.create() + except Exception as e: + self.logger.warning(f"Failed to initialize LLM client: {e}") + self._client = None + return self._client + + def _generate_summary(self, contributor: ContributorSummary) -> str: + """Generate a summary for a contributor using LLM.""" + client = self._get_client() + if client is None: + return self._generate_simple_summary(contributor) + + # Build context for LLM + commits_text = "\n".join( + f"- {c.message} ({c.additions}+ / {c.deletions}-)" + for c in contributor.commits[:10] # Limit to recent 10 + ) + + prs_text = "\n".join(f"- [{pr.state}] {pr.title}" for pr in contributor.pull_requests[:5]) + + diary_text = "\n".join( + f"- [{e.date}] {e.content[:100]}..." for e in contributor.diary_entries[:3] + ) + + if self.language == "zh": + prompt = f"""请为以下贡献者生成一份简洁的周报总结： + +贡献者：{contributor.username} + +本周提交记录： +{commits_text or "无提交记录"} + +Pull Requests： +{prs_text or "无PR记录"} + +个人日志： +{diary_text or "无日志"} + +统计数据： +- 提交次数：{contributor.total_commits} +- PR数量：{contributor.total_prs}（已合并：{contributor.merged_prs}） +- 代码变更：+{contributor.total_additions} / -{contributor.total_deletions} + +请生成一份100-200字的工作总结，突出主要贡献和成果。""" + else: + prompt = f"""Generate a brief weekly report summary for the following contributor: + +Contributor: {contributor.username} + +Commits this week: +{commits_text or "No commits"} + +Pull Requests: +{prs_text or "No PRs"} + +Personal notes: +{diary_text or "No notes"} + +Statistics: +- Commits: {contributor.total_commits} +- PRs: {contributor.total_prs} (Merged: {contributor.merged_prs}) +- Lines changed: +{contributor.total_additions} / -{contributor.total_deletions} + +Generate a 100-200 word work summary highlighting key contributions.""" + + try: + messages = [{"role": "user", "content": prompt}] + response = client.chat(messages) + if hasattr(response, "content"): + return response.content + return str(response) + except Exception as e: + self.logger.warning(f"LLM generation failed: {e}") + return self._generate_simple_summary(contributor) + + def _generate_simple_summary(self, contributor: ContributorSummary) -> str: + """Generate a simple summary without LLM.""" + if self.language == "zh": + return ( + f"{contributor.username} 本周贡献了 {contributor.total_commits} 次提交，" + f"提交了 {contributor.total_prs} 个PR（已合并 {contributor.merged_prs} 个），" + f"代码变更 +{contributor.total_additions}/-{contributor.total_deletions} 行。" + ) + else: + return ( + f"{contributor.username} contributed {contributor.total_commits} commits, " + f"submitted {contributor.total_prs} PRs ({contributor.merged_prs} merged), " + f"with +{contributor.total_additions}/-{contributor.total_deletions} lines changed." + ) + + def execute(self, data: dict[str, Any]) -> dict[str, Any] | None: + """Generate summary for contributor updates.""" + if data.get("type") != "contributor_update": + return data + + author = data.get("author", "") + contributor = data.get("contributor") + + # Only generate summary once per author + if author in self._processed_authors: + return None + + self._processed_authors.add(author) + + if contributor: + summary = self._generate_summary(contributor) + data["llm_summary"] = summary + + return data + + +class ReportSink(SinkFunction): + """ + SinkFunction that collects and outputs the final report. + + Supports multiple output formats: console, markdown, JSON. + """ + + def __init__( + self, + output_format: str = "markdown", + output_path: str | Path | None = None, + repos: list[str] | None = None, + days: int | None = None, + period: ReportPeriod | str = ReportPeriod.WEEKLY, + language: str = "zh", + use_llm: bool = True, + **kwargs, + ) -> None: + """ + Initialize report sink. + + Args: + output_format: Output format ("console", "markdown", "json"). + output_path: Path to save the report file. + repos: List of repositories being tracked. + days: Number of days in the report period (overrides period). + period: Report period type (weekly/monthly/quarterly/yearly). + language: Output language ("zh" or "en"). + use_llm: Whether to use LLM for leaderboard generation. + """ + super().__init__(**kwargs) + self.output_format = output_format + self.output_path = Path(output_path) if output_path else None + self.repos = repos or [] + + # Handle period parameter + if isinstance(period, str): + period = ReportPeriod(period.lower()) + self.period = period + + # Days can override period + if days is not None: + self.days = days + else: + self.days = get_period_days(period) + + self.language = language + self.use_llm = use_llm + self.contributors: dict[str, dict[str, Any]] = {} + self.start_time = time.time() + self._llm_client = None + + # Date range based on period + self.start_date, self.end_date = calculate_period_dates(period) + + def _get_llm_client(self): + """Lazy initialization of LLM client.""" + if self._llm_client is None: + try: + from sage.llm import UnifiedInferenceClient + + self._llm_client = UnifiedInferenceClient.create() + except Exception as e: + self.logger.warning(f"Failed to initialize LLM client: {e}") + self._llm_client = None + return self._llm_client + + def _get_period_text(self) -> dict[str, str]: + """Get period-specific text for report generation.""" + period_texts = { + ReportPeriod.WEEKLY: { + "zh_title": "本周贡献排行榜", + "en_title": "Weekly Contribution Leaderboard", + "zh_period": "本周", + "en_period": "this week", + "zh_summary_type": "周报", + "en_summary_type": "weekly report", + }, + ReportPeriod.MONTHLY: { + "zh_title": "本月贡献排行榜", + "en_title": "Monthly Contribution Leaderboard", + "zh_period": "本月", + "en_period": "this month", + "zh_summary_type": "月报", + "en_summary_type": "monthly report", + }, + ReportPeriod.QUARTERLY: { + "zh_title": "本季度贡献排行榜", + "en_title": "Quarterly Contribution Leaderboard", + "zh_period": "本季度", + "en_period": "this quarter", + "zh_summary_type": "季报", + "en_summary_type": "quarterly report", + }, + ReportPeriod.YEARLY: { + "zh_title": "年度贡献排行榜", + "en_title": "Yearly Contribution Leaderboard", + "zh_period": "本年度", + "en_period": "this year", + "zh_summary_type": "年报", + "en_summary_type": "yearly report", + }, + } + return period_texts.get(self.period, period_texts[ReportPeriod.WEEKLY]) + + def _generate_leaderboard(self, report: WeeklyReport) -> str: + """Generate a contribution leaderboard with visual chart and optional LLM comments.""" + # Sort contributors by contribution score + sorted_contributors = sorted( + report.contributors, + key=lambda c: c.total_commits + c.merged_prs * 3, + reverse=True, + ) + + # Always generate visual chart first + chart = self._generate_leaderboard_chart(sorted_contributors) + + if not self.use_llm: + return self._generate_simple_leaderboard(report) + + client = self._get_llm_client() + if client is None: + return self._generate_simple_leaderboard(report) + + # Build contributor stats for LLM + contributors_data = [] + for c in report.contributors: + contributors_data.append( + { + "name": c.username, + "commits": c.total_commits, + "prs": c.total_prs, + "merged_prs": c.merged_prs, + "additions": c.total_additions, + "deletions": c.total_deletions, + } + ) + + # Sort by contribution (commits + merged PRs * 3) + contributors_data.sort(key=lambda x: x["commits"] + x["merged_prs"] * 3, reverse=True) + + stats_text = "\n".join( + f"- {i + 1}. {c['name']}: {c['commits']} commits, " + f"{c['prs']} PRs ({c['merged_prs']} merged), " + f"+{c['additions']}/-{c['deletions']} lines" + for i, c in enumerate(contributors_data) + ) + + # Get period-specific text + period_text = self._get_period_text() + + if self.language == "zh": + prompt = f"""根据以下贡献者数据，生成一份简洁有趣的{period_text["zh_period"]}贡献度排行榜评语。 +请用emoji和简短评语让排行榜更生动。 + +贡献者数据（按贡献度排序）： +{stats_text} + +时间范围：{report.start_date} ~ {report.end_date} +总提交数：{report.total_commits} +总PR数：{report.total_prs}（已合并：{report.total_merged_prs}） + +要求： +1. 用🥇🥈🥉标注前三名 +2. 每个人配一句简短有趣的评语（10-20字） +3. 最后加一句团队总结或鼓励语 +4. 保持简洁，总共不超过300字""" + else: + prompt = f"""Based on the contributor data below, generate brief and fun {period_text["en_period"]} contribution leaderboard comments. +Use emojis and short comments to make it engaging. + +Contributor data (sorted by contribution): +{stats_text} + +Period: {report.start_date} ~ {report.end_date} +Total commits: {report.total_commits} +Total PRs: {report.total_prs} (merged: {report.total_merged_prs}) + +Requirements: +1. Use 🥇🥈🥉 for top 3 +2. Add a short fun comment for each person (10-20 words) +3. End with a team summary or encouragement +4. Keep it concise, under 300 words total""" + + try: + messages = [{"role": "user", "content": prompt}] + response = client.chat(messages) + llm_comments = response.content if hasattr(response, "content") else str(response) + + # Combine chart with LLM comments + title = ( + f"## 🏆 {period_text['zh_title']}" + if self.language == "zh" + else f"## 🏆 {period_text['en_title']}" + ) + return f"{title}\n\n{chart}\n\n{llm_comments}" + except Exception as e: + self.logger.warning(f"LLM leaderboard generation failed: {e}") + return self._generate_simple_leaderboard(report) + + def _generate_simple_leaderboard(self, report: WeeklyReport) -> str: + """Generate a simple leaderboard without LLM.""" + # Sort contributors by contribution score + sorted_contributors = sorted( + report.contributors, + key=lambda c: c.total_commits + c.merged_prs * 3, + reverse=True, + ) + + # Generate visual bar chart + chart = self._generate_leaderboard_chart(sorted_contributors) + + # Get period-specific text + period_text = self._get_period_text() + + if self.language == "zh": + lines = [f"## 🏆 {period_text['zh_title']}", ""] + lines.append(chart) + lines.append("") + medals = ["🥇", "🥈", "🥉"] + for i, c in enumerate(sorted_contributors): + medal = medals[i] if i < 3 else f"{i + 1}." + lines.append( + f"{medal} **{c.username}** - " + f"{c.total_commits} commits, {c.merged_prs} merged PRs" + ) + lines.append("") + lines.append( + f"*{period_text['zh_period']}团队共完成 {report.total_commits} 次提交，" + f"合并 {report.total_merged_prs} 个PR，继续加油。*" + ) + else: + lines = [f"## 🏆 {period_text['en_title']}", ""] + lines.append(chart) + lines.append("") + medals = ["🥇", "🥈", "🥉"] + for i, c in enumerate(sorted_contributors): + medal = medals[i] if i < 3 else f"{i + 1}." + lines.append( + f"{medal} **{c.username}** - " + f"{c.total_commits} commits, {c.merged_prs} merged PRs" + ) + lines.append("") + lines.append( + f"*Team completed {report.total_commits} commits and " + f"merged {report.total_merged_prs} PRs {period_text['en_period']}. Keep it up.*" + ) + + return "\n".join(lines) + + def _generate_leaderboard_chart(self, sorted_contributors: list) -> str: + """Generate an ASCII/Unicode bar chart for the leaderboard.""" + if not sorted_contributors: + return "" + + # Calculate contribution scores + scores = [] + for c in sorted_contributors: + score = c.total_commits + c.merged_prs * 3 + scores.append((c.username, score, c.total_commits, c.merged_prs)) + + max_score = max(s[1] for s in scores) if scores else 1 + max_name_len = max(len(s[0]) for s in scores) if scores else 10 + max_name_len = min(max_name_len, 20) # Cap at 20 chars + + # Bar characters + bar_full = "█" + bar_width = 30 + + lines = ["```"] + lines.append(f"{'Contributor':<{max_name_len}} │ {'Score':>6} │ Progress") + lines.append(f"{'─' * max_name_len}─┼{'─' * 8}┼{'─' * (bar_width + 10)}") + + medals = ["🥇", "🥈", "🥉"] + + for i, (name, score, commits, prs) in enumerate(scores): + # Truncate long names + display_name = name[:max_name_len] if len(name) > max_name_len else name + + # Calculate bar length + bar_len = int((score / max_score) * bar_width) if max_score > 0 else 0 + bar = bar_full * bar_len + + # Medal for top 3 + medal = medals[i] if i < 3 else " " + + # Color-like indicators using different bar styles + lines.append( + f"{display_name:<{max_name_len}} │ {score:>6} │ {bar} {medal} ({commits}c+{prs}pr)" + ) + + lines.append("```") + lines.append("") + lines.append("*Score = commits + merged_prs × 3*") + + return "\n".join(lines) + + def execute(self, data: dict[str, Any]) -> None: + """Collect data for the report.""" + if data.get("type") == "contributor_update": + author = data.get("author", "Unknown") + self.contributors[author] = { + "contributor": data.get("contributor"), + "llm_summary": data.get("llm_summary", ""), + } + + def close(self) -> None: + """Generate and output the final report.""" + elapsed = time.time() - self.start_time + + # Build report + report = WeeklyReport( + start_date=self.start_date.strftime("%Y-%m-%d"), + end_date=self.end_date.strftime("%Y-%m-%d"), + repos=self.repos, + ) + + for author, data in self.contributors.items(): + contributor = data.get("contributor") + if contributor: + report.contributors.append(contributor) + + report.calculate_overall_stats() + + # Output based on format + if self.output_format == "console": + self._output_console(report, elapsed) + elif self.output_format == "markdown": + self._output_markdown(report, elapsed) + elif self.output_format == "json": + self._output_json(report, elapsed) + + def _output_console(self, report: WeeklyReport, elapsed: float) -> None: + """Output report to console.""" + print("\n" + "=" * 70) + print("Weekly Work Report / 周报") + print("=" * 70) + print(f"Period: {report.start_date} ~ {report.end_date}") + print(f"Repositories: {', '.join(report.repos)}") + print("=" * 70) + + print( + f"\n Total: {report.total_commits} commits, " + f"{report.total_prs} PRs ({report.total_merged_prs} merged)" + ) + print("-" * 70) + + for contributor in report.contributors: + print(f"\n {contributor.username}") + print(f" Commits: {contributor.total_commits}") + print(f" PRs: {contributor.total_prs} (merged: {contributor.merged_prs})") + print(f" Lines: +{contributor.total_additions} / -{contributor.total_deletions}") + + # Recent commits + if contributor.commits: + print(" Recent commits:") + for commit in contributor.commits[:3]: + print(f" - {commit.message[:60]}") + + # LLM Summary + llm_summary = self.contributors.get(contributor.username, {}).get("llm_summary", "") + if llm_summary: + print(f" Summary: {llm_summary[:200]}...") + + # Generate and print leaderboard + print("\n" + "-" * 70) + leaderboard = self._generate_leaderboard(report) + print(leaderboard) + + print("\n" + "=" * 70) + print(f" Generated in {elapsed:.2f}s") + print("=" * 70 + "\n") + + def _output_markdown(self, report: WeeklyReport, elapsed: float) -> None: + """Output report as Markdown.""" + lines = [ + "# Weekly Work Report", + "", + f"**Period:** {report.start_date} ~ {report.end_date}", + f"**Repositories:** {', '.join(report.repos)}", + f"**Generated:** {report.generated_at}", + "", + "## Summary", + "", + f"- Total Commits: {report.total_commits}", + f"- Total PRs: {report.total_prs} ({report.total_merged_prs} merged)", + "", + "## Contributors", + "", + ] + + for contributor in report.contributors: + lines.extend( + [ + f"### {contributor.username}", + "", + f"- **Commits:** {contributor.total_commits}", + f"- **PRs:** {contributor.total_prs} (merged: {contributor.merged_prs})", + f"- **Lines Changed:** +{contributor.total_additions} / -{contributor.total_deletions}", + "", + ] + ) + + if contributor.commits: + lines.append("**Recent Commits:**") + for commit in contributor.commits[:5]: + lines.append(f"- [{commit.sha[:7]}]({commit.url}) {commit.message[:60]}") + lines.append("") + + if contributor.pull_requests: + lines.append("**Pull Requests:**") + for pr in contributor.pull_requests[:5]: + status = "" if pr.state == "MERGED" else f"[{pr.state}] " + lines.append(f"- {status}[#{pr.number}]({pr.url}) {pr.title}") + lines.append("") + + llm_summary = self.contributors.get(contributor.username, {}).get("llm_summary", "") + if llm_summary: + lines.extend( + [ + "**AI Summary:**", + "", + f"> {llm_summary}", + "", + ] + ) + + # Add leaderboard section + leaderboard = self._generate_leaderboard(report) + lines.extend( + [ + "", + leaderboard, + "", + "---", + f"*Report generated in {elapsed:.2f}s by SAGE Work Report Generator*", + ] + ) + + content = "\n".join(lines) + + if self.output_path: + self.output_path.parent.mkdir(parents=True, exist_ok=True) + self.output_path.write_text(content, encoding="utf-8") + print(f" Report saved to: {self.output_path}") + else: + print(content) + + def _output_json(self, report: WeeklyReport, elapsed: float) -> None: + """Output report as JSON.""" + data = { + "start_date": report.start_date, + "end_date": report.end_date, + "repos": report.repos, + "generated_at": report.generated_at, + "summary": { + "total_commits": report.total_commits, + "total_prs": report.total_prs, + "total_merged_prs": report.total_merged_prs, + }, + "contributors": [], + } + + for contributor in report.contributors: + contrib_data = { + "username": contributor.username, + "stats": { + "commits": contributor.total_commits, + "prs": contributor.total_prs, + "merged_prs": contributor.merged_prs, + "additions": contributor.total_additions, + "deletions": contributor.total_deletions, + }, + "commits": [ + { + "sha": c.sha, + "message": c.message, + "url": c.url, + "date": c.committed_date, + } + for c in contributor.commits + ], + "pull_requests": [ + { + "number": pr.number, + "title": pr.title, + "state": pr.state, + "url": pr.url, + } + for pr in contributor.pull_requests + ], + "llm_summary": self.contributors.get(contributor.username, {}).get( + "llm_summary", "" + ), + } + data["contributors"].append(contrib_data) + + # Add leaderboard + leaderboard = self._generate_leaderboard(report) + data["leaderboard"] = leaderboard + + content = json.dumps(data, indent=2, ensure_ascii=False) + + if self.output_path: + self.output_path.parent.mkdir(parents=True, exist_ok=True) + self.output_path.write_text(content, encoding="utf-8") + print(f" Report saved to: {self.output_path}") + else: + print(content) + + +class ConsoleSink(SinkFunction): + """Simple sink that logs each item processed.""" + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) + self.count = 0 + + def execute(self, data: dict[str, Any]) -> None: + """Log processing progress.""" + self.count += 1 + data_type = data.get("type", "unknown") + author = data.get("author", "Unknown") + self.logger.info(f"[{self.count}] Processed {data_type} from {author}") diff --git a/sage-apps/src/sage/apps/work_report_generator/pipeline.py b/sage-apps/src/sage/apps/work_report_generator/pipeline.py new file mode 100644 index 0000000..3b03bc2 --- /dev/null +++ b/sage-apps/src/sage/apps/work_report_generator/pipeline.py @@ -0,0 +1,359 @@ +""" +Work Report Generator Pipeline + +Main pipeline implementation using SAGE operators for weekly/monthly/quarterly/yearly report generation. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from pathlib import Path + +from sage.kernel.api.local_environment import LocalEnvironment + +from .models import ReportPeriod +from .operators import ( + ConsoleSink, + ContributorAggregator, + DiaryEntrySource, + GitHubDataSource, + LLMReportGenerator, + ReportSink, + calculate_period_dates, + get_period_days, +) + + +def run_work_report_pipeline( + repos: list[str] | None = None, + days: int | None = None, + period: ReportPeriod | str = ReportPeriod.WEEKLY, + branch: str = "main-dev", + output_format: str = "markdown", + output_path: str | Path | None = None, + diary_path: str | Path | None = None, + language: str = "zh", + github_token: str | None = None, + verbose: bool = False, + use_llm: bool = True, + include_submodules: bool = True, +) -> str | None: + """ + Run the work report generation pipeline using SAGE framework. + + This pipeline: + 1. Fetches commits and PRs from GitHub repositories (main + submodules) + 2. Optionally loads diary entries from local files + 3. Aggregates data by contributor + 4. Generates LLM-powered summaries (optional) + 5. Outputs the final report + + Args: + repos: List of GitHub repositories in "owner/repo" format. + If None, defaults to SAGE main repo + all submodules. + days: Number of days to look back for contributions (overrides period). + period: Report period type (weekly/monthly/quarterly/yearly). + branch: Branch name to fetch commits from. Default "main-dev". + output_format: Output format ("console", "markdown", "json"). + output_path: Path to save the report file. If None, outputs to console. + diary_path: Path to diary directory or file for personal notes. + language: Report language ("zh" for Chinese, "en" for English). + github_token: GitHub personal access token. Uses env var if not provided. + verbose: Enable verbose logging. + use_llm: Whether to use LLM for generating summaries. + include_submodules: If True and repos is None, include all SAGE submodules. + + Returns: + Path to the generated report file, or None if outputting to console. + + Example: + >>> run_work_report_pipeline( + ... period="monthly", + ... branch="main-dev", + ... output_format="markdown", + ... output_path="reports/monthly_report.md" + ... ) + """ + # Handle period parameter + if isinstance(period, str): + period = ReportPeriod(period.lower()) + + # Calculate date range based on period or days + if days is not None: + end_date = datetime.now() + start_date = end_date - timedelta(days=days) + actual_days = days + else: + start_date, end_date = calculate_period_dates(period) + actual_days = get_period_days(period) + + # Get repo list for display + display_repos = ( + repos + if repos + else GitHubDataSource.SAGE_REPOS + if include_submodules + else ["intellistream/SAGE"] + ) + + # Print header + print("=" * 70) + print(f" SAGE Work Report Generator - {period.english_name}") + print("=" * 70) + print(f"Period: {start_date.strftime('%Y-%m-%d')} ~ {end_date.strftime('%Y-%m-%d')}") + print(f"Report Type: {period.display_name} ({period.value})") + print(f"Target Branch: {branch}") + print(f"Repositories ({len(display_repos)}):") + for repo in display_repos: + print(f" - {repo}") + print(f"Output Format: {output_format}") + print(f"Language: {'Chinese' if language == 'zh' else 'English'}") + print(f"LLM Summary: {'Enabled' if use_llm else 'Disabled'}") + print("=" * 70) + print() + + # Reset aggregator state (important for multiple pipeline runs) + ContributorAggregator.reset() + + # Create SAGE environment + env = LocalEnvironment("work_report_generator") + + # Determine output path + if output_path is None and output_format != "console": + output_dir = Path(".sage/reports") + output_dir.mkdir(parents=True, exist_ok=True) + ext = "json" if output_format == "json" else "md" + # Use period name in filename + output_path = output_dir / f"{period.value}_report_{end_date.strftime('%Y%m%d')}.{ext}" + + # Build pipeline: + # 1. GitHubDataSource: Fetch commits and PRs from main-dev branch + # 2. ContributorAggregator: Group by contributor + # 3. LLMReportGenerator: Generate AI summaries (optional) + # 4. ReportSink: Output final report + + pipeline = env.from_batch( + GitHubDataSource, + repos=repos, + days=actual_days, + period=period, + branch=branch, + github_token=github_token, + include_submodules=include_submodules, + ).map(ContributorAggregator) + + # Optionally add LLM summary generation + if use_llm: + pipeline = pipeline.map(LLMReportGenerator, language=language) + + # Add report sink + pipeline = pipeline.sink( + ReportSink, + output_format=output_format, + output_path=output_path, + repos=repos, + days=actual_days, + period=period, + language=language, + use_llm=use_llm, + ) + + # Optionally add verbose logging + if verbose: + pipeline.sink(ConsoleSink) + + # Handle diary entries if provided + if diary_path: + diary_pipeline = env.from_batch( + DiaryEntrySource, + diary_path=diary_path, + days=actual_days, + period=period, + ).map(ContributorAggregator) + + if use_llm: + diary_pipeline = diary_pipeline.map(LLMReportGenerator, language=language) + + diary_pipeline.sink( + ReportSink, + output_format=output_format, + output_path=output_path, + repos=repos, + days=actual_days, + period=period, + ) + + # Execute pipeline + print(" Starting pipeline...") + print() + env.submit(autostop=True) + + return str(output_path) if output_path else None + + +def main(): + """Main entry point for command-line usage.""" + import argparse + + parser = argparse.ArgumentParser( + description="SAGE Work Report Generator - Generate periodic work reports", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate weekly report for all SAGE repositories (default) + python -m sage.apps.work_report_generator.pipeline + + # Generate monthly report + python -m sage.apps.work_report_generator.pipeline --period monthly + + # Generate quarterly report + python -m sage.apps.work_report_generator.pipeline --period quarterly + + # Generate yearly report + python -m sage.apps.work_report_generator.pipeline --period yearly + + # Specify custom repositories + python -m sage.apps.work_report_generator.pipeline \\ + --repos intellistream/SAGE,intellistream/sageLLM + + # Use different branch + python -m sage.apps.work_report_generator.pipeline --branch main + + # Main repo only (without submodules) + python -m sage.apps.work_report_generator.pipeline --no-submodules + + # Custom time range (overrides period) + python -m sage.apps.work_report_generator.pipeline \\ + --days 14 \\ + --output reports/biweekly.md \\ + --format markdown + + # With diary entries + python -m sage.apps.work_report_generator.pipeline \\ + --diary-path ./diaries \\ + --language zh + + # Skip LLM for faster generation + python -m sage.apps.work_report_generator.pipeline --no-llm + """, + ) + + parser.add_argument( + "--period", + "-p", + type=str, + choices=["weekly", "monthly", "quarterly", "yearly"], + default="weekly", + help="Report period type (default: weekly)", + ) + + parser.add_argument( + "--repos", + "-r", + type=str, + help="Comma-separated list of repositories (owner/repo format). " + "If not specified, fetches from all SAGE repos including submodules.", + ) + + parser.add_argument( + "--branch", + "-b", + type=str, + default="main-dev", + help="Branch name to fetch commits from (default: main-dev)", + ) + + parser.add_argument( + "--days", + "-d", + type=int, + default=None, + help="Number of days to look back (overrides --period)", + ) + + parser.add_argument( + "--format", + "-f", + type=str, + choices=["console", "markdown", "json"], + default="markdown", + help="Output format (default: markdown)", + ) + + parser.add_argument( + "--output", + "-o", + type=str, + help="Output file path", + ) + + parser.add_argument( + "--diary-path", + type=str, + help="Path to diary directory or file", + ) + + parser.add_argument( + "--language", + "-l", + type=str, + choices=["zh", "en"], + default="zh", + help="Report language (default: zh)", + ) + + parser.add_argument( + "--token", + "-t", + type=str, + help="GitHub personal access token", + ) + + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Enable verbose output", + ) + + parser.add_argument( + "--no-llm", + action="store_true", + help="Disable LLM summary generation", + ) + + parser.add_argument( + "--no-submodules", + action="store_true", + help="Only fetch from main SAGE repo, skip submodules", + ) + + args = parser.parse_args() + + # Parse repositories + repos = None + if args.repos: + repos = [r.strip() for r in args.repos.split(",")] + + # Run pipeline + result = run_work_report_pipeline( + repos=repos, + days=args.days, + period=args.period, + branch=args.branch, + output_format=args.format, + output_path=args.output, + diary_path=args.diary_path, + language=args.language, + github_token=args.token, + verbose=args.verbose, + use_llm=not args.no_llm, + include_submodules=not args.no_submodules, + ) + + if result: + print(f"\n Report generated: {result}") + + +if __name__ == "__main__": + main() diff --git a/sage-apps/tests/__init__.py b/sage-apps/tests/__init__.py new file mode 100644 index 0000000..ca8b5e0 --- /dev/null +++ b/sage-apps/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for sage-apps package.""" diff --git a/sage-apps/tests/article_monitoring/__init__.py b/sage-apps/tests/article_monitoring/__init__.py new file mode 100644 index 0000000..8ce0db1 --- /dev/null +++ b/sage-apps/tests/article_monitoring/__init__.py @@ -0,0 +1 @@ +"""Tests for Article Monitoring application""" diff --git a/sage-apps/tests/article_monitoring/test_operators.py b/sage-apps/tests/article_monitoring/test_operators.py new file mode 100644 index 0000000..27533ad --- /dev/null +++ b/sage-apps/tests/article_monitoring/test_operators.py @@ -0,0 +1,322 @@ +"""Unit tests for Article Monitoring operators + +Tests for sage.apps.article_monitoring.operators module +""" + +import pytest + +from sage.apps.article_monitoring.operators import ( + Article, + ArticleLogSink, + ArticleRankingSink, + ArticleScorer, + ArxivSource, + KeywordFilter, + SemanticFilter, +) + + +class TestArticle: + """Test Article dataclass""" + + def test_article_creation(self): + """Test creating an article""" + article = Article( + id="2401.12345", + title="Test Article", + authors=["Author One", "Author Two"], + abstract="This is a test abstract about machine learning.", + published="2024-01-15", + categories=["cs.AI", "cs.LG"], + url="https://arxiv.org/abs/2401.12345", + ) + + assert article.id == "2401.12345" + assert article.title == "Test Article" + assert len(article.authors) == 2 + assert article.keyword_score == 0.0 + assert article.semantic_score == 0.0 + assert article.total_score == 0.0 + + +class TestArxivSource: + """Test ArxivSource operator""" + + def test_source_creation(self): + """Test creating arXiv source""" + source = ArxivSource(category="cs.AI", max_results=10) + + assert source.category == "cs.AI" + assert source.max_results == 10 + assert source.index == 0 + assert source.fetched is False + + def test_source_generates_mock_articles(self): + """Test source generates mock articles when API unavailable""" + source = ArxivSource(category="cs.AI", max_results=3) + + # Force using mock data + mock_articles = source._get_mock_articles() + + assert len(mock_articles) == 5 # Default mock data has 5 articles + assert all("id" in article for article in mock_articles) + assert all("title" in article for article in mock_articles) + + def test_source_execute_returns_articles_one_by_one(self): + """Test source executes and returns one article at a time""" + source = ArxivSource(category="cs.AI", max_results=5) + + # Manually set mock articles to test iteration + source.articles = source._get_mock_articles() + source.fetched = True + + # Get first article + article1 = source.execute() + assert article1 is not None + assert "id" in article1 + + # Get second article + article2 = source.execute() + assert article2 is not None + assert article2["id"] != article1["id"] + + def test_source_parse_xml_with_missing_elements(self): + """Test XML parsing handles missing elements gracefully""" + source = ArxivSource(category="cs.AI", max_results=10) + + # Create malformed XML with missing required elements + malformed_xml = """ + + + http://arxiv.org/abs/2401.00001 + Valid Article + This has all required fields + 2024-01-01T00:00:00Z + Test Author + + + + http://arxiv.org/abs/2401.00002 + + Incomplete Article + + + """ + + try: + articles = source._parse_arxiv_response(malformed_xml) + # Should only parse the valid entry, skip malformed ones + assert len(articles) >= 1 + if len(articles) > 0: + assert articles[0]["id"] == "2401.00001" + except Exception: + # XML parsing might fail entirely, which is also acceptable + pass + + def test_source_parse_xml_with_missing_author_name(self): + """Test XML parsing handles authors without names""" + source = ArxivSource(category="cs.AI", max_results=10) + + xml_with_authors = """ + + + http://arxiv.org/abs/2401.00001 + Test Article + Test summary + 2024-01-01T00:00:00Z + Valid Author + + + + """ + + try: + articles = source._parse_arxiv_response(xml_with_authors) + # Should have at least one article + assert len(articles) >= 1 + if len(articles) > 0 and "authors" in articles[0]: + # Should have authors + assert len(articles[0]["authors"]) >= 1 + except Exception: + # XML parsing might fail, acceptable + pass + + +class TestKeywordFilter: + """Test KeywordFilter operator""" + + def test_filter_creation(self): + """Test creating keyword filter""" + keywords = ["machine learning", "deep learning", "AI"] + filter_op = KeywordFilter(keywords=keywords, min_score=2.0) + + assert len(filter_op.keywords) == 3 + assert filter_op.min_score == 2.0 + + def test_filter_article_high_match(self): + """Test filtering article with high keyword match""" + filter_op = KeywordFilter(keywords=["stream", "processing", "data"], min_score=2.0) + + article_data = { + "id": "2401.00001", + "title": "Real-time Stream Processing", + "abstract": "We present a novel approach for processing data streams...", + "authors": ["John Doe"], + } + + result = filter_op.execute(article_data) + assert result is not None + assert result["keyword_score"] >= 2.0 + + def test_filter_article_low_match(self): + """Test filtering article with low keyword match""" + filter_op = KeywordFilter(keywords=["quantum", "computing"], min_score=2.0) + + article_data = { + "id": "2401.00002", + "title": "Machine Learning Basics", + "abstract": "Introduction to machine learning concepts...", + "authors": ["Jane Smith"], + } + + result = filter_op.execute(article_data) + assert result is None + + +class TestSemanticFilter: + """Test SemanticFilter operator""" + + def test_filter_creation(self): + """Test creating semantic filter""" + topics = ["distributed systems", "stream processing"] + filter_op = SemanticFilter(interest_topics=topics, min_similarity=0.3) + + assert len(filter_op.interest_topics) == 2 + assert filter_op.min_similarity == 0.3 + + def test_filter_semantic_match(self): + """Test semantic filtering with matching content""" + filter_op = SemanticFilter( + interest_topics=["machine learning neural networks"], min_similarity=0.05 + ) + + article_data = { + "id": "2401.00001", + "title": "Deep Neural Networks for Classification", + "abstract": "We explore deep learning and neural network architectures...", + "keyword_score": 2.0, + } + + result = filter_op.execute(article_data) + assert result is not None + assert "semantic_score" in result + assert result["semantic_score"] > 0 + + +class TestArticleScorer: + """Test ArticleScorer operator""" + + def test_scorer_calculates_total_score(self): + """Test scorer calculates total score""" + scorer = ArticleScorer() + + article_data = { + "id": "2401.00001", + "title": "Test Article", + "keyword_score": 3.0, + "semantic_score": 2.5, + } + + result = scorer.execute(article_data) + assert result["total_score"] == 5.5 + + +class TestArticleRankingSink: + """Test ArticleRankingSink operator""" + + def test_sink_creation(self): + """Test creating ranking sink""" + sink = ArticleRankingSink() + + assert len(sink.articles) == 0 + + def test_sink_collects_articles(self): + """Test sink collects articles""" + sink = ArticleRankingSink() + + article1 = { + "id": "2401.00001", + "title": "Article 1", + "total_score": 5.0, + "authors": ["Author 1"], + "url": "http://test.com/1", + } + + article2 = { + "id": "2401.00002", + "title": "Article 2", + "total_score": 3.0, + "authors": ["Author 2"], + "url": "http://test.com/2", + } + + sink.execute(article1) + sink.execute(article2) + + assert len(sink.articles) == 2 + + +class TestArticleLogSink: + """Test ArticleLogSink operator""" + + def test_sink_logs_articles(self): + """Test sink logs articles""" + sink = ArticleLogSink() + + article = { + "id": "2401.00001", + "title": "Test Article for Logging", + "total_score": 4.5, + } + + sink.execute(article) + assert sink.count == 1 + + +@pytest.mark.integration +class TestArticleMonitoringPipeline: + """Integration tests for article monitoring pipeline""" + + def test_end_to_end_flow(self): + """Test complete article monitoring flow""" + # Create operators + source = ArxivSource(category="cs.AI", max_results=3) + keyword_filter = KeywordFilter(keywords=["learning", "neural"], min_score=1.0) + semantic_filter = SemanticFilter(interest_topics=["machine learning"], min_similarity=0.03) + scorer = ArticleScorer() + ranking_sink = ArticleRankingSink() + + # Execute pipeline manually + processed_count = 0 + while True: + article = source.execute() + if article is None: + break + + # Apply filters + filtered = keyword_filter.execute(article) + if filtered is None: + continue + + filtered = semantic_filter.execute(filtered) + if filtered is None: + continue + + # Score and collect + scored = scorer.execute(filtered) + ranking_sink.execute(scored) + processed_count += 1 + + # Should have processed at least some articles + assert processed_count >= 0 + assert len(ranking_sink.articles) >= 0 diff --git a/sage-apps/tests/auto_scaling_chat/__init__.py b/sage-apps/tests/auto_scaling_chat/__init__.py new file mode 100644 index 0000000..0b9528b --- /dev/null +++ b/sage-apps/tests/auto_scaling_chat/__init__.py @@ -0,0 +1 @@ +"""Tests for Auto-Scaling Chat application""" diff --git a/sage-apps/tests/auto_scaling_chat/test_operators.py b/sage-apps/tests/auto_scaling_chat/test_operators.py new file mode 100644 index 0000000..43fdeda --- /dev/null +++ b/sage-apps/tests/auto_scaling_chat/test_operators.py @@ -0,0 +1,294 @@ +"""Unit tests for Auto-Scaling Chat operators + +Tests for sage.apps.auto_scaling_chat.operators module +""" + +import pytest + +from sage.apps.auto_scaling_chat.operators import ( + AutoScaler, + LoadBalancer, + MetricsCollector, + RequestProcessor, + ScalingEventsSink, + UserRequest, + UserTrafficSource, +) + + +class TestUserRequest: + """Test UserRequest dataclass""" + + def test_request_creation(self): + """Test creating a user request""" + request = UserRequest( + request_id=1, + user_id=101, + timestamp="2024-01-15T10:00:00", + message="Hello, how are you?", + ) + + assert request.request_id == 1 + assert request.user_id == 101 + assert request.message == "Hello, how are you?" + assert request.processing_time == 0.0 + assert request.server_id is None + + +class TestUserTrafficSource: + """Test UserTrafficSource operator""" + + def test_source_creation(self): + """Test creating user traffic source""" + source = UserTrafficSource(duration=10, base_rate=5, peak_rate=20) + + assert source.duration == 10 + assert source.base_rate == 5 + assert source.peak_rate == 20 + assert source.request_count == 0 + + def test_source_generates_requests(self): + """Test source generates user requests""" + source = UserTrafficSource(duration=1, base_rate=50, peak_rate=100) + + # Execute a few times + requests = [] + for _ in range(10): + result = source.execute() + if result is not None: + requests.append(result) + + # Should generate at least some requests with high rate + assert len(requests) >= 0 + + def test_source_respects_duration(self): + """Test source stops after duration""" + source = UserTrafficSource(duration=0.1, base_rate=10, peak_rate=20) + + import time + + time.sleep(0.15) # Wait past duration + + result = source.execute() + assert result is None # Should return None after duration + + +class TestLoadBalancer: + """Test LoadBalancer operator""" + + def test_balancer_creation(self): + """Test creating load balancer""" + balancer = LoadBalancer(initial_servers=3) + + assert balancer.num_servers == 3 + assert balancer.current_server == 0 + + def test_balancer_round_robin(self): + """Test round-robin load balancing""" + balancer = LoadBalancer(initial_servers=3) + + request1 = {"request_id": 1, "message": "test1"} + request2 = {"request_id": 2, "message": "test2"} + request3 = {"request_id": 3, "message": "test3"} + request4 = {"request_id": 4, "message": "test4"} + + result1 = balancer.execute(request1) + result2 = balancer.execute(request2) + result3 = balancer.execute(request3) + result4 = balancer.execute(request4) + + # Should distribute across servers 0, 1, 2, 0 + assert result1["server_id"] == 0 + assert result2["server_id"] == 1 + assert result3["server_id"] == 2 + assert result4["server_id"] == 0 + + def test_balancer_scale_servers(self): + """Test scaling number of servers""" + balancer = LoadBalancer(initial_servers=2) + + balancer.scale_servers(5) + assert balancer.num_servers == 5 + + +class TestAutoScaler: + """Test AutoScaler operator""" + + def test_scaler_creation(self): + """Test creating auto scaler""" + scaler = AutoScaler( + scale_up_threshold=30, scale_down_threshold=10, min_servers=2, max_servers=10 + ) + + assert scaler.scale_up_threshold == 30 + assert scaler.scale_down_threshold == 10 + assert scaler.min_servers == 2 + assert scaler.max_servers == 10 + assert scaler.current_servers == 2 + + def test_scaler_scale_up_decision(self): + """Test scale up decision""" + scaler = AutoScaler(scale_up_threshold=20, min_servers=2, max_servers=10) + + # High load should trigger scale up + high_load_data = {"current_load": 50, "request_id": 1} + + result = scaler.execute(high_load_data) + + # Should scale up + assert result["scaling_action"] in ["scale_up", "cooldown"] + if result["scaling_action"] == "scale_up": + assert result["total_servers"] == 3 + + def test_scaler_respects_cooldown(self): + """Test cooldown period""" + scaler = AutoScaler(scale_up_threshold=20, cooldown=2.0) + + # First scale up + data1 = {"current_load": 50} + scaler.execute(data1) + + # Immediate second attempt should be in cooldown + data2 = {"current_load": 50} + result2 = scaler.execute(data2) + + assert result2["scaling_action"] == "cooldown" + + +class TestRequestProcessor: + """Test RequestProcessor operator""" + + def test_processor_creation(self): + """Test creating request processor""" + processor = RequestProcessor(processing_time=0.1) + + assert processor.processing_time == 0.1 + + def test_processor_processes_request(self): + """Test processing a request""" + processor = RequestProcessor(processing_time=0.01) + + request_data = { + "request_id": 1, + "user_id": 123, + "message": "Hello", + "server_id": 0, + } + + result = processor.execute(request_data) + + assert result is not None + assert "processing_time" in result + assert "processed_at" in result + assert result["request_id"] == 1 + + +class TestMetricsCollector: + """Test MetricsCollector operator""" + + def test_collector_creation(self): + """Test creating metrics collector""" + collector = MetricsCollector() + + assert collector.total_requests == 0 + assert collector.scaling_events == 0 + assert collector.peak_servers == 0 + assert collector.peak_load == 0 + + def test_collector_collects_metrics(self): + """Test collecting metrics""" + collector = MetricsCollector() + + data1 = {"current_load": 10, "total_servers": 2, "scaling_action": "none"} + data2 = {"current_load": 30, "total_servers": 3, "scaling_action": "scale_up"} + data3 = {"current_load": 15, "total_servers": 3, "scaling_action": "none"} + + collector.execute(data1) + collector.execute(data2) + collector.execute(data3) + + assert collector.total_requests == 3 + assert collector.peak_load == 30 + assert collector.peak_servers == 3 + assert collector.scaling_events == 1 + + +class TestScalingEventsSink: + """Test ScalingEventsSink operator""" + + def test_sink_creation(self): + """Test creating scaling events sink""" + sink = ScalingEventsSink() + + assert len(sink.events) == 0 + + def test_sink_logs_scaling_events(self): + """Test logging scaling events""" + sink = ScalingEventsSink() + + # Non-scaling event should not be logged + data1 = {"scaling_action": "none", "total_servers": 2} + sink.execute(data1) + + assert len(sink.events) == 0 + + # Scaling events should be logged + data2 = { + "scaling_action": "scale_up", + "total_servers": 3, + "current_load": 50, + "timestamp": "2024-01-15T10:00:00", + } + sink.execute(data2) + + assert len(sink.events) == 1 + assert sink.events[0]["action"] == "scale_up" + assert sink.events[0]["servers"] == 3 + + +@pytest.mark.integration +class TestAutoScalingPipeline: + """Integration tests for auto-scaling chat system""" + + def test_end_to_end_flow(self): + """Test complete auto-scaling flow""" + # Create operators + source = UserTrafficSource(duration=2, base_rate=10, peak_rate=30) + balancer = LoadBalancer(initial_servers=2) + scaler = AutoScaler(scale_up_threshold=20, scale_down_threshold=5, min_servers=2) + processor = RequestProcessor(processing_time=0.01) + metrics = MetricsCollector() + events_sink = ScalingEventsSink() + + # Execute pipeline manually + processed = 0 + max_iterations = 100 # Safety limit + + for _ in range(max_iterations): + # Generate traffic + request_data = source.execute() + if request_data is None: + break + + # Load balance + balanced = balancer.execute(request_data) + + # Make scaling decision + scaled = scaler.execute(balanced) + + # Update balancer if scaled + if scaled["scaling_action"] in ["scale_up", "scale_down"]: + balancer.scale_servers(scaled["total_servers"]) + + # Process request + processed_request = processor.execute(scaled) + + # Collect metrics + metrics.execute(processed_request) + events_sink.execute(processed_request) + + processed += 1 + + # Verify execution + assert processed >= 0 + assert metrics.total_requests == processed diff --git a/sage-apps/tests/medical_diagnosis/__init__.py b/sage-apps/tests/medical_diagnosis/__init__.py new file mode 100644 index 0000000..abfd540 --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/__init__.py @@ -0,0 +1 @@ +"""Medical diagnosis application tests.""" diff --git a/sage-apps/tests/medical_diagnosis/conftest.py b/sage-apps/tests/medical_diagnosis/conftest.py new file mode 100644 index 0000000..421bbe6 --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/conftest.py @@ -0,0 +1,72 @@ +"""pytest configuration for medical diagnosis tests.""" + +import json +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def prepare_minimal_test_data(): + """ + Prepare minimal test data for medical diagnosis tests. + + This fixture automatically runs before any tests in this directory. + It creates a minimal test dataset if the real data is not available. + """ + # Get the medical diagnosis app directory + test_dir = Path(__file__).parent + app_dir = test_dir.parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" + data_dir = app_dir / "data" / "processed" + test_index_path = data_dir / "test_index.json" + + # If real data exists, skip creating mock data + if test_index_path.exists(): + print(f"✓ Using existing test data at {test_index_path}") + return + + # Create minimal mock test data + print(f"✓ Creating minimal test data at {test_index_path}") + data_dir.mkdir(parents=True, exist_ok=True) + + # Create minimal test cases (no actual images needed for structure tests) + mock_test_cases = [ + { + "case_id": "mock_001", + "patient_id": "P001", + "age": 45, + "gender": "男", + "disease": "正常", + "severity": "无", + "image_path": "mock_images/case_001.npy", + }, + { + "case_id": "mock_002", + "patient_id": "P002", + "age": 52, + "gender": "女", + "disease": "椎间盘突出", + "severity": "中度", + "image_path": "mock_images/case_002.npy", + }, + ] + + # Save test index + with open(test_index_path, "w", encoding="utf-8") as f: + json.dump(mock_test_cases, f, ensure_ascii=False, indent=2) + + # Create mock image directory + mock_image_dir = data_dir / "mock_images" + mock_image_dir.mkdir(parents=True, exist_ok=True) + + # Create placeholder image files (empty numpy arrays) + import numpy as np + + for case in mock_test_cases: + image_path = data_dir / case["image_path"] + if not image_path.exists(): + # Create a minimal numpy array as placeholder + mock_image = np.zeros((64, 64), dtype=np.float32) + np.save(image_path, mock_image) + + print(f"✓ Created {len(mock_test_cases)} mock test cases") diff --git a/sage-apps/tests/medical_diagnosis/demo_knowledge_loading.py b/sage-apps/tests/medical_diagnosis/demo_knowledge_loading.py new file mode 100644 index 0000000..701cd2b --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/demo_knowledge_loading.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +演示医学知识库的知识加载功能 +Demonstrate the knowledge loading functionality of the MedicalKnowledgeBase +""" + +import json +import sys +from pathlib import Path + +# Add the source directory to the path +src_dir = Path(__file__).parent.parent.parent / "src" +sys.path.insert(0, str(src_dir)) + +from sage.apps.medical_diagnosis.tools.knowledge_base import MedicalKnowledgeBase + + +def create_sample_data(): + """创建示例数据文件用于演示""" + print("=" * 80) + print("创建示例数据文件...") + print("=" * 80) + + # 创建临时数据目录 + data_dir = Path(__file__).parent / "test_data" + data_dir.mkdir(exist_ok=True) + + # 创建报告目录 + reports_dir = data_dir / "reports" + reports_dir.mkdir(exist_ok=True) + + # 创建示例报告 + sample_reports = [ + { + "filename": "case_0001_report.txt", + "content": """患者信息: + 年龄: 48岁 + 性别: 男 + 主诉: 腰痛伴左下肢放射痛 + +影像描述: + 腰椎MRI T2加权矢状位: L4/L5椎间盘向后突出，压迫硬膜囊。相应节段椎管变窄，神经根可能受压。 + +主要发现: + - 病变节段: L4/L5 + - 病变类型: 椎间盘突出 + - 严重程度: 中度 + +诊断结论: + L4/L5椎间盘突出症，程度中度。 + +治疗建议: + 建议卧床休息2-3周，牵引治疗。口服非甾体抗炎药及神经营养药物。保守治疗无效时考虑手术治疗。 + +注: 本报告仅供参考，请结合临床症状和其他检查结果综合判断。 +""", + }, + { + "filename": "case_0002_report.txt", + "content": """患者信息: + 年龄: 60岁 + 性别: 男 + 主诉: 慢性腰痛 + +影像描述: + 腰椎MRI T2加权矢状位: L3/L4, L4/L5, L5/S1多节段退行性变。椎管尚通畅，未见明显神经根受压。 + +主要发现: + - 病变节段: 多节段 + - 病变类型: 多节段退行性变 + - 严重程度: 中度 + +诊断结论: + 腰椎退行性变，L3/L4、L4/L5椎间盘突出，程度中度。 + +治疗建议: + 适当休息，避免久坐久站。可进行腰背肌锻炼，如游泳、普拉提等。必要时物理治疗。 + +注: 本报告仅供参考，请结合临床症状和其他检查结果综合判断。 +""", + }, + ] + + for report in sample_reports: + report_path = reports_dir / report["filename"] + with open(report_path, "w", encoding="utf-8") as f: + f.write(report["content"]) + print(f" ✓ 创建报告: {report['filename']}") + + # 创建统计文件 + stats = { + "total_samples": 100, + "train_samples": 80, + "test_samples": 20, + "disease_distribution": { + "正常": 10, + "轻度退行性变": 20, + "椎间盘突出": 25, + "多节段退行性变": 15, + "椎管狭窄": 12, + "椎间盘脱出": 8, + "骨质增生": 10, + }, + } + + stats_path = data_dir / "stats.json" + with open(stats_path, "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + print(" ✓ 创建统计文件: stats.json") + + # 创建病例数据库 + cases = [ + { + "case_id": "case_0001", + "patient_id": "P0001", + "age": 48, + "gender": "男", + "disease": "椎间盘突出", + "severity": "中度", + "image_path": "images/case_0001.jpg", + "report_path": "reports/case_0001_report.txt", + }, + { + "case_id": "case_0002", + "patient_id": "P0002", + "age": 60, + "gender": "男", + "disease": "多节段退行性变", + "severity": "中度", + "image_path": "images/case_0002.jpg", + "report_path": "reports/case_0002_report.txt", + }, + ] + + all_cases_path = data_dir / "all_cases.json" + with open(all_cases_path, "w", encoding="utf-8") as f: + json.dump(cases, f, ensure_ascii=False, indent=2) + print(" ✓ 创建病例数据库: all_cases.json") + + print(f"\n✓ 示例数据创建完成，保存在: {data_dir.absolute()}") + print() + + return str(data_dir) + + +def test_knowledge_loading(): + """测试知识加载功能""" + print("=" * 80) + print("测试医学知识库 - 不使用数据集") + print("=" * 80) + print() + + # 测试1：不使用数据集 + print("1️⃣ 测试基本初始化（仅使用默认知识）") + print("-" * 80) + config = {} + kb = MedicalKnowledgeBase(config) + + print("\n📊 知识库统计:") + print(f" - 知识条目总数: {len(kb.knowledge_base)}") + print(f" - 病例数据库大小: {len(kb.case_database)}") + + print("\n📚 默认知识主题:") + for i, knowledge in enumerate(kb.knowledge_base[:5], 1): + topic = knowledge.get("topic", "Unknown") + source = knowledge.get("source", "default") + print(f" {i}. {topic} (来源: {source})") + + print("\n" + "=" * 80) + print() + + # 测试2：使用示例数据 + print("2️⃣ 测试从数据集加载知识") + print("-" * 80) + + # 创建示例数据 + data_path = create_sample_data() + + config_with_data = {"data_path": data_path} + kb_with_data = MedicalKnowledgeBase(config_with_data) + + print("\n📊 知识库统计（使用数据集）:") + print(f" - 知识条目总数: {len(kb_with_data.knowledge_base)}") + print(f" - 病例数据库大小: {len(kb_with_data.case_database)}") + + print("\n📚 知识主题:") + for i, knowledge in enumerate(kb_with_data.knowledge_base, 1): + topic = knowledge.get("topic", "Unknown") + source = knowledge.get("source", "default") + case_count = knowledge.get("case_count", "N/A") + if case_count != "N/A": + print(f" {i}. {topic} (来源: {source}, 病例数: {case_count})") + else: + print(f" {i}. {topic} (来源: {source})") + + print("\n" + "=" * 80) + print() + + # 测试3：知识检索 + print("3️⃣ 测试知识检索功能") + print("-" * 80) + + queries = ["腰椎间盘突出", "退行性变", "椎管狭窄"] + + for query in queries: + results = kb_with_data.retrieve_knowledge(query, top_k=2) + print(f"\n🔍 查询: '{query}'") + print(f" 找到 {len(results)} 条相关知识:") + for i, result in enumerate(results, 1): + topic = result.get("topic", "Unknown") + print(f" {i}. {topic}") + + print("\n" + "=" * 80) + print() + + # 测试4：相似病例检索 + print("4️⃣ 测试相似病例检索") + print("-" * 80) + + queries_cases = ["椎间盘突出", "退行性变", "腰痛"] + + for query in queries_cases: + results = kb_with_data.retrieve_similar_cases(query, {}, top_k=3) + print(f"\n🔍 查询: '{query}'") + print(f" 找到 {len(results)} 个相似病例:") + for i, case in enumerate(results, 1): + case_id = case.get("case_id", "Unknown") + diagnosis = case.get("diagnosis", "Unknown") + score = case.get("similarity_score", 0.0) + print(f" {i}. {case_id}: {diagnosis} (相似度: {score:.2f})") + + print("\n" + "=" * 80) + print() + + # 清理测试数据 + print("🧹 清理测试数据...") + import shutil + + test_data_dir = Path(__file__).parent / "test_data" + if test_data_dir.exists(): + shutil.rmtree(test_data_dir) + print(" ✓ 测试数据已删除") + + print("\n" + "=" * 80) + print("✅ 测试完成!") + print("=" * 80) + + +if __name__ == "__main__": + test_knowledge_loading() diff --git a/sage-apps/tests/medical_diagnosis/test_diagnosis.py b/sage-apps/tests/medical_diagnosis/test_diagnosis.py new file mode 100644 index 0000000..0deeebc --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/test_diagnosis.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +""" +医疗诊断系统简单测试 +使用预处理好的数据测试诊断流程（不需要真实的SAGE服务） +""" + +import json +import os +import random +from pathlib import Path + +import pytest + +# 获取医疗诊断目录路径（用于访问数据和配置文件） +medical_diagnosis_dir = ( + Path(__file__).parent.parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" +) +# 导入医疗诊断模块 +from sage.apps.medical_diagnosis.agents.diagnostic_agent import DiagnosticAgent # noqa: E402 + + +@pytest.mark.skipif( + os.getenv("SAGE_ENABLE_MEDICAL_DIAGNOSIS") not in {"1", "true", "True"}, + reason="Medical diagnosis integration test disabled by default; set SAGE_ENABLE_MEDICAL_DIAGNOSIS=1 to run.", +) +def test_single_case(): + """测试单个病例诊断""" + + print("=" * 80) + print("🧪 医疗诊断系统测试 - 单病例模式") + print("=" * 80) + + # 加载测试数据 + data_dir = medical_diagnosis_dir / "data" / "processed" + test_index_path = data_dir / "test_index.json" + + if not test_index_path.exists(): + pytest.skip( + f"Test data not available at {test_index_path}. Run scripts/prepare_data.py first to generate test data." + ) + + with open(test_index_path, encoding="utf-8") as f: + test_cases = json.load(f) + + # 随机选择一个测试病例 + case = random.choice(test_cases) + + print("\n📋 测试病例信息:") + print(f" - 病例ID: {case['case_id']}") + print(f" - 患者ID: {case['patient_id']}") + print(f" - 年龄: {case['age']}岁") + print(f" - 性别: {case['gender']}") + print(f" - 真实疾病: {case['disease']}") + print(f" - 严重程度: {case['severity']}") + + # 构建完整路径 + image_path = data_dir / case["image_path"] + + print(f"\n📸 MRI影像: {image_path}") + print(f" (文件存在: {image_path.exists()})") + + # 准备患者信息 + patient_info = { + "patient_id": case["patient_id"], + "age": case["age"], + "gender": case["gender"], + "symptoms": "腰痛伴下肢放射痛" if case["severity"] != "无" else "体检发现", + } + + # 初始化诊断系统 + print("\n🔧 初始化诊断系统...") + + config_path = medical_diagnosis_dir / "config" / "agent_config.yaml" + + try: + # 创建诊断Agent + agent = DiagnosticAgent(config_path=str(config_path)) + print(" ✓ DiagnosticAgent 初始化成功") + + # 执行诊断 + print("\n🔍 开始诊断...") + result = agent.diagnose(image_path=str(image_path), patient_info=patient_info) + + # 显示结果 + print("\n" + "=" * 80) + print("📊 诊断结果摘要") + print("=" * 80) + + print(f"\n病例ID: {case['patient_id']}") + print(f"诊断时间: {result.timestamp}") + print("\n影像分析:") + print(f" - 质量评分: {result.quality_score:.2f}") + print(f" - 检测到的病变: {len(result.findings)} 处") + + print("\n主要发现:") + for finding in result.findings[:3]: + print(f" - {finding}") + + print("\n诊断结论:") + for i, diagnosis in enumerate(result.diagnoses, 1): + print(f" {i}. {diagnosis}") + + print(f"\n 置信度: {result.confidence:.2%}") + + print(f"\n相似病例参考: {len(result.similar_cases)} 个") + + print("\n📄 完整诊断报告:") + print("-" * 80) + print(result.report) + print("-" * 80) + + # 对比真实标签 + print(f"\n✅ 真实疾病: {case['disease']}") + print(f" 诊断结果中是否包含: {'是' if case['disease'] in result.report else '否'}") + + except Exception as e: + print(f"❌ 诊断过程出错: {e}") + import traceback + + traceback.print_exc() + + print("\n" + "=" * 80) + + +@pytest.mark.skipif( + os.getenv("SAGE_ENABLE_MEDICAL_DIAGNOSIS") not in {"1", "true", "True"}, + reason="Medical diagnosis integration test disabled by default; set SAGE_ENABLE_MEDICAL_DIAGNOSIS=1 to run.", +) +def test_batch_mode(): + """测试批量诊断模式""" + + print("=" * 80) + print("🧪 医疗诊断系统测试 - 批量模式") + print("=" * 80) + + # 加载测试数据 + data_dir = medical_diagnosis_dir / "data" / "processed" + test_index_path = data_dir / "test_index.json" + + if not test_index_path.exists(): + pytest.skip( + f"Test data not available at {test_index_path}. Run scripts/prepare_data.py first to generate test data." + ) + + with open(test_index_path, encoding="utf-8") as f: + test_cases = json.load(f) + + # 选择前5个病例 + cases_to_test = test_cases[:5] + + print(f"\n📋 准备批量诊断 {len(cases_to_test)} 个病例...") + + # 准备批量病例 + batch_cases = [] + for case in cases_to_test: + image_path = data_dir / case["image_path"] + patient_info = { + "patient_id": case["patient_id"], + "age": case["age"], + "gender": case["gender"], + "symptoms": "腰痛" if case["severity"] != "无" else "体检", + } + batch_cases.append( + { + "image_path": str(image_path), + "patient_info": patient_info, + } + ) + + # 初始化Agent + config_path = medical_diagnosis_dir / "config" / "agent_config.yaml" + agent = DiagnosticAgent(config_path=str(config_path)) + + # 批量诊断 + print("\n🔍 开始批量诊断...") + output_dir = medical_diagnosis_dir / "data" / "test_results" + + results = agent.batch_diagnose(cases=batch_cases, output_dir=str(output_dir)) + + # 显示结果摘要 + print("\n" + "=" * 80) + print("📊 批量诊断结果摘要") + print("=" * 80) + + for i, result in enumerate(results, 1): + print(f"\n{i}. 病例 {cases_to_test[i - 1]['case_id']}") + print(f" 诊断: {', '.join(result.diagnoses)}") + print(f" 置信度: {result.confidence:.2%}") + print(f" 发现: {len(result.findings)} 处") + + print(f"\n✅ 批量诊断完成，结果已保存到: {output_dir}") + print("=" * 80) + + +def main(): + """主函数""" + + import argparse + + parser = argparse.ArgumentParser(description="医疗诊断系统测试") + parser.add_argument( + "--mode", + type=str, + default="single", + choices=["single", "batch"], + help="测试模式: single(单病例) 或 batch(批量)", + ) + + args = parser.parse_args() + + if args.mode == "single": + test_single_case() + elif args.mode == "batch": + test_batch_mode() + + +if __name__ == "__main__": + main() diff --git a/sage-apps/tests/medical_diagnosis/test_image_analyzer.py b/sage-apps/tests/medical_diagnosis/test_image_analyzer.py new file mode 100644 index 0000000..777b927 --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/test_image_analyzer.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Tests for ImageAnalyzer image loading and feature extraction.""" + +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest + +from sage.apps.medical_diagnosis.agents.image_analyzer import TORCH_AVAILABLE, ImageAnalyzer + + +# --------------------------------------------------------------------------- +# Common fixtures +# --------------------------------------------------------------------------- +@pytest.fixture +def analyzer(): + """Create an ImageAnalyzer instance for image-loading tests.""" + config = {"models": {"vision_model": "test-model"}} + return ImageAnalyzer(config) + + +@pytest.fixture +def temp_dir(): + """Create a temporary directory for test files.""" + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + + +@pytest.fixture +def basic_config(): + """Configuration used for feature extraction tests.""" + return { + "models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}, + "image_processing": {"feature_extraction": {"method": "clip", "dimension": 768}}, + } + + +@pytest.fixture +def dinov2_config(): + """Configuration for testing DINOv2 feature extraction.""" + return { + "models": {"vision_model": "Qwen/Qwen2-VL-7B-Instruct"}, + "image_processing": {"feature_extraction": {"method": "dinov2", "dimension": 768}}, + } + + +@pytest.fixture +def mock_image(): + """Create a mock grayscale numpy image.""" + return np.random.randint(0, 255, (512, 512), dtype=np.uint8) + + +@pytest.fixture +def mock_pil_image(): + """Create a mock RGB PIL image for feature extraction tests.""" + if not TORCH_AVAILABLE: + pytest.skip("PyTorch not available") + + from PIL import Image + + return Image.fromarray(np.random.randint(0, 255, (512, 512, 3), dtype=np.uint8)) + + +# --------------------------------------------------------------------------- +# Image loading unit tests +# --------------------------------------------------------------------------- +class TestLoadImage: + """Tests for the _load_image method.""" + + def test_load_image_nonexistent_file(self, analyzer, temp_dir): + fake_path = temp_dir / "nonexistent.png" + result = analyzer._load_image(fake_path) + assert result is None + + def test_load_image_dicom_extension(self, analyzer, temp_dir): + dcm_path = temp_dir / "test.dcm" + dcm_path.touch() + + with patch.object(analyzer, "_load_dicom_image", return_value=None) as mock_dicom: + analyzer._load_image(dcm_path) + mock_dicom.assert_called_once_with(dcm_path) + + def test_load_image_dicom_extension_uppercase(self, analyzer, temp_dir): + dcm_path = temp_dir / "test.DCM" + dcm_path.touch() + + with patch.object(analyzer, "_load_dicom_image", return_value=None) as mock_dicom: + analyzer._load_image(dcm_path) + mock_dicom.assert_called_once_with(dcm_path) + + def test_load_image_png_extension(self, analyzer, temp_dir): + png_path = temp_dir / "test.png" + png_path.touch() + + with patch.object(analyzer, "_load_regular_image", return_value=None) as mock_regular: + analyzer._load_image(png_path) + mock_regular.assert_called_once_with(png_path) + + def test_load_image_jpg_extension(self, analyzer, temp_dir): + jpg_path = temp_dir / "test.jpg" + jpg_path.touch() + + with patch.object(analyzer, "_load_regular_image", return_value=None) as mock_regular: + analyzer._load_image(jpg_path) + mock_regular.assert_called_once_with(jpg_path) + + +class TestLoadDicomImage: + """Tests for the _load_dicom_image method.""" + + def test_load_dicom_success(self, analyzer, temp_dir): + dcm_path = temp_dir / "test.dcm" + dcm_path.touch() + + mock_dicom = MagicMock() + mock_dicom.pixel_array = np.array([[100, 200], [150, 250]], dtype=np.int16) + del mock_dicom.WindowCenter + del mock_dicom.WindowWidth + + with patch.dict("sys.modules", {"pydicom": MagicMock()}): + import sys + + sys.modules["pydicom"].dcmread.return_value = mock_dicom + result = analyzer._load_dicom_image(dcm_path) + + assert result is not None + assert result.dtype == np.uint8 + assert result.shape == (2, 2) + assert result.min() >= 0 + assert result.max() <= 255 + + def test_load_dicom_with_window_level(self, analyzer, temp_dir): + dcm_path = temp_dir / "test.dcm" + dcm_path.touch() + + mock_dicom = MagicMock() + mock_dicom.pixel_array = np.array([[0, 500], [1000, 2000]], dtype=np.int16) + mock_dicom.WindowCenter = 500 + mock_dicom.WindowWidth = 1000 + + with patch.dict("sys.modules", {"pydicom": MagicMock()}): + import sys + + sys.modules["pydicom"].dcmread.return_value = mock_dicom + result = analyzer._load_dicom_image(dcm_path) + + assert result is not None + assert result.dtype == np.uint8 + + def test_load_dicom_pydicom_not_installed(self, analyzer, temp_dir): + dcm_path = temp_dir / "test.dcm" + dcm_path.touch() + + with patch.dict("sys.modules", {"pydicom": None}): + original_import = __import__ + + def mock_import(name, *args, **kwargs): + if name == "pydicom": + raise ImportError("No module named 'pydicom'") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=mock_import): + result = analyzer._load_dicom_image(dcm_path) + + assert result is None + + def test_load_dicom_invalid_file(self, analyzer, temp_dir): + dcm_path = temp_dir / "invalid.dcm" + dcm_path.write_bytes(b"not a valid dicom file") + + result = analyzer._load_dicom_image(dcm_path) + assert result is None or isinstance(result, np.ndarray) + + +class TestLoadRegularImage: + """Tests for the _load_regular_image method.""" + + def test_load_regular_image_png(self, analyzer, temp_dir): + png_path = temp_dir / "test.png" + + try: + from PIL import Image + + img = Image.new("RGB", (10, 10), color="red") + img.save(png_path) + result = analyzer._load_regular_image(png_path) + except ImportError: + pytest.skip("PIL not installed") + + assert result is not None + assert isinstance(result, np.ndarray) + assert result.shape == (10, 10) + assert result.dtype == np.uint8 + + def test_load_regular_image_jpg(self, analyzer, temp_dir): + jpg_path = temp_dir / "test.jpg" + + try: + from PIL import Image + + img = Image.new("L", (20, 20), color=128) + img.save(jpg_path) + result = analyzer._load_regular_image(jpg_path) + except ImportError: + pytest.skip("PIL not installed") + + assert result is not None + assert isinstance(result, np.ndarray) + assert result.shape == (20, 20) + assert result.dtype == np.uint8 + + def test_load_regular_image_grayscale_conversion(self, analyzer, temp_dir): + png_path = temp_dir / "color.png" + + try: + from PIL import Image + + img = Image.new("RGB", (5, 5), color=(255, 0, 0)) + img.save(png_path) + result = analyzer._load_regular_image(png_path) + except ImportError: + pytest.skip("PIL not installed") + + assert result is not None + assert len(result.shape) == 2 + + def test_load_regular_image_invalid_file(self, analyzer, temp_dir): + invalid_path = temp_dir / "invalid.png" + invalid_path.write_bytes(b"not a valid image file") + + result = analyzer._load_regular_image(invalid_path) + assert result is None + + +class TestImageAnalyzerLoadingIntegration: + """Integration tests for the image loading pipeline.""" + + def test_analyze_with_valid_image(self, analyzer, temp_dir): + try: + from PIL import Image + + img_path = temp_dir / "mri_test.png" + img = Image.new("L", (100, 100), color=128) + img.save(img_path) + result = analyzer.analyze(str(img_path)) + except ImportError: + pytest.skip("PIL not installed") + + assert "vertebrae" in result + assert "discs" in result + assert "abnormalities" in result + assert "image_quality" in result + assert result["image_quality"] > 0 + + def test_analyze_with_nonexistent_image(self, analyzer, temp_dir): + fake_path = temp_dir / "nonexistent.png" + result = analyzer.analyze(str(fake_path)) + + assert "vertebrae" in result + assert "discs" in result + assert "abnormalities" in result + + def test_image_quality_assessment(self, analyzer, temp_dir): + try: + from PIL import Image + + img_path = temp_dir / "quality_test.png" + img = Image.new("L", (50, 50), color=100) + img.save(img_path) + image = analyzer._load_image(Path(img_path)) + quality = analyzer._assess_quality(image) + except ImportError: + pytest.skip("PIL not installed") + + assert 0.0 <= quality <= 1.0 + + def test_image_quality_none_image(self, analyzer): + assert analyzer._assess_quality(None) == 0.0 + + +# --------------------------------------------------------------------------- +# Feature extraction tests from main-dev +# --------------------------------------------------------------------------- +class TestImageAnalyzerFeatureExtraction: + """Unit tests for ImageAnalyzer feature extraction.""" + + def test_initialization_without_torch(self, basic_config): + with patch("sage.apps.medical_diagnosis.agents.image_analyzer.TORCH_AVAILABLE", False): + analyzer = ImageAnalyzer(basic_config) + assert analyzer.feature_model is None + assert analyzer.feature_processor is None + + def test_initialization_with_clip(self, basic_config): + if not TORCH_AVAILABLE: + pytest.skip("PyTorch not available") + + with ( + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPProcessor") as mock_proc, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPModel") as mock_model, + ): + mock_proc.from_pretrained.return_value = MagicMock() + mock_model.from_pretrained.return_value = MagicMock() + analyzer = ImageAnalyzer(basic_config) + + assert analyzer.feature_method == "clip" + + def test_initialization_with_dinov2(self, dinov2_config): + if not TORCH_AVAILABLE: + pytest.skip("PyTorch not available") + + with ( + patch( + "sage.apps.medical_diagnosis.agents.image_analyzer.AutoImageProcessor" + ) as mock_proc, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.AutoModel") as mock_model, + ): + mock_proc.from_pretrained.return_value = MagicMock() + mock_model.from_pretrained.return_value = MagicMock() + analyzer = ImageAnalyzer(dinov2_config) + + assert analyzer.feature_method == "dinov2" + + def test_mock_features_fallback(self, basic_config, mock_image): + with patch("sage.apps.medical_diagnosis.agents.image_analyzer.TORCH_AVAILABLE", False): + analyzer = ImageAnalyzer(basic_config) + features = analyzer._extract_features(mock_image) + + assert features is not None + assert isinstance(features, np.ndarray) + assert features.shape == (768,) + assert features.dtype == np.float32 + + def test_extract_features_with_none_image(self, basic_config): + analyzer = ImageAnalyzer(basic_config) + assert analyzer._extract_features(None) is None + + def test_extract_features_with_numpy_array(self, basic_config, mock_image): + with patch("sage.apps.medical_diagnosis.agents.image_analyzer.TORCH_AVAILABLE", False): + analyzer = ImageAnalyzer(basic_config) + features = analyzer._extract_features(mock_image) + + assert features is not None + assert isinstance(features, np.ndarray) + assert len(features.shape) == 1 + + def test_extract_features_error_handling(self, basic_config, mock_image): + if not TORCH_AVAILABLE: + pytest.skip("PyTorch not available") + + with ( + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPProcessor") as mock_proc, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPModel") as mock_model, + ): + mock_proc.from_pretrained.return_value = MagicMock() + mock_model_instance = MagicMock() + mock_model_instance.cuda.return_value = mock_model_instance + mock_model_instance.get_image_features.side_effect = RuntimeError("Model error") + mock_model.from_pretrained.return_value = mock_model_instance + analyzer = ImageAnalyzer(basic_config) + features = analyzer._extract_features(mock_image) + + assert features is not None + assert isinstance(features, np.ndarray) + + def test_feature_dimension_config(self, mock_image): + custom_config = { + "models": {"vision_model": "test"}, + "image_processing": {"feature_extraction": {"method": "clip", "dimension": 512}}, + } + + with patch("sage.apps.medical_diagnosis.agents.image_analyzer.TORCH_AVAILABLE", False): + analyzer = ImageAnalyzer(custom_config) + features = analyzer._extract_features(mock_image) + + assert features.shape == (512,) + + def test_analyze_with_nonexistent_image(self, basic_config): + analyzer = ImageAnalyzer(basic_config) + result = analyzer.analyze("nonexistent_image.jpg") + + assert "vertebrae" in result + assert "discs" in result + assert "abnormalities" in result + assert "image_quality" in result + assert "image_embedding" in result + + @pytest.mark.skipif(not TORCH_AVAILABLE, reason="PyTorch not available") + def test_clip_feature_extraction_shape(self, basic_config, mock_pil_image): + with ( + patch( + "sage.apps.medical_diagnosis.agents.image_analyzer.CLIPProcessor" + ) as mock_proc_cls, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPModel") as mock_model_cls, + ): + import torch + + mock_processor = MagicMock() + mock_processor.return_value = {"pixel_values": torch.randn(1, 3, 224, 224)} + mock_proc_cls.from_pretrained.return_value = mock_processor + + mock_model = MagicMock() + mock_features = torch.randn(1, 512) + mock_model.get_image_features.return_value = mock_features + mock_model_cls.from_pretrained.return_value = mock_model + + analyzer = ImageAnalyzer(basic_config) + analyzer.feature_processor = mock_processor + analyzer.feature_model = mock_model + features = analyzer._extract_clip_features(mock_pil_image) + + assert isinstance(features, np.ndarray) + assert features.dtype == np.float32 + assert len(features.shape) == 1 + assert np.isclose(np.linalg.norm(features), 1.0, rtol=1e-5) + + @pytest.mark.skipif(not TORCH_AVAILABLE, reason="PyTorch not available") + def test_dinov2_feature_extraction_shape(self, dinov2_config, mock_pil_image): + with ( + patch( + "sage.apps.medical_diagnosis.agents.image_analyzer.AutoImageProcessor" + ) as mock_proc_cls, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.AutoModel") as mock_model_cls, + ): + import torch + + mock_processor = MagicMock() + mock_processor.return_value = {"pixel_values": torch.randn(1, 3, 224, 224)} + mock_proc_cls.from_pretrained.return_value = mock_processor + + mock_model = MagicMock() + mock_output = MagicMock() + mock_output.last_hidden_state = torch.randn(1, 197, 768) + mock_model.return_value = mock_output + mock_model_cls.from_pretrained.return_value = mock_model + + analyzer = ImageAnalyzer(dinov2_config) + analyzer.feature_processor = mock_processor + analyzer.feature_model = mock_model + features = analyzer._extract_dinov2_features(mock_pil_image) + + assert isinstance(features, np.ndarray) + assert features.dtype == np.float32 + assert len(features.shape) == 1 + assert np.isclose(np.linalg.norm(features), 1.0, rtol=1e-5) + + def test_unknown_feature_method(self, mock_image): + config = { + "models": {"vision_model": "test"}, + "image_processing": {"feature_extraction": {"method": "unknown", "dimension": 768}}, + } + + analyzer = ImageAnalyzer(config) + features = analyzer._extract_features(mock_image) + + assert features is not None + assert features.shape == (768,) + + +class TestImageAnalyzerFeatureIntegration: + """Integration-style tests focusing on feature extraction pipeline.""" + + def test_full_analysis_pipeline(self, basic_config, tmp_path): + if not TORCH_AVAILABLE: + pytest.skip("PyTorch not available") + + from PIL import Image + + test_image_path = tmp_path / "test_mri.png" + test_image = Image.fromarray( + np.random.randint(0, 255, (512, 512), dtype=np.uint8), mode="L" + ) + test_image.save(test_image_path) + + with ( + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPProcessor") as mock_proc, + patch("sage.apps.medical_diagnosis.agents.image_analyzer.CLIPModel") as mock_model, + ): + mock_proc.from_pretrained.return_value = MagicMock() + mock_model.from_pretrained.return_value = MagicMock() + analyzer = ImageAnalyzer(basic_config) + result = analyzer.analyze(str(test_image_path)) + + assert "vertebrae" in result + assert "discs" in result + assert "abnormalities" in result + assert "image_quality" in result + assert "image_embedding" in result + assert result["image_path"] == str(test_image_path) + assert len(result["vertebrae"]) == 5 + assert len(result["discs"]) == 5 + + def test_create_mock_analysis(self, basic_config): + analyzer = ImageAnalyzer(basic_config) + result = analyzer._create_mock_analysis() + + assert isinstance(result["vertebrae"], list) + assert isinstance(result["discs"], list) + assert isinstance(result["abnormalities"], list) + assert isinstance(result["image_quality"], float) + assert isinstance(result["image_embedding"], np.ndarray) + assert result["image_embedding"].shape == (768,) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/sage-apps/tests/medical_diagnosis/test_image_quality.py b/sage-apps/tests/medical_diagnosis/test_image_quality.py new file mode 100644 index 0000000..ce8d071 --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/test_image_quality.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +测试影像质量评估算法 +""" + +import numpy as np +import pytest + +from sage.apps.medical_diagnosis.agents.image_analyzer import ImageAnalyzer + + +class TestImageQualityAssessment: + """测试影像质量评估功能""" + + @pytest.fixture + def analyzer(self): + """创建ImageAnalyzer实例""" + config = {"models": {"vision_model": "test-model"}} + return ImageAnalyzer(config) + + def test_assess_quality_none_image(self, analyzer): + """测试None图像的质量评估""" + score = analyzer._assess_quality(None) + assert score == 0.0, "None图像应返回0.0分数" + + def test_assess_quality_high_quality_image(self, analyzer): + """测试高质量图像（高对比度、清晰）""" + # 创建一个清晰的高对比度图像（棋盘格） + size = 100 + image = np.zeros((size, size), dtype=np.uint8) + # 创建棋盘格模式 + for i in range(size): + for j in range(size): + if (i // 10 + j // 10) % 2 == 0: + image[i, j] = 255 + + score = analyzer._assess_quality(image) + # 高质量图像应该有较高分数 + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + assert score > 0.6, f"高质量图像应有较高分数 (>0.6)，实际: {score}" + + def test_assess_quality_low_quality_blurry_image(self, analyzer): + """测试低质量图像（模糊）""" + # 创建模糊图像（均匀灰度） + size = 100 + image = np.ones((size, size), dtype=np.uint8) * 128 + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + # 模糊图像应该有较低分数（对比度低、清晰度低） + assert score < 0.8, f"模糊图像应有较低分数 (<0.8)，实际: {score}" + + def test_assess_quality_noisy_image(self, analyzer): + """测试噪声图像""" + # 创建随机噪声图像 + size = 100 + np.random.seed(42) + image = np.random.randint(0, 256, (size, size), dtype=np.uint8) + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + # 噪声图像可能有不同分数，主要测试不会崩溃 + + def test_assess_quality_dark_image(self, analyzer): + """测试暗图像（亮度过低）""" + # 创建暗图像 + size = 100 + image = np.ones((size, size), dtype=np.uint8) * 30 # 很暗 + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + # 亮度不佳的图像分数应该受影响 + + def test_assess_quality_bright_image(self, analyzer): + """测试亮图像（亮度过高）""" + # 创建亮图像 + size = 100 + image = np.ones((size, size), dtype=np.uint8) * 240 # 很亮 + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + + def test_assess_quality_normalized_image(self, analyzer): + """测试已归一化图像（0-1范围）""" + # 创建0-1范围的图像 + size = 100 + image = np.random.rand(size, size).astype(np.float32) + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + + def test_assess_quality_edge_cases(self, analyzer): + """测试边缘情况""" + # 测试单像素图像 + small_image = np.array([[128]], dtype=np.uint8) + score = analyzer._assess_quality(small_image) + assert 0.0 <= score <= 1.0, "单像素图像应返回有效分数" + + # 测试全黑图像 + black_image = np.zeros((50, 50), dtype=np.uint8) + score = analyzer._assess_quality(black_image) + assert 0.0 <= score <= 1.0, "全黑图像应返回有效分数" + + # 测试全白图像 + white_image = np.ones((50, 50), dtype=np.uint8) * 255 + score = analyzer._assess_quality(white_image) + assert 0.0 <= score <= 1.0, "全白图像应返回有效分数" + + def test_assess_quality_realistic_medical_image(self, analyzer): + """测试模拟真实医学影像""" + # 创建模拟MRI影像（带有结构和渐变） + size = 200 + image = np.zeros((size, size), dtype=np.uint8) + + # 添加背景渐变 + for i in range(size): + for j in range(size): + image[i, j] = int(100 + 50 * np.sin(i / 20) * np.cos(j / 20)) + + # 添加一些"结构"（模拟椎体） + for k in range(5): + y_center = 40 + k * 30 + for i in range(max(0, y_center - 15), min(size, y_center + 15)): + for j in range(80, 120): + dist = np.sqrt((i - y_center) ** 2 + (j - 100) ** 2) + if dist < 15: + image[i, j] = 180 + + score = analyzer._assess_quality(image) + assert 0.0 <= score <= 1.0, f"分数应在[0,1]范围内，实际: {score}" + # 有结构的图像应该有合理的分数 + assert 0.3 <= score <= 0.9, f"模拟医学影像应有合理分数，实际: {score}" + + def test_assess_quality_consistency(self, analyzer): + """测试评估一致性（同一图像多次评估应得到相同结果）""" + size = 100 + image = np.random.RandomState(42).randint(0, 256, (size, size), dtype=np.uint8) + + score1 = analyzer._assess_quality(image) + score2 = analyzer._assess_quality(image) + + assert score1 == score2, "同一图像的评估应该一致" + + def test_assess_quality_different_types(self, analyzer): + """测试不同数据类型的图像""" + size = 50 + + # uint8 类型 + img_uint8 = np.random.RandomState(42).randint(0, 256, (size, size), dtype=np.uint8) + score_uint8 = analyzer._assess_quality(img_uint8) + assert 0.0 <= score_uint8 <= 1.0 + + # float32 类型 (0-1) + img_float32 = img_uint8.astype(np.float32) / 255.0 + score_float32 = analyzer._assess_quality(img_float32) + assert 0.0 <= score_float32 <= 1.0 + + # 两种类型应该给出相近的结果 + assert abs(score_uint8 - score_float32) < 0.1, "不同类型相同内容应给出相近分数" + + def test_quality_penalizes_added_noise(self, analyzer): + """干净图像应比加噪图像获得更高分""" + size = 128 + gradient = np.tile(np.linspace(30, 220, size, dtype=np.float32), (size, 1)) + clean = gradient.astype(np.uint8) + rng = np.random.RandomState(7) + noisy = np.clip(clean + rng.normal(0, 25, clean.shape), 0, 255).astype(np.uint8) + + clean_score = analyzer._assess_quality(clean) + noisy_score = analyzer._assess_quality(noisy) + + assert clean_score > noisy_score, "加噪后应降低质量分数" + + def test_brightness_histogram_balance(self, analyzer): + """亮度直方图平衡的图像应优于过曝/欠曝图像""" + size = 200 + balanced_line = np.linspace(0, 255, size, dtype=np.float32) + balanced = np.tile(balanced_line, (size, 1)).astype(np.uint8) + bright = np.ones((size, size), dtype=np.uint8) * 240 + dark = np.ones((size, size), dtype=np.uint8) * 15 + + balanced_score = analyzer._assess_quality(balanced) + bright_score = analyzer._assess_quality(bright) + dark_score = analyzer._assess_quality(dark) + + assert balanced_score > bright_score + assert balanced_score > dark_score diff --git a/sage-apps/tests/medical_diagnosis/test_knowledge_base.py b/sage-apps/tests/medical_diagnosis/test_knowledge_base.py new file mode 100644 index 0000000..b32a586 --- /dev/null +++ b/sage-apps/tests/medical_diagnosis/test_knowledge_base.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +""" +Tests for MedicalKnowledgeBase +""" + +import json +from pathlib import Path + +import pytest + +from sage.apps.medical_diagnosis.tools.knowledge_base import MedicalKnowledgeBase + + +class TestMedicalKnowledgeBase: + """Test MedicalKnowledgeBase class""" + + def test_initialization_basic(self): + """Test basic initialization without data files""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + assert kb is not None + assert isinstance(kb.knowledge_base, list) + assert len(kb.knowledge_base) > 0 # Should have at least default knowledge + assert isinstance(kb.case_database, list) + + def test_default_knowledge_loaded(self): + """Test that default knowledge is loaded""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + # Check that default knowledge contains expected topics + topics = [k["topic"] for k in kb.knowledge_base] + assert "腰椎间盘突出症" in topics + assert "腰椎退行性变" in topics + assert "椎管狭窄" in topics + + def test_retrieve_knowledge_basic(self): + """Test basic knowledge retrieval""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + # Search for knowledge about disc herniation + results = kb.retrieve_knowledge("腰椎间盘突出", top_k=3) + + assert isinstance(results, list) + # Should find at least one result + assert len(results) >= 1 + # First result should be relevant + assert "腰椎" in results[0]["topic"] or "椎间盘" in results[0]["topic"] + + def test_retrieve_similar_cases_basic(self): + """Test basic case retrieval""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + # Search for similar cases + results = kb.retrieve_similar_cases(query="腰痛伴下肢麻木", image_features={}, top_k=3) + + assert isinstance(results, list) + assert len(results) > 0 + assert len(results) <= 3 + # Each result should have required fields + for case in results: + assert "case_id" in case + assert "diagnosis" in case + + def test_add_case(self): + """Test adding a new case""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + initial_count = len(kb.case_database) + + new_case = { + "case_id": "TEST_001", + "age": 50, + "gender": "male", + "diagnosis": "腰椎间盘突出症", + } + + kb.add_case(new_case) + + assert len(kb.case_database) == initial_count + 1 + assert kb.case_database[-1]["case_id"] == "TEST_001" + + def test_update_knowledge(self): + """Test updating knowledge base""" + config = {"verbose": False} + kb = MedicalKnowledgeBase(config) + + initial_count = len(kb.knowledge_base) + + new_knowledge = { + "topic": "测试疾病", + "content": "这是一个测试疾病的描述", + "treatment": "测试治疗方法", + } + + kb.update_knowledge(new_knowledge) + + assert len(kb.knowledge_base) == initial_count + 1 + assert kb.knowledge_base[-1]["topic"] == "测试疾病" + + def test_load_from_nonexistent_path(self): + """Test loading with nonexistent data path""" + config = {"data_path": "/nonexistent/path/to/data", "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Should still initialize with default knowledge + assert len(kb.knowledge_base) >= 3 + assert len(kb.case_database) == 0 + + def test_configuration_options(self): + """Test configuration options work correctly""" + config = { + "verbose": False, + "enable_dataset_knowledge": False, + "enable_report_knowledge": False, + "enable_case_database": False, + } + kb = MedicalKnowledgeBase(config) + + # Should only have default knowledge (3 entries) + assert len(kb.knowledge_base) == 3 + assert len(kb.case_database) == 0 + + def test_max_reports_configuration(self): + """Test max_reports configuration option""" + config = { + "verbose": False, + "max_reports": 10, + } + kb = MedicalKnowledgeBase(config) + + # Should initialize without error + assert kb._max_reports == 10 + + @pytest.mark.skipif( + not ( + Path(__file__).parent.parent.parent + / "src" + / "sage" + / "apps" + / "medical_diagnosis" + / "data" + / "processed" + / "stats.json" + ).exists(), + reason="Test data not available", + ) + def test_load_from_dataset(self): + """Test loading knowledge from actual dataset if available""" + # Point to the actual data directory + medical_dir = ( + Path(__file__).parent.parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" + ) + data_path = medical_dir / "data" / "processed" + + config = {"data_path": str(data_path), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Should have loaded additional knowledge + assert len(kb.knowledge_base) > 3 # More than just default knowledge + + @pytest.mark.skipif( + not ( + Path(__file__).parent.parent.parent + / "src" + / "sage" + / "apps" + / "medical_diagnosis" + / "data" + / "processed" + / "all_cases.json" + ).exists(), + reason="Test data not available", + ) + def test_load_case_database(self): + """Test loading case database from dataset if available""" + # Point to the actual data directory + medical_dir = ( + Path(__file__).parent.parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" + ) + data_path = medical_dir / "data" / "processed" + + config = {"data_path": str(data_path), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Should have loaded cases + assert len(kb.case_database) > 0 + + # Test case retrieval with loaded cases + results = kb.retrieve_similar_cases(query="椎间盘突出", image_features={}, top_k=3) + + assert len(results) > 0 + + +class TestMedicalKnowledgeBaseWithMockData: + """Test MedicalKnowledgeBase with mock data files""" + + @pytest.fixture + def mock_data_dir(self, tmp_path): + """Create a mock data directory with test files""" + # Create processed data directory + processed_dir = tmp_path / "processed" + processed_dir.mkdir() + + # Create stats.json + stats = { + "total_samples": 100, + "disease_distribution": { + "椎间盘突出": 30, + "椎管狭窄": 20, + "正常": 50, # This should be filtered out + }, + "severity_distribution": { + "轻度": 40, + "中度": 35, + "重度": 25, + }, + } + with open(processed_dir / "stats.json", "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False) + + # Create all_cases.json + cases = [ + { + "case_id": "case_0001", + "patient_id": "P0001", + "age": 45, + "gender": "男", + "disease": "椎间盘突出", + "severity": "中度", + "image_path": "images/case_0001.jpg", + "report_path": "reports/case_0001_report.txt", + }, + { + "case_id": "case_0002", + "patient_id": "P0002", + "age": 55, + "gender": "女", + "disease": "椎管狭窄", + "severity": "重度", + "image_path": "images/case_0002.jpg", + "report_path": "reports/case_0002_report.txt", + }, + ] + with open(processed_dir / "all_cases.json", "w", encoding="utf-8") as f: + json.dump(cases, f, ensure_ascii=False) + + # Create reports directory with mock reports + reports_dir = processed_dir / "reports" + reports_dir.mkdir() + + # Create mock reports + report_1 = """患者信息: + 年龄: 45岁 + 性别: 男 + 主诉: 腰痛伴右下肢放射痛3周 + +影像描述: + 腰椎MRI T2加权矢状位: L4/L5椎间盘向后突出,压迫硬膜囊。 + +主要发现: + - 病变节段: L4/L5 + - 病变类型: 椎间盘突出 + - 严重程度: 中度 + +诊断结论: + 椎间盘突出，程度中度。 + +治疗建议: + 建议卧床休息2-3周，牵引治疗。口服非甾体抗炎药及神经营养药物。 +""" + with open(reports_dir / "case_0001_report.txt", "w", encoding="utf-8") as f: + f.write(report_1) + + report_2 = """患者信息: + 年龄: 55岁 + 性别: 女 + 主诉: 腰痛伴双下肢麻木、无力2月 + +影像描述: + 腰椎MRI T2加权矢状位: L4/L5椎管狭窄,硬膜囊受压。 + +主要发现: + - 病变节段: L4/L5 + - 病变类型: 椎管狭窄 + - 严重程度: 重度 + +诊断结论: + 椎管狭窄，程度重度。 + +治疗建议: + 建议尽早手术治疗(椎间盘摘除术或椎管减压术)，以解除神经压迫。 +""" + with open(reports_dir / "case_0002_report.txt", "w", encoding="utf-8") as f: + f.write(report_2) + + return processed_dir + + def test_load_knowledge_from_mock_dataset(self, mock_data_dir): + """Test loading knowledge from mock stats.json""" + config = {"data_path": str(mock_data_dir), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Should have default knowledge (3) + dataset knowledge (2, excluding "正常") + topics = [k["topic"] for k in kb.knowledge_base] + assert "椎间盘突出" in topics + assert "椎管狭窄" in topics + assert "正常" not in topics # "正常" should be filtered out + + def test_load_knowledge_from_mock_reports(self, mock_data_dir): + """Test loading knowledge from mock report files""" + config = {"data_path": str(mock_data_dir), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Check that report knowledge was loaded + report_knowledge = [k for k in kb.knowledge_base if k.get("source") == "medical_reports"] + assert len(report_knowledge) >= 1 + + # Check content of extracted knowledge + for knowledge in report_knowledge: + assert "topic" in knowledge + assert "content" in knowledge + assert "treatment" in knowledge + + def test_load_case_database_from_mock(self, mock_data_dir): + """Test loading case database from mock all_cases.json""" + config = {"data_path": str(mock_data_dir), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Should have loaded 2 cases + assert len(kb.case_database) == 2 + + # Check case structure + for case in kb.case_database: + assert "case_id" in case + assert "age" in case + assert "gender" in case + assert "diagnosis" in case + assert "severity" in case + + def test_retrieve_cases_from_mock_database(self, mock_data_dir): + """Test case retrieval with mock database""" + config = {"data_path": str(mock_data_dir), "verbose": False} + kb = MedicalKnowledgeBase(config) + + # Search for cases related to disc herniation + results = kb.retrieve_similar_cases(query="椎间盘突出", image_features={}, top_k=3) + + assert len(results) >= 1 + # At least one result should be about disc herniation + assert any("椎间盘突出" in r.get("diagnosis", "") for r in results) + + def test_disable_dataset_loading(self, mock_data_dir): + """Test that dataset loading can be disabled""" + config = { + "data_path": str(mock_data_dir), + "verbose": False, + "enable_dataset_knowledge": False, + } + kb = MedicalKnowledgeBase(config) + + # Should not have knowledge from dataset_statistics source + dataset_knowledge = [ + k for k in kb.knowledge_base if k.get("source") == "dataset_statistics" + ] + assert len(dataset_knowledge) == 0 + + def test_disable_report_loading(self, mock_data_dir): + """Test that report loading can be disabled""" + config = { + "data_path": str(mock_data_dir), + "verbose": False, + "enable_report_knowledge": False, + } + kb = MedicalKnowledgeBase(config) + + # Should not have knowledge from medical_reports source + report_knowledge = [k for k in kb.knowledge_base if k.get("source") == "medical_reports"] + assert len(report_knowledge) == 0 + + def test_disable_case_database_loading(self, mock_data_dir): + """Test that case database loading can be disabled""" + config = { + "data_path": str(mock_data_dir), + "verbose": False, + "enable_case_database": False, + } + kb = MedicalKnowledgeBase(config) + + # Should not have loaded any cases + assert len(kb.case_database) == 0 + + def test_max_reports_limit(self, mock_data_dir): + """Test max_reports configuration limits report loading""" + config = { + "data_path": str(mock_data_dir), + "verbose": False, + "max_reports": 1, # Only read 1 report + } + kb = MedicalKnowledgeBase(config) + + # Should have limited number of report knowledge entries + report_knowledge = [k for k in kb.knowledge_base if k.get("source") == "medical_reports"] + # With max_reports=1, we should have at most 1 unique disease from reports + assert len(report_knowledge) <= 1 diff --git a/sage-apps/tests/smart_home/__init__.py b/sage-apps/tests/smart_home/__init__.py new file mode 100644 index 0000000..a39b843 --- /dev/null +++ b/sage-apps/tests/smart_home/__init__.py @@ -0,0 +1 @@ +"""Tests for Smart Home application""" diff --git a/sage-apps/tests/smart_home/test_operators.py b/sage-apps/tests/smart_home/test_operators.py new file mode 100644 index 0000000..1fe5966 --- /dev/null +++ b/sage-apps/tests/smart_home/test_operators.py @@ -0,0 +1,377 @@ +"""Unit tests for Smart Home Application + +Tests for sage.apps.smart_home module +""" + +import pytest + +from sage.apps.smart_home.operators import ( + DeviceEvent, + DeviceExecutor, + DeviceType, + EnvironmentMonitor, + EventLogSink, + LaundryWorkflowSource, + TaskStatus, + WorkflowProgressSink, +) + + +class TestDeviceEvent: + """Test DeviceEvent dataclass""" + + def test_event_creation(self): + """Test creating a device event""" + event = DeviceEvent( + event_id=1, + device_id="robot_01", + device_type=DeviceType.ROBOT, + event_type="task_started", + timestamp="2024-01-15T10:00:00", + data={"task": "pickup_laundry", "status": "running"}, + ) + + assert event.event_id == 1 + assert event.device_id == "robot_01" + assert event.device_type == DeviceType.ROBOT + assert event.event_type == "task_started" + assert "task" in event.data + + def test_event_with_sensor_data(self): + """Test event with sensor data""" + event = DeviceEvent( + event_id=2, + device_id="humidity_01", + device_type=DeviceType.HUMIDITY_SENSOR, + event_type="reading", + timestamp="2024-01-15T10:00:01", + data={"humidity": 65.0, "temperature": 22.5}, + ) + + assert event.device_type == DeviceType.HUMIDITY_SENSOR + assert event.data["humidity"] == 65.0 + + +class TestDeviceType: + """Test DeviceType enum""" + + def test_device_types(self): + """Test all device types are defined""" + assert DeviceType.ROBOT.value == "robot" + assert DeviceType.WASHER.value == "washer" + assert DeviceType.DRYER.value == "dryer" + assert DeviceType.HUMIDITY_SENSOR.value == "humidity_sensor" + assert DeviceType.MOTION_SENSOR.value == "motion_sensor" + + +class TestTaskStatus: + """Test TaskStatus enum""" + + def test_task_statuses(self): + """Test all task statuses are defined""" + assert TaskStatus.PENDING.value == "pending" + assert TaskStatus.RUNNING.value == "running" + assert TaskStatus.COMPLETED.value == "completed" + assert TaskStatus.FAILED.value == "failed" + + +class TestLaundryWorkflowSource: + """Test LaundryWorkflowSource operator""" + + def test_source_creation(self): + """Test creating laundry workflow source""" + source = LaundryWorkflowSource(num_cycles=2) + + assert source.num_cycles == 2 + assert source.current_cycle == 0 + assert source.step == 0 + + def test_source_generates_workflow_tasks(self): + """Test source generates workflow tasks""" + source = LaundryWorkflowSource(num_cycles=1) + + # Execute to get first task + task = source.execute() + + # Should generate workflow task + assert task is not None + assert "task" in task + assert "device" in task + assert "cycle" in task + assert task["cycle"] == 1 + + def test_source_workflow_steps(self): + """Test source generates all workflow steps""" + source = LaundryWorkflowSource(num_cycles=1) + + tasks = [] + while True: + task = source.execute() + if task is None: + break + tasks.append(task) + + # Should have 6 steps in workflow + assert len(tasks) == 6 + assert tasks[0]["task"] == "check_environment" + assert tasks[1]["task"] == "collect_laundry" + assert tasks[2]["task"] == "wash" + + def test_source_with_multiple_cycles(self): + """Test source with multiple laundry cycles""" + source = LaundryWorkflowSource(num_cycles=2) + + tasks = [] + while True: + task = source.execute() + if task is None: + break + tasks.append(task) + + # Should have 6 steps * 2 cycles = 12 tasks + assert len(tasks) == 12 + assert tasks[0]["cycle"] == 1 + assert tasks[6]["cycle"] == 2 + + +class TestDeviceExecutor: + """Test DeviceExecutor operator""" + + def test_executor_creation(self): + """Test creating device executor""" + executor = DeviceExecutor() + + assert executor.event_counter == 0 + + def test_execute_robot_task(self): + """Test executing robot task""" + executor = DeviceExecutor() + + task_data = { + "task": "collect_laundry", + "device": "robot_001", + "description": "Robot collects laundry", + "params": {"from": "basket", "to": "washer"}, + "duration": 0.01, + } + + result = executor.execute(task_data) + assert result is not None + assert result["status"] == TaskStatus.COMPLETED.value + assert "completed_at" in result + assert "event_id" in result + + def test_execute_washer_task(self): + """Test executing washer task""" + executor = DeviceExecutor() + + task_data = { + "task": "wash", + "device": "washer_001", + "description": "Wash laundry", + "duration": 0.01, + } + + result = executor.execute(task_data) + assert result is not None + assert result["status"] == TaskStatus.COMPLETED.value + assert result["success"] is True + + def test_execute_dryer_task(self): + """Test executing dryer task""" + executor = DeviceExecutor() + + task_data = { + "task": "dry", + "device": "dryer_001", + "description": "Dry laundry", + "duration": 0.01, + } + + result = executor.execute(task_data) + assert result is not None + assert result["status"] == TaskStatus.COMPLETED.value + assert result["success"] is True + + def test_execute_humidity_sensor(self): + """Test executing humidity sensor task""" + executor = DeviceExecutor() + + task_data = { + "task": "check_environment", + "device": "humid_sensor_001", + "description": "Check humidity", + "duration": 0.01, + } + + result = executor.execute(task_data) + assert result is not None + assert "humidity" in result + assert DeviceExecutor.MIN_HUMIDITY <= result["humidity"] <= DeviceExecutor.MAX_HUMIDITY + + +class TestEnvironmentMonitor: + """Test EnvironmentMonitor operator""" + + def test_monitor_creation(self): + """Test creating environment monitor""" + monitor = EnvironmentMonitor() + + assert monitor is not None + + def test_monitor_check_environment_optimal(self): + """Test monitor with optimal humidity""" + monitor = EnvironmentMonitor() + + task_data = {"task": "check_environment", "humidity": 55.0} + + result = monitor.execute(task_data) + assert result is not None + assert result["environment_status"] == "optimal" + + def test_monitor_check_environment_too_dry(self): + """Test monitor with low humidity""" + monitor = EnvironmentMonitor() + + task_data = {"task": "check_environment", "humidity": 35.0} + + result = monitor.execute(task_data) + assert result is not None + assert result["environment_status"] == "too_dry" + + def test_monitor_check_environment_too_humid(self): + """Test monitor with high humidity""" + monitor = EnvironmentMonitor() + + task_data = {"task": "check_environment", "humidity": 75.0} + + result = monitor.execute(task_data) + assert result is not None + assert result["environment_status"] == "too_humid" + + def test_monitor_non_environment_task(self): + """Test monitor passes through non-environment tasks""" + monitor = EnvironmentMonitor() + + task_data = {"task": "wash", "device": "washer_001"} + + result = monitor.execute(task_data) + assert result is not None + assert result["task"] == "wash" + + +class TestWorkflowProgressSink: + """Test WorkflowProgressSink operator""" + + def test_sink_creation(self): + """Test creating workflow progress sink""" + sink = WorkflowProgressSink() + + assert sink.tasks_completed == 0 + assert sink.total_steps == 0 + + def test_sink_collects_task(self): + """Test sink collects workflow task""" + sink = WorkflowProgressSink() + + task_data = { + "task": "collect_laundry", + "device": "robot_001", + "cycle": 1, + "step": 2, + "total_steps": 6, + } + + sink.execute(task_data) + assert sink.tasks_completed == 1 + assert sink.total_steps == 6 + + def test_sink_tracks_multiple_tasks(self): + """Test sink tracks multiple workflow tasks""" + sink = WorkflowProgressSink() + + tasks = [ + {"task": "check_environment", "cycle": 1, "step": i + 1, "total_steps": 6} + for i in range(6) + ] + + for task in tasks: + sink.execute(task) + + assert sink.tasks_completed == 6 + + +class TestEventLogSink: + """Test EventLogSink operator""" + + def test_sink_creation(self): + """Test creating event log sink""" + sink = EventLogSink() + + assert len(sink.events) == 0 + + def test_sink_logs_event(self): + """Test sink logs device event""" + sink = EventLogSink() + + event_data = { + "task": "wash", + "device": "washer_001", + "status": TaskStatus.COMPLETED.value, + } + + sink.execute(event_data) + assert len(sink.events) == 1 + assert sink.events[0]["device"] == "washer_001" + + def test_sink_logs_multiple_events(self): + """Test sink logs multiple events""" + sink = EventLogSink() + + events = [{"task": f"task_{i}", "device": "robot_001", "step": i} for i in range(10)] + + for event in events: + sink.execute(event) + + assert len(sink.events) == 10 + + +@pytest.mark.integration +class TestSmartHomePipeline: + """Integration tests for smart home laundry workflow""" + + def test_workflow_execution(self): + """Test complete laundry workflow execution""" + # Create operators + source = LaundryWorkflowSource(num_cycles=1) + executor = DeviceExecutor() + monitor = EnvironmentMonitor() + progress_sink = WorkflowProgressSink() + log_sink = EventLogSink() + + # Execute workflow + while True: + task = source.execute() + if task is None: + break + + # Execute task + result = executor.execute(task) + + # Monitor environment + monitored = monitor.execute(result) + + # Collect results + progress_sink.execute(monitored) + log_sink.execute(monitored) + + # Verify completion + assert progress_sink.tasks_completed == 6 + assert len(log_sink.events) == 6 + + @pytest.mark.skip(reason="Pipeline infrastructure test") + def test_pipeline_creation(self): + """Test creating the smart home pipeline""" + # Placeholder for pipeline creation test + # Would require actual pipeline module + pass diff --git a/sage-apps/tests/test_medical_diagnosis.py b/sage-apps/tests/test_medical_diagnosis.py new file mode 100644 index 0000000..907589d --- /dev/null +++ b/sage-apps/tests/test_medical_diagnosis.py @@ -0,0 +1,136 @@ +""" +Tests for medical diagnosis application +""" + +import importlib.util +from pathlib import Path + +import pytest + + +class TestMedicalDiagnosisStructure: + """Test medical diagnosis app structure""" + + @pytest.fixture + def medical_dir(self): + """Get medical diagnosis directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" + + def test_medical_directory_exists(self, medical_dir): + """Verify medical diagnosis directory exists""" + assert medical_dir.exists(), "Medical diagnosis directory should exist" + + def test_required_files_exist(self, medical_dir): + """Verify required files exist""" + required_files = [ + "__init__.py", + "README.md", + "run_diagnosis.py", + ] + + for file_name in required_files: + file_path = medical_dir / file_name + assert file_path.exists(), f"Required file not found: {file_name}" + + def test_required_directories_exist(self, medical_dir): + """Verify required directories exist""" + required_dirs = [ + "agents", + "config", + "tools", + ] + + for dir_name in required_dirs: + dir_path = medical_dir / dir_name + assert dir_path.exists(), f"Required directory not found: {dir_name}" + + def test_run_diagnosis_not_empty(self, medical_dir): + """Verify run_diagnosis.py is not empty""" + run_file = medical_dir / "run_diagnosis.py" + content = run_file.read_text() + assert len(content.strip()) > 0, "run_diagnosis.py should not be empty" + + def test_run_diagnosis_has_main_or_class(self, medical_dir): + """Verify run_diagnosis.py has executable code""" + run_file = medical_dir / "run_diagnosis.py" + content = run_file.read_text() + + # Check if file has either main() function or class definition + has_main = "def main(" in content or "def main (" in content + has_class = "class " in content + has_if_name = 'if __name__ == "__main__"' in content + + assert has_main or has_class or has_if_name, ( + "run_diagnosis.py should have main() function, class definition, or if __name__ == '__main__'" + ) + + +class TestMedicalDiagnosisImports: + """Test medical diagnosis imports""" + + @pytest.fixture + def medical_dir(self): + """Get medical diagnosis directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" + + def test_run_diagnosis_imports(self, medical_dir): + """Test run_diagnosis.py can be imported""" + run_file = medical_dir / "run_diagnosis.py" + + spec = importlib.util.spec_from_file_location("run_diagnosis", run_file) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + # Module loaded successfully + assert True + except ImportError as e: + # Import errors are expected if dependencies are missing + pytest.skip(f"Skipping due to missing dependencies: {e}") + except Exception as e: + pytest.fail(f"Failed to import run_diagnosis.py: {e}") + + def test_agents_directory_has_content(self, medical_dir): + """Verify agents directory has Python files""" + agents_dir = medical_dir / "agents" + python_files = list(agents_dir.glob("*.py")) + + # Should have at least __init__.py + assert len(python_files) > 0, "agents directory should have Python files" + + def test_tools_directory_has_content(self, medical_dir): + """Verify tools directory has Python files""" + tools_dir = medical_dir / "tools" + python_files = list(tools_dir.glob("*.py")) + + # Should have at least __init__.py + assert len(python_files) > 0, "tools directory should have Python files" + + +class TestMedicalDiagnosisConfig: + """Test medical diagnosis configuration""" + + @pytest.fixture + def config_dir(self): + """Get config directory path""" + return ( + Path(__file__).parent.parent / "src" / "sage" / "apps" / "medical_diagnosis" / "config" + ) + + def test_config_directory_exists(self, config_dir): + """Verify config directory exists""" + assert config_dir.exists(), "Config directory should exist" + + def test_config_files_exist(self, config_dir): + """Verify config files exist""" + config_files = ( + list(config_dir.glob("*.yaml")) + + list(config_dir.glob("*.json")) + + list(config_dir.glob("*.toml")) + ) + + # Should have at least one config file + if len(config_files) > 0: + assert True, "Config directory has configuration files" + else: + pytest.skip("No config files found (might be optional)") diff --git a/sage-apps/tests/test_package.py b/sage-apps/tests/test_package.py new file mode 100644 index 0000000..e4706f6 --- /dev/null +++ b/sage-apps/tests/test_package.py @@ -0,0 +1,17 @@ +"""Test sage.apps package metadata.""" + +from sage.apps import __version__ + + +def test_version(): + """Test package version is defined.""" + assert __version__ is not None + assert isinstance(__version__, str) + assert len(__version__) > 0 + + +def test_package_imports(): + """Test basic package imports work.""" + import sage.apps + + assert hasattr(sage.apps, "__version__") diff --git a/sage-apps/tests/test_video_app.py b/sage-apps/tests/test_video_app.py new file mode 100644 index 0000000..435822e --- /dev/null +++ b/sage-apps/tests/test_video_app.py @@ -0,0 +1,160 @@ +""" +Tests for video application +""" + +import importlib.util +from pathlib import Path + +import pytest + + +class TestVideoAppStructure: + """Test video app structure""" + + @pytest.fixture + def video_dir(self): + """Get video directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "video" + + def test_video_directory_exists(self, video_dir): + """Verify video directory exists""" + assert video_dir.exists(), "Video directory should exist" + + def test_required_files_exist(self, video_dir): + """Verify required files exist""" + required_files = [ + "__init__.py", + "video_intelligence_pipeline.py", + ] + + for file_name in required_files: + file_path = video_dir / file_name + assert file_path.exists(), f"Required file not found: {file_name}" + + def test_required_directories_exist(self, video_dir): + """Verify required directories exist""" + required_dirs = [ + "operators", + "config", + ] + + for dir_name in required_dirs: + dir_path = video_dir / dir_name + assert dir_path.exists(), f"Required directory not found: {dir_name}" + + def test_video_intelligence_pipeline_not_empty(self, video_dir): + """Verify video_intelligence_pipeline.py is not empty""" + pipeline_file = video_dir / "video_intelligence_pipeline.py" + content = pipeline_file.read_text() + assert len(content.strip()) > 0, "video_intelligence_pipeline.py should not be empty" + + def test_video_intelligence_has_main_or_class(self, video_dir): + """Verify video_intelligence_pipeline.py has executable code""" + pipeline_file = video_dir / "video_intelligence_pipeline.py" + content = pipeline_file.read_text() + + # Check if file has either main() function or class definition + has_main = "def main(" in content or "def main (" in content + has_class = "class " in content + has_if_name = 'if __name__ == "__main__"' in content + + assert has_main or has_class or has_if_name, ( + "video_intelligence_pipeline.py should have main() function, class definition, or if __name__ == '__main__'" + ) + + +class TestVideoAppImports: + """Test video app imports""" + + @pytest.fixture + def video_dir(self): + """Get video directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "video" + + def test_video_intelligence_pipeline_imports(self, video_dir): + """Test video_intelligence_pipeline.py can be imported""" + pipeline_file = video_dir / "video_intelligence_pipeline.py" + + spec = importlib.util.spec_from_file_location("video_intelligence_pipeline", pipeline_file) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + # Module loaded successfully + assert True + except ImportError as e: + # Import errors are expected if dependencies are missing + pytest.skip(f"Skipping due to missing dependencies: {e}") + except Exception as e: + pytest.fail(f"Failed to import video_intelligence_pipeline.py: {e}") + + def test_operators_directory_has_content(self, video_dir): + """Verify operators directory has Python files""" + operators_dir = video_dir / "operators" + python_files = list(operators_dir.glob("*.py")) + + # Should have at least __init__.py + assert len(python_files) > 0, "operators directory should have Python files" + + +class TestVideoAppOperators: + """Test video app operators""" + + @pytest.fixture + def operators_dir(self): + """Get operators directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "video" / "operators" + + def test_operators_not_empty(self, operators_dir): + """Verify operator files are not empty""" + python_files = list(operators_dir.glob("*.py")) + python_files = [f for f in python_files if f.name != "__init__.py"] + + for py_file in python_files: + content = py_file.read_text() + assert len(content.strip()) > 0, f"{py_file.name} should not be empty" + + def test_operators_have_classes(self, operators_dir): + """Verify operator files have class definitions""" + python_files = list(operators_dir.glob("*.py")) + python_files = [f for f in python_files if f.name != "__init__.py"] + + if len(python_files) == 0: + pytest.skip("No operator files found") + + for py_file in python_files: + content = py_file.read_text() + + # Check if file has class definition + has_class = "class " in content + + # Operator files should typically have class definitions + if not has_class: + pytest.skip(f"{py_file.name} might be a utility file without classes") + + +class TestVideoAppConfig: + """Test video app configuration""" + + @pytest.fixture + def config_dir(self): + """Get config directory path""" + return Path(__file__).parent.parent / "src" / "sage" / "apps" / "video" / "config" + + def test_config_directory_exists(self, config_dir): + """Verify config directory exists""" + assert config_dir.exists(), "Config directory should exist" + + def test_config_files_exist(self, config_dir): + """Verify config files exist""" + config_files = ( + list(config_dir.glob("*.yaml")) + + list(config_dir.glob("*.json")) + + list(config_dir.glob("*.toml")) + ) + + # Should have at least one config file + if len(config_files) > 0: + assert True, "Config directory has configuration files" + else: + pytest.skip("No config files found (might be optional)") diff --git a/sage-apps/tests/work_report_generator/__init__.py b/sage-apps/tests/work_report_generator/__init__.py new file mode 100644 index 0000000..8b727a0 --- /dev/null +++ b/sage-apps/tests/work_report_generator/__init__.py @@ -0,0 +1 @@ +"""Tests for Work Report Generator module""" diff --git a/sage-apps/tests/work_report_generator/test_models.py b/sage-apps/tests/work_report_generator/test_models.py new file mode 100644 index 0000000..6fda280 --- /dev/null +++ b/sage-apps/tests/work_report_generator/test_models.py @@ -0,0 +1,296 @@ +"""Unit tests for Work Report Generator data models. + +Tests for sage.apps.work_report_generator.models module. +""" + +from sage.apps.work_report_generator.models import ( + ContributorSummary, + DiaryEntry, + GitHubCommit, + GitHubPullRequest, + WeeklyReport, +) + + +class TestGitHubCommit: + """Test GitHubCommit dataclass.""" + + def test_commit_creation(self): + """Test creating a commit.""" + commit = GitHubCommit( + sha="abc123def456", # pragma: allowlist secret + message="feat: Add new feature", + author="developer1", + author_email="dev1@example.com", + committed_date="2024-01-15T10:00:00Z", + repo="intellistream/SAGE", + url="https://github.com/intellistream/SAGE/commit/abc123def456", + additions=100, + deletions=50, + changed_files=5, + ) + + assert commit.sha == "abc123def456" # pragma: allowlist secret + assert commit.message == "feat: Add new feature" + assert commit.author == "developer1" + assert commit.additions == 100 + assert commit.deletions == 50 + + def test_commit_from_graphql(self): + """Test creating commit from GraphQL response.""" + graphql_data = { + "oid": "abc123", + "message": "fix: Bug fix\n\nDetailed description", + "committedDate": "2024-01-15T10:00:00Z", + "url": "https://github.com/test/repo/commit/abc123", + "additions": 10, + "deletions": 5, + "changedFiles": 2, + "author": { + "name": "Test User", + "email": "test@example.com", + "user": {"login": "testuser"}, + }, + } + + commit = GitHubCommit.from_graphql(graphql_data, "test/repo") + + assert commit.sha == "abc123" + assert commit.message == "fix: Bug fix" # Only first line + assert commit.author == "testuser" + assert commit.author_email == "test@example.com" + assert commit.repo == "test/repo" + assert commit.additions == 10 + + def test_commit_from_graphql_missing_user(self): + """Test creating commit when user info is missing.""" + graphql_data = { + "oid": "def456", + "message": "Update file", + "committedDate": "2024-01-15T10:00:00Z", + "url": "https://github.com/test/repo/commit/def456", + "additions": 5, + "deletions": 2, + "changedFiles": 1, + "author": { + "name": "External User", + "email": "external@example.com", + "user": None, + }, + } + + commit = GitHubCommit.from_graphql(graphql_data, "test/repo") + + assert commit.sha == "def456" + assert commit.author == "External User" + + +class TestGitHubPullRequest: + """Test GitHubPullRequest dataclass.""" + + def test_pr_creation(self): + """Test creating a pull request.""" + pr = GitHubPullRequest( + number=100, + title="feat: Add weekly report generator", + author="developer1", + state="MERGED", + created_at="2024-01-15T10:00:00Z", + merged_at="2024-01-16T10:00:00Z", + closed_at=None, + repo="intellistream/SAGE", + url="https://github.com/intellistream/SAGE/pull/100", + additions=500, + deletions=100, + changed_files=15, + labels=["feature", "enhancement"], + reviewers=["reviewer1"], + ) + + assert pr.number == 100 + assert pr.state == "MERGED" + assert len(pr.labels) == 2 + assert pr.merged_at is not None + + def test_pr_from_graphql(self): + """Test creating PR from GraphQL response.""" + graphql_data = { + "number": 42, + "title": "Fix: Resolve issue", + "state": "MERGED", + "createdAt": "2024-01-15T10:00:00Z", + "mergedAt": "2024-01-16T10:00:00Z", + "closedAt": None, + "url": "https://github.com/test/repo/pull/42", + "additions": 20, + "deletions": 10, + "changedFiles": 3, + "author": {"login": "developer"}, + "labels": {"nodes": [{"name": "bug"}, {"name": "priority:high"}]}, + "reviewRequests": {"nodes": []}, + } + + pr = GitHubPullRequest.from_graphql(graphql_data, "test/repo") + + assert pr.number == 42 + assert pr.title == "Fix: Resolve issue" + assert pr.author == "developer" + assert pr.state == "MERGED" + assert "bug" in pr.labels + + +class TestDiaryEntry: + """Test DiaryEntry dataclass.""" + + def test_diary_creation(self): + """Test creating a diary entry.""" + entry = DiaryEntry( + date="2024-01-15", + author="developer1", + content="Today I worked on the report generator.", + tags=["development", "feature"], + category="work", + ) + + assert entry.date == "2024-01-15" + assert entry.author == "developer1" + assert len(entry.tags) == 2 + + def test_diary_from_dict(self): + """Test creating diary entry from dictionary.""" + data = { + "date": "2024-01-15", + "author": "user1", + "content": "Daily standup notes", + "tags": ["meeting"], + "category": "notes", + } + + entry = DiaryEntry.from_dict(data) + + assert entry.date == "2024-01-15" + assert entry.author == "user1" + assert entry.category == "notes" + + def test_diary_from_dict_defaults(self): + """Test diary entry defaults when creating from incomplete dict.""" + data = {"content": "Some content"} + + entry = DiaryEntry.from_dict(data) + + assert entry.author == "Unknown" + assert entry.category == "general" + assert entry.tags == [] + + +class TestContributorSummary: + """Test ContributorSummary dataclass.""" + + def test_contributor_creation(self): + """Test creating a contributor summary.""" + summary = ContributorSummary(username="developer1") + + assert summary.username == "developer1" + assert len(summary.commits) == 0 + assert summary.total_commits == 0 + + def test_contributor_calculate_stats(self): + """Test calculating contributor statistics.""" + summary = ContributorSummary(username="developer1") + + # Add commits + summary.commits = [ + GitHubCommit( + sha="abc", + message="Commit 1", + author="developer1", + author_email="dev@example.com", + committed_date="2024-01-15T10:00:00Z", + repo="test/repo", + url="https://github.com/test/repo/commit/abc", + additions=100, + deletions=50, + changed_files=5, + ), + GitHubCommit( + sha="def", + message="Commit 2", + author="developer1", + author_email="dev@example.com", + committed_date="2024-01-16T10:00:00Z", + repo="test/repo", + url="https://github.com/test/repo/commit/def", + additions=50, + deletions=20, + changed_files=3, + ), + ] + + # Add PRs + summary.pull_requests = [ + GitHubPullRequest( + number=1, + title="PR 1", + author="developer1", + state="MERGED", + created_at="2024-01-15T10:00:00Z", + merged_at="2024-01-16T10:00:00Z", + closed_at=None, + repo="test/repo", + url="https://github.com/test/repo/pull/1", + additions=200, + deletions=100, + ), + ] + + summary.calculate_stats() + + assert summary.total_commits == 2 + assert summary.total_prs == 1 + assert summary.merged_prs == 1 + assert summary.total_additions == 350 # 100 + 50 + 200 + assert summary.total_deletions == 170 # 50 + 20 + 100 + + +class TestWeeklyReport: + """Test WeeklyReport dataclass.""" + + def test_report_creation(self): + """Test creating a weekly report.""" + report = WeeklyReport( + start_date="2024-01-08", + end_date="2024-01-15", + repos=["intellistream/SAGE"], + ) + + assert report.start_date == "2024-01-08" + assert report.end_date == "2024-01-15" + assert len(report.repos) == 1 + assert report.total_commits == 0 + + def test_report_calculate_overall_stats(self): + """Test calculating overall report statistics.""" + report = WeeklyReport( + start_date="2024-01-08", + end_date="2024-01-15", + repos=["test/repo"], + ) + + # Create contributors + contrib1 = ContributorSummary(username="dev1") + contrib1.total_commits = 10 + contrib1.total_prs = 3 + contrib1.merged_prs = 2 + + contrib2 = ContributorSummary(username="dev2") + contrib2.total_commits = 5 + contrib2.total_prs = 2 + contrib2.merged_prs = 1 + + report.contributors = [contrib1, contrib2] + report.calculate_overall_stats() + + assert report.total_commits == 15 + assert report.total_prs == 5 + assert report.total_merged_prs == 3 + assert report.generated_at != "" diff --git a/sage-apps/tests/work_report_generator/test_operators.py b/sage-apps/tests/work_report_generator/test_operators.py new file mode 100644 index 0000000..0b78a0d --- /dev/null +++ b/sage-apps/tests/work_report_generator/test_operators.py @@ -0,0 +1,638 @@ +"""Unit tests for Work Report Generator operators. + +Tests for sage.apps.work_report_generator.operators module. +""" + +import json +import tempfile +from datetime import datetime, timedelta +from pathlib import Path +from unittest.mock import MagicMock, patch + +from sage.apps.work_report_generator.models import ( + ContributorSummary, + DiaryEntry, + GitHubCommit, + GitHubPullRequest, +) +from sage.apps.work_report_generator.operators import ( + ConsoleSink, + ContributorAggregator, + DiaryEntrySource, + GitHubDataSource, + LLMReportGenerator, + ReportSink, +) + + +class TestGitHubDataSource: + """Test GitHubDataSource operator.""" + + def test_source_creation(self): + """Test creating GitHub data source.""" + source = GitHubDataSource( + repos=["intellistream/SAGE"], + days=7, + ) + + assert source.repos == ["intellistream/SAGE"] + assert source.days == 7 + assert source.current_index == 0 + assert source.fetched is False + + def test_source_creation_with_token(self): + """Test creating source with token.""" + source = GitHubDataSource( + repos=["test/repo"], + days=14, + github_token="test_token", + ) + + assert source.github_token == "test_token" + assert "Bearer test_token" in source.headers["Authorization"] + + def test_source_date_range(self): + """Test source calculates correct date range.""" + source = GitHubDataSource(repos=["test/repo"], days=7) + + expected_start = datetime.now() - timedelta(days=7) + assert abs((source.start_date - expected_start).total_seconds()) < 1 + assert abs((source.end_date - datetime.now()).total_seconds()) < 1 + + def test_source_mock_data(self): + """Test source generates mock data when no token.""" + source = GitHubDataSource(repos=["test/repo"], days=7, github_token="") + + source._use_mock_data() + + assert len(source.commits) > 0 + assert len(source.pull_requests) > 0 + + def test_source_execute_returns_commits_then_prs(self): + """Test source emits commits first, then PRs.""" + source = GitHubDataSource(repos=["test/repo"], days=7, github_token="") + + # Use mock data + source._use_mock_data() + source.fetched = True + + # First items should be commits + result = source.execute() + assert result is not None + assert result["type"] == "commit" + + @patch("sage.apps.work_report_generator.operators.requests.post") + def test_source_graphql_error_handling(self, mock_post): + """Test source handles GraphQL errors gracefully.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.raise_for_status.return_value = None + mock_response.json.return_value = {"errors": ["Test error"]} + mock_post.return_value = mock_response + + source = GitHubDataSource( + repos=["test/repo"], + days=7, + github_token="test_token", + ) + + result = source._execute_graphql("query { viewer { login } }") + assert "errors" in result + + +class TestDiaryEntrySource: + """Test DiaryEntrySource operator.""" + + def test_source_creation(self): + """Test creating diary entry source.""" + source = DiaryEntrySource(diary_path="/tmp/diaries", days=7) + + assert source.diary_path == Path("/tmp/diaries") + assert source.days == 7 + assert source.fetched is False + + def test_source_no_path(self): + """Test source with no path specified.""" + source = DiaryEntrySource(diary_path=None, days=7) + + assert source.diary_path is None + + def test_source_load_json_entries(self): + """Test loading entries from JSON file.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create test JSON file + diary_file = Path(tmpdir) / "diaries.json" + entries = [ + { + "date": datetime.now().strftime("%Y-%m-%d"), + "author": "testuser", + "content": "Test entry", + "tags": ["test"], + } + ] + diary_file.write_text(json.dumps(entries)) + + source = DiaryEntrySource(diary_path=diary_file, days=7) + source._load_entries() + + assert len(source.entries) == 1 + assert source.entries[0].author == "testuser" + + def test_source_load_markdown_entries(self): + """Test loading entries from Markdown files.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create test markdown file + date_str = datetime.now().strftime("%Y-%m-%d") + md_file = Path(tmpdir) / f"{date_str}.md" + md_file.write_text("# Daily Log\n\nWorked on tests.") + + source = DiaryEntrySource(diary_path=tmpdir, days=7) + source._load_entries() + + assert len(source.entries) == 1 + assert "Daily Log" in source.entries[0].content + + def test_source_date_range_filter(self): + """Test entries are filtered by date range.""" + source = DiaryEntrySource(diary_path=None, days=7) + + # Date within range + recent_date = datetime.now().strftime("%Y-%m-%d") + assert source._is_in_date_range(recent_date) is True + + # Date outside range + old_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d") + assert source._is_in_date_range(old_date) is False + + def test_source_execute(self): + """Test source execution returns entries one by one.""" + source = DiaryEntrySource(diary_path=None, days=7) + + # Manually add entries + source.entries = [ + DiaryEntry( + date="2024-01-15", + author="user1", + content="Entry 1", + ), + DiaryEntry( + date="2024-01-16", + author="user2", + content="Entry 2", + ), + ] + source.fetched = True + + # Get first entry + result1 = source.execute() + assert result1 is not None + assert result1["type"] == "diary_entry" + assert result1["data"].content == "Entry 1" + + # Get second entry + result2 = source.execute() + assert result2 is not None + assert result2["data"].content == "Entry 2" + + # No more entries + result3 = source.execute() + assert result3 is None + + +class TestContributorAggregator: + """Test ContributorAggregator operator.""" + + def setup_method(self): + """Reset aggregator state before each test.""" + ContributorAggregator.reset() + + def test_aggregator_creation(self): + """Test creating aggregator.""" + aggregator = ContributorAggregator() + assert aggregator is not None + + def test_aggregator_commit(self): + """Test aggregating commits.""" + aggregator = ContributorAggregator() + + commit = GitHubCommit( + sha="abc123", + message="Test commit", + author="developer1", + author_email="dev@example.com", + committed_date="2024-01-15T10:00:00Z", + repo="test/repo", + url="https://github.com/test/repo/commit/abc123", + ) + + result = aggregator.execute( + { + "type": "commit", + "data": commit, + "author": "developer1", + } + ) + + assert result["author"] == "developer1" + assert result["contributor"].total_commits == 1 + + def test_aggregator_pull_request(self): + """Test aggregating pull requests.""" + aggregator = ContributorAggregator() + + pr = GitHubPullRequest( + number=1, + title="Test PR", + author="developer1", + state="MERGED", + created_at="2024-01-15T10:00:00Z", + merged_at="2024-01-16T10:00:00Z", + closed_at=None, + repo="test/repo", + url="https://github.com/test/repo/pull/1", + ) + + result = aggregator.execute( + { + "type": "pull_request", + "data": pr, + "author": "developer1", + } + ) + + assert result["contributor"].total_prs == 1 + assert result["contributor"].merged_prs == 1 + + def test_aggregator_diary_entry(self): + """Test aggregating diary entries.""" + aggregator = ContributorAggregator() + + entry = DiaryEntry( + date="2024-01-15", + author="developer1", + content="Test entry", + ) + + result = aggregator.execute( + { + "type": "diary_entry", + "data": entry, + "author": "developer1", + } + ) + + assert len(result["contributor"].diary_entries) == 1 + + def test_aggregator_multiple_contributors(self): + """Test aggregating data for multiple contributors.""" + aggregator = ContributorAggregator() + + # Add data for dev1 + commit1 = GitHubCommit( + sha="abc", + message="Commit 1", + author="dev1", + author_email="", + committed_date="2024-01-15T10:00:00Z", + repo="test/repo", + url="", + ) + aggregator.execute({"type": "commit", "data": commit1, "author": "dev1"}) + + # Add data for dev2 + commit2 = GitHubCommit( + sha="def", + message="Commit 2", + author="dev2", + author_email="", + committed_date="2024-01-15T11:00:00Z", + repo="test/repo", + url="", + ) + aggregator.execute({"type": "commit", "data": commit2, "author": "dev2"}) + + contributors = ContributorAggregator.get_all_contributors() + assert len(contributors) == 2 + assert "dev1" in contributors + assert "dev2" in contributors + + def test_aggregator_reset(self): + """Test resetting aggregator state.""" + aggregator = ContributorAggregator() + + commit = GitHubCommit( + sha="abc", + message="Test", + author="dev1", + author_email="", + committed_date="", + repo="", + url="", + ) + aggregator.execute({"type": "commit", "data": commit, "author": "dev1"}) + + assert len(ContributorAggregator.get_all_contributors()) > 0 + + ContributorAggregator.reset() + + assert len(ContributorAggregator.get_all_contributors()) == 0 + + def test_aggregator_username_normalization(self): + """Test username normalization for duplicate contributors.""" + ContributorAggregator.reset() + aggregator = ContributorAggregator() + + # Test ShuhaoZhangTony variants + commit1 = GitHubCommit( + sha="abc1", + message="Commit from ShuhaoZhangTony", + author="ShuhaoZhangTony", + author_email="", + committed_date="2024-01-15T10:00:00Z", + repo="test/repo", + url="", + ) + aggregator.execute({"type": "commit", "data": commit1, "author": "ShuhaoZhangTony"}) + + commit2 = GitHubCommit( + sha="abc2", + message="Commit from Shuhao Zhang", + author="Shuhao Zhang", + author_email="", + committed_date="2024-01-15T11:00:00Z", + repo="test/repo", + url="", + ) + result = aggregator.execute({"type": "commit", "data": commit2, "author": "Shuhao Zhang"}) + + # Both should map to ShuhaoZhangTony + assert result["author"] == "ShuhaoZhangTony" + contributors = ContributorAggregator.get_all_contributors() + assert "ShuhaoZhangTony" in contributors + assert "Shuhao Zhang" not in contributors + assert contributors["ShuhaoZhangTony"].total_commits == 2 + + def test_aggregator_copilot_normalization(self): + """Test Copilot username normalization.""" + ContributorAggregator.reset() + aggregator = ContributorAggregator() + + commit1 = GitHubCommit( + sha="cop1", + message="Commit from Copilot", + author="Copilot", + author_email="", + committed_date="2024-01-15T10:00:00Z", + repo="test/repo", + url="", + ) + aggregator.execute({"type": "commit", "data": commit1, "author": "Copilot"}) + + commit2 = GitHubCommit( + sha="cop2", + message="Commit from copilot-swe-agent", + author="copilot-swe-agent", + author_email="", + committed_date="2024-01-15T11:00:00Z", + repo="test/repo", + url="", + ) + result = aggregator.execute( + {"type": "commit", "data": commit2, "author": "copilot-swe-agent"} + ) + + # Both should map to Copilot + assert result["author"] == "Copilot" + contributors = ContributorAggregator.get_all_contributors() + assert "Copilot" in contributors + assert "copilot-swe-agent" not in contributors + assert contributors["Copilot"].total_commits == 2 + + def test_normalize_username_static_method(self): + """Test the normalize_username static method.""" + # ShuhaoZhangTony variants + assert ContributorAggregator.normalize_username("ShuhaoZhangTony") == "ShuhaoZhangTony" + assert ContributorAggregator.normalize_username("Shuhao Zhang") == "ShuhaoZhangTony" + assert ContributorAggregator.normalize_username("shuhao zhang") == "ShuhaoZhangTony" + assert ContributorAggregator.normalize_username("shuhaozhangtony") == "ShuhaoZhangTony" + + # Copilot variants + assert ContributorAggregator.normalize_username("Copilot") == "Copilot" + assert ContributorAggregator.normalize_username("copilot-swe-agent") == "Copilot" + assert ContributorAggregator.normalize_username("github-copilot") == "Copilot" + + # Unknown users should stay unchanged + assert ContributorAggregator.normalize_username("some-random-user") == "some-random-user" + + # Empty string should return Unknown + assert ContributorAggregator.normalize_username("") == "Unknown" + + +class TestLLMReportGenerator: + """Test LLMReportGenerator operator.""" + + def test_generator_creation(self): + """Test creating LLM report generator.""" + generator = LLMReportGenerator(language="zh") + + assert generator.language == "zh" + assert generator._client is None + + def test_generator_simple_summary_zh(self): + """Test generating simple Chinese summary.""" + generator = LLMReportGenerator(language="zh") + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 10 + contributor.total_prs = 3 + contributor.merged_prs = 2 + contributor.total_additions = 500 + contributor.total_deletions = 100 + + summary = generator._generate_simple_summary(contributor) + + assert "testuser" in summary + assert "10" in summary + assert "3" in summary + + def test_generator_simple_summary_en(self): + """Test generating simple English summary.""" + generator = LLMReportGenerator(language="en") + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 5 + contributor.total_prs = 2 + contributor.merged_prs = 1 + contributor.total_additions = 200 + contributor.total_deletions = 50 + + summary = generator._generate_simple_summary(contributor) + + assert "testuser" in summary + assert "contributed" in summary + + def test_generator_execute_non_contributor(self): + """Test generator passes through non-contributor data.""" + generator = LLMReportGenerator(language="zh") + + result = generator.execute({"type": "other", "data": "test"}) + + assert result["type"] == "other" + + def test_generator_execute_contributor(self): + """Test generator processes contributor updates.""" + generator = LLMReportGenerator(language="zh") + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 5 + + result = generator.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + } + ) + + assert "llm_summary" in result + + def test_generator_skips_duplicate_authors(self): + """Test generator only processes each author once.""" + generator = LLMReportGenerator(language="zh") + + contributor = ContributorSummary(username="testuser") + + # First call + result1 = generator.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + } + ) + assert result1 is not None + + # Second call should return None + result2 = generator.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + } + ) + assert result2 is None + + +class TestReportSink: + """Test ReportSink operator.""" + + def test_sink_creation(self): + """Test creating report sink.""" + sink = ReportSink( + output_format="markdown", + repos=["test/repo"], + days=7, + ) + + assert sink.output_format == "markdown" + assert sink.repos == ["test/repo"] + assert sink.days == 7 + + def test_sink_execute_collects_data(self): + """Test sink collects contributor data.""" + sink = ReportSink(output_format="console", repos=["test/repo"], days=7) + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 5 + + sink.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + "llm_summary": "Test summary", + } + ) + + assert "testuser" in sink.contributors + + def test_sink_markdown_output(self): + """Test generating markdown output.""" + with tempfile.NamedTemporaryFile(suffix=".md", delete=False) as f: + output_path = Path(f.name) + + try: + sink = ReportSink( + output_format="markdown", + output_path=output_path, + repos=["test/repo"], + days=7, + ) + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 5 + contributor.calculate_stats() + + sink.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + } + ) + + sink.close() + + content = output_path.read_text() + assert "Weekly Work Report" in content + assert "testuser" in content + finally: + output_path.unlink(missing_ok=True) + + def test_sink_json_output(self): + """Test generating JSON output.""" + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as f: + output_path = Path(f.name) + + try: + sink = ReportSink( + output_format="json", + output_path=output_path, + repos=["test/repo"], + days=7, + ) + + contributor = ContributorSummary(username="testuser") + contributor.total_commits = 5 + contributor.calculate_stats() + + sink.execute( + { + "type": "contributor_update", + "author": "testuser", + "contributor": contributor, + } + ) + + sink.close() + + content = output_path.read_text() + data = json.loads(content) + assert "contributors" in data + assert len(data["contributors"]) == 1 + finally: + output_path.unlink(missing_ok=True) + + +class TestConsoleSink: + """Test ConsoleSink operator.""" + + def test_console_sink_counts(self): + """Test console sink counts processed items.""" + sink = ConsoleSink() + + sink.execute({"type": "commit", "author": "dev1"}) + assert sink.count == 1 + + sink.execute({"type": "pr", "author": "dev2"}) + assert sink.count == 2 diff --git a/tutorials/L1-common/README.md b/tutorials/L1-common/README.md new file mode 100644 index 0000000..da41d02 --- /dev/null +++ b/tutorials/L1-common/README.md @@ -0,0 +1,92 @@ +# L1: Common - 基础层示例 + +> 对应 SAGE 包：`sage-common` + +## 📖 层级说明 + +**Common** 层是 SAGE 的基础设施层，提供所有其他层依赖的核心功能： + +- 配置管理系统 +- 日志系统 +- 类型定义 +- 工具函数 +- **统一推理客户端** (UnifiedInferenceClient) + +## 📚 包含示例 + +### 基础入门 + +- `hello_world.py` - 最简单的 SAGE 程序 + +### 统一推理客户端 + +- `unified_inference_client_example.py` - **推荐** 使用 UnifiedInferenceClient 进行 LLM 对话和 Embedding + +### Embedding 服务 + +- `embedding_server_example.py` - 本地 Embedding 服务器使用示例 + +### 配置系统 + +- `config_demo.py` - 配置文件加载和管理 + +### 日志系统 + +- `logging_demo.py` - 日志配置和使用 + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. SAGE 的基本概念和术语 +1. 如何使用 **UnifiedInferenceClient** 进行 LLM 对话和 Embedding +1. 如何使用配置系统 +1. 如何使用日志系统 +1. 为后续学习 Kernel 层做好准备 + +## 🚀 快速开始 (UnifiedInferenceClient) + +### 1. 启动服务 + +```bash +# 启动 Gateway（包含 Control Plane） +sage gateway start + +# 启动 LLM 引擎 +sage llm engine start Qwen/Qwen2.5-0.5B-Instruct + +# (可选) 启动 Embedding 引擎 +sage llm engine start BAAI/bge-m3 --engine-kind embedding +``` + +### 2. 运行示例 + +```bash +python examples/tutorials/L1-common/unified_inference_client_example.py +``` + +### 3. 或直接使用 Python + +```python +from sage.llm import UnifiedInferenceClient + +# 连接到 Gateway Control Plane +client = UnifiedInferenceClient.create( + control_plane_url="http://localhost:8000/v1" +) + +# 对话 +response = client.chat([{"role": "user", "content": "Hello"}]) +print(response) + +# Embedding +vectors = client.embed(["text1", "text2"]) +print(f"向量维度: {len(vectors[0])}") +``` + +## ⏭️ 下一步 + +学完基础层后，继续学习： + +- **L2-platform/** - 平台服务 +- **L3-kernel/** - 核心 API 和流处理引擎 diff --git a/tutorials/L1-common/embedding_server_example.py b/tutorials/L1-common/embedding_server_example.py new file mode 100644 index 0000000..0ae151f --- /dev/null +++ b/tutorials/L1-common/embedding_server_example.py @@ -0,0 +1,224 @@ +""" +Embedding Server 使用示例 + +这个示例展示如何使用本地 embedding 服务器， +无需修改任何原有代码，直接通过 apply_embedding_model 调用。 + +Requirements: + pip install isage-common>=0.2.0 + +Test Configuration: + @test_category: tutorials + @test_speed: quick + @test_skip_ci: true + @test:skip - Requires embedding server to be running +""" + +import os +import sys +from pathlib import Path + +from sage.common.components.sage_embedding.embedding_api import apply_embedding_model +from sage.common.config.ports import SagePorts + +# Check test mode +_IS_TEST_MODE = ( + os.getenv("SAGE_TEST_MODE") == "true" + or os.getenv("SAGE_EXAMPLES_MODE") == "test" + or os.getenv("CI") == "true" +) + +# Use SagePorts for embedding port configuration +EMBEDDING_PORT = SagePorts.EMBEDDING_SECONDARY # 8091 + + +def example_basic_usage(): + """基本使用示例""" + print("=" * 60) + print("示例 1: 基本使用") + print("=" * 60) + + # In test mode, just show configuration + if _IS_TEST_MODE: + print("\n[Test mode] Would connect to embedding server:") + print(f" URL: http://localhost:{EMBEDDING_PORT}/v1") + print(" Model: BAAI/bge-m3") + print(" Skipping actual server connection") + return + + # 创建 embedding 模型实例（连接到本地服务器） + embedding_model = apply_embedding_model( + name="openai", # 使用 openai 方法（兼容 OpenAI API） + model="BAAI/bge-m3", # 模型名称（任意，服务器会忽略） + base_url=f"http://localhost:{EMBEDDING_PORT}/v1", # 本地服务器地址 + api_key="dummy", # 本地服务不需要真实的 API key + ) + + # 测试 embedding + text = "Hello, this is a test sentence." + print(f"\nInput text: {text}") + + embedding = embedding_model.embed(text) + print(f"Embedding dimension: {len(embedding)}") + print(f"First 5 values: {embedding[:5]}") + + +def example_batch_processing(): + """批量处理示例""" + print("\n" + "=" * 60) + print("示例 2: 批量处理") + print("=" * 60) + + # 多个文本 + texts = [ + "What is machine learning?", + "Deep learning is a subset of machine learning.", + "Natural language processing uses neural networks.", + ] + + # In test mode, just show configuration + if _IS_TEST_MODE: + print(f"\n[Test mode] Would process {len(texts)} texts:") + for text in texts: + print(f" - {text}") + print(" Skipping actual server connection") + return + + embedding_model = apply_embedding_model( + name="openai", + model="BAAI/bge-m3", + base_url=f"http://localhost:{EMBEDDING_PORT}/v1", + api_key="dummy", + ) + + print(f"\nProcessing {len(texts)} texts...") + for i, text in enumerate(texts, 1): + embedding = embedding_model.embed(text) + print(f"{i}. Text: '{text[:50]}...' -> Embedding dim: {len(embedding)}") + + +def example_with_different_server(): + """使用不同服务器端口的示例""" + print("\n" + "=" * 60) + print("示例 3: 使用不同服务器") + print("=" * 60) + + # In test mode, just show configuration + if _IS_TEST_MODE: + print("\n[Test mode] Would connect to custom server:") + print(" URL: http://localhost:8081/v1") + print(" Model: custom-model") + print(" Skipping actual server connection") + return + + # 假设你在端口 8081 运行另一个模型 + embedding_model = apply_embedding_model( + name="openai", + model="custom-model", + base_url="http://localhost:8081/v1", # 不同端口 + api_key="dummy", + ) + + text = "Testing with different server port" + try: + embedding = embedding_model.embed(text) + print(f"Success. Embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error (expected if server not running): {e}") + + +def example_error_handling(): + """错误处理示例""" + print("\n" + "=" * 60) + print("示例 4: 错误处理") + print("=" * 60) + + # In test mode, just show configuration + if _IS_TEST_MODE: + print("\n[Test mode] Would test error handling:") + print(" - Empty text handling") + print(" - Long text (truncation) handling") + print(" Skipping actual server connection") + return + + embedding_model = apply_embedding_model( + name="openai", + model="BAAI/bge-m3", + base_url=f"http://localhost:{EMBEDDING_PORT}/v1", + api_key="dummy", + ) + + # 测试空文本 + try: + embedding = embedding_model.embed("") + print(f"Empty text embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error with empty text: {e}") + + # 测试很长的文本（会被截断） + long_text = "This is a very long sentence. " * 100 + try: + embedding = embedding_model.embed(long_text) + print(f"Long text (truncated) embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error with long text: {e}") + + +def main(): + """主函数""" + + # In test mode, skip early + if _IS_TEST_MODE: + print("\n" + "=" * 60) + print("Embedding Server 使用示例 [Test Mode]") + print("=" * 60) + print("\n[Test mode] Running examples without server connection") + + example_basic_usage() + example_batch_processing() + example_with_different_server() + example_error_handling() + + print("\n" + "=" * 60) + print("所有示例完成. (Test mode - no actual server calls)") + print("=" * 60) + return + + print("\n") + print("=" * 60) + print("Embedding Server 使用示例") + print("=" * 60) + print("\n请确保 embedding 服务器已启动:") + print( + " bash packages/sage-common/src/sage/common/components/sage_embedding/start_embedding_server.sh" + f" {EMBEDDING_PORT}" + ) + print("\n或手动启动:") + print( + " python packages/sage-common/src/sage/common/components/sage_embedding/embedding_server.py" + f" --model BAAI/bge-m3 --port {EMBEDDING_PORT}" + ) + print("\n" + "=" * 60 + "\n") + + try: + # 运行示例 + example_basic_usage() + example_batch_processing() + example_with_different_server() + example_error_handling() + + print("\n" + "=" * 60) + print("所有示例完成.") + print("=" * 60) + + except Exception as e: + print(f"\n错误: {e}") + print("\n请确保 embedding 服务器正在运行:") + print( + " bash packages/sage-common/src/sage/common/components/sage_embedding/start_embedding_server.sh" + f" {EMBEDDING_PORT}" + ) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L1-common/hello_world.py b/tutorials/L1-common/hello_world.py new file mode 100644 index 0000000..6dcbfab --- /dev/null +++ b/tutorials/L1-common/hello_world.py @@ -0,0 +1,47 @@ +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 批处理数据源：作用是生成10条"Hello, World!"字符串 +class HelloBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 10 # 生成10个数据包后返回None + + def execute(self): + if self.counter >= self.max_count: + return None # 返回None表示批处理完成 + self.counter += 1 + return f"Hello, World! #{self.counter}" + + +# 简单的 MapFunction，将内容转大写 +class UpperCaseMap(MapFunction): + def execute(self, data): + return data.upper() + + +# 简单 SinkFunction，直接打印结果 +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +def main(): + env = LocalEnvironment("Hello_World") + + # 批处理源 -> map -> sink + env.from_batch(HelloBatch).map(UpperCaseMap).sink(PrintSink) + + env.submit(autostop=True) + print("Hello World 批处理示例结束") + + +if __name__ == "__main__": + # 关闭日志输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L1-common/unified_inference_client_example.py b/tutorials/L1-common/unified_inference_client_example.py new file mode 100644 index 0000000..d72ac91 --- /dev/null +++ b/tutorials/L1-common/unified_inference_client_example.py @@ -0,0 +1,281 @@ +""" +UnifiedInferenceClient 完整使用示例 + +本示例演示如何使用 SAGE 统一推理客户端进行 LLM 对话和 Embedding 向量化。 + +Requirements: + pip install isage-llm-core>=0.2.0 + +Prerequisites: + 1. 启动 Gateway: `sage gateway start` + 2. 启动 LLM 引擎: `sage llm engine start Qwen/Qwen2.5-0.5B-Instruct` + 3. 启动 Embedding 引擎: `sage llm engine start BAAI/bge-m3 --engine-kind embedding` + +Test Configuration: + @test_category: tutorials + @test_speed: quick + @test_skip_ci: true + @test:skip - Requires Gateway and engines to be running +""" + +from __future__ import annotations + +import math +import os +import sys +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from sage.common.config.ports import SagePorts +from sage.llm import UnifiedInferenceClient + +if TYPE_CHECKING: + pass + +# Check test mode +_IS_TEST_MODE = ( + os.getenv("SAGE_TEST_MODE") == "true" + or os.getenv("SAGE_EXAMPLES_MODE") == "test" + or os.getenv("CI") == "true" +) + + +def _cosine_similarity(v1: list[float], v2: list[float]) -> float: + """计算两个向量的余弦相似度.""" + dot = sum(a * b for a, b in zip(v1, v2)) + norm1 = math.sqrt(sum(a * a for a in v1)) + norm2 = math.sqrt(sum(b * b for b in v2)) + return dot / (norm1 * norm2) if norm1 * norm2 > 0 else 0 + + +def example_basic_chat(): + """示例 1: 基础对话""" + print("=" * 60) + print("示例 1: 基础对话") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Would connect to Gateway:") + print(f" URL: http://localhost:{SagePorts.GATEWAY_DEFAULT}/v1") + print(" Skipping actual connection") + return + + # 创建客户端 - 连接到 Gateway Control Plane + client = UnifiedInferenceClient.create( + control_plane_url=f"http://localhost:{SagePorts.GATEWAY_DEFAULT}/v1" + ) + + # 单轮对话 + messages = [{"role": "user", "content": "用一句话介绍什么是人工智能"}] + response = client.chat(messages) + + print(f"\n用户: {messages[0]['content']}") + print(f"助手: {response}") + + +def example_multi_turn_conversation(): + """示例 2: 多轮对话""" + print("\n" + "=" * 60) + print("示例 2: 多轮对话") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Multi-turn conversation demo") + print(" Would send multiple messages to LLM") + print(" Skipping actual connection") + return + + client = UnifiedInferenceClient.create( + control_plane_url=f"http://localhost:{SagePorts.GATEWAY_DEFAULT}/v1" + ) + + # 构建多轮对话历史 + messages = [ + {"role": "user", "content": "什么是机器学习?"}, + ] + + response1 = cast(str, client.chat(messages)) + print(f"\n用户: {messages[0]['content']}") + print(f"助手: {response1}") + + # 添加助手回复和新的用户问题 + messages.append({"role": "assistant", "content": response1}) + messages.append({"role": "user", "content": "它和深度学习有什么区别?"}) + + response2 = client.chat(messages) + print(f"\n用户: {messages[-1]['content']}") + print(f"助手: {response2}") + + +def example_embedding(): + """示例 3: 文本向量化""" + print("\n" + "=" * 60) + print("示例 3: 文本向量化 (Embedding)") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Embedding demo") + print(" Would generate embeddings for sample texts") + print(" Skipping actual connection") + return + + client = UnifiedInferenceClient.create( + control_plane_url=f"http://localhost:{SagePorts.GATEWAY_DEFAULT}/v1" + ) + + # 批量向量化 + texts = [ + "人工智能是计算机科学的一个分支", + "机器学习是人工智能的子集", + "深度学习使用多层神经网络", + ] + + vectors = cast(list[list[float]], client.embed(texts)) + + print(f"\n处理了 {len(texts)} 个文本:") + for i, (text, vec) in enumerate(zip(texts, vectors), 1): + print(f" {i}. '{text[:30]}...' -> 向量维度: {len(vec)}") + + print("\n文本相似度:") + sim_01 = _cosine_similarity(vectors[0], vectors[1]) + sim_02 = _cosine_similarity(vectors[0], vectors[2]) + sim_12 = _cosine_similarity(vectors[1], vectors[2]) + print(f" 文本1 vs 文本2: {sim_01:.4f}") + print(f" 文本1 vs 文本3: {sim_02:.4f}") + print(f" 文本2 vs 文本3: {sim_12:.4f}") + + +def example_combined_workflow(): + """示例 4: 综合工作流 (Chat + Embedding)""" + print("\n" + "=" * 60) + print("示例 4: 综合工作流 (RAG 简化示例)") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Combined workflow demo") + print(" Would demonstrate RAG-like pipeline") + print(" Skipping actual connection") + return + + client = UnifiedInferenceClient.create( + control_plane_url=f"http://localhost:{SagePorts.GATEWAY_DEFAULT}/v1" + ) + + # 模拟知识库 + knowledge_base = [ + "SAGE 是一个用于构建 AI/LLM 数据处理管道的 Python 框架", + "SAGE 支持声明式数据流定义", + "SAGE Gateway 提供统一的 API 入口和 Control Plane 引擎管理", + "UnifiedInferenceClient 是 SAGE 的统一推理客户端", + ] + + # 用户问题 + query = "什么是 SAGE Gateway?" + print(f"\n用户问题: {query}") + + # Step 1: 向量化问题和知识库 + print("\nStep 1: 向量化...") + query_vector = cast(list[list[float]], client.embed([query]))[0] + kb_vectors = cast(list[list[float]], client.embed(knowledge_base)) + + # Step 2: 检索最相关文档 + similarities = [_cosine_similarity(query_vector, kv) for kv in kb_vectors] + best_idx = similarities.index(max(similarities)) + context = knowledge_base[best_idx] + print(f"Step 2: 检索到相关文档: '{context}'") + + # Step 3: 生成回答 + print("Step 3: 生成回答...") + messages = [ + { + "role": "system", + "content": f"根据以下上下文回答用户问题。\n\n上下文: {context}", + }, + {"role": "user", "content": query}, + ] + response = client.chat(messages) + print(f"\n最终回答: {response}") + + +def example_auto_detection(): + """示例 5: 自动检测模式""" + print("\n" + "=" * 60) + print("示例 5: 自动检测模式") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Auto-detection demo") + print(" Would try to auto-detect services") + print(" Skipping actual connection") + return + + # 自动检测可用服务 + # 会依次尝试: Control Plane (8000) -> 本地服务 (8901/8090) -> 云端 API + client = UnifiedInferenceClient.create() + + print("\n自动检测到的服务配置:") + print(f" LLM available: {client._llm_available}") + print(f" Embedding available: {client._embedding_available}") + + # 使用自动检测的服务 + response = cast(str, client.chat([{"role": "user", "content": "Hello"}])) + print(f"\n测试响应: {response[:100]}...") + + +def example_error_handling(): + """示例 6: 错误处理""" + print("\n" + "=" * 60) + print("示例 6: 错误处理") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test mode] Error handling demo") + print(" Would demonstrate exception handling") + return + + try: + # 尝试连接不存在的服务 + client = UnifiedInferenceClient.create(control_plane_url="http://localhost:9999/v1") + client.chat([{"role": "user", "content": "test"}]) + except ConnectionError as e: + print(f"\n捕获到连接错误: {e}") + print("提示: 请确保 Gateway 已启动 (sage gateway start)") + except Exception as e: + print(f"\n捕获到错误: {type(e).__name__}: {e}") + + +def main(): + """运行所有示例""" + print("\n" + "=" * 60) + print("SAGE UnifiedInferenceClient 完整示例") + print("=" * 60) + + if _IS_TEST_MODE: + print("\n[Test Mode] 运行模式检测") + print(" SAGE_TEST_MODE:", os.getenv("SAGE_TEST_MODE")) + print(" SAGE_EXAMPLES_MODE:", os.getenv("SAGE_EXAMPLES_MODE")) + print(" CI:", os.getenv("CI")) + print("\n将跳过实际服务连接，仅显示配置信息") + else: + print("\n运行前请确保:") + print(" 1. Gateway 已启动: sage gateway start") + print(" 2. LLM 引擎已启动: sage llm engine start Qwen/Qwen2.5-0.5B-Instruct") + print( + " 3. Embedding 引擎已启动: sage llm engine start BAAI/bge-m3 --engine-kind embedding" + ) + print("\n如需跳过连接测试，设置环境变量: SAGE_TEST_MODE=true") + + example_basic_chat() + example_multi_turn_conversation() + example_embedding() + example_combined_workflow() + example_auto_detection() + example_error_handling() + + print("\n" + "=" * 60) + print("所有示例执行完毕") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L2-platform/README.md b/tutorials/L2-platform/README.md new file mode 100644 index 0000000..cd422eb --- /dev/null +++ b/tutorials/L2-platform/README.md @@ -0,0 +1,48 @@ +# L2: Platform - 平台服务层示例 + +> 对应 SAGE 包：`sage-platform` + +## 📖 层级说明 + +**Platform** 层提供平台服务抽象： + +- 消息队列抽象 (Queue Descriptor) +- KV 存储后端 (Storage Backend) +- 服务基类 (BaseService) +- 调度系统 (Scheduler) + +## 📚 目录结构 + +``` +L5-platform/ +├── scheduler/ # 调度系统示例 +└── deployment/ # 部署方案示例 +``` + +## 🎯 学习路径 + +### 1️⃣ 调度系统 (`scheduler/`) + +理解任务调度： + +- `scheduler_comparison.py` - 调度器对比 +- `remote_env.py` - 远程环境 + +### 2️⃣ 部署方案 (`deployment/`) + +生产环境部署（待添加） + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. SAGE 的调度机制 +1. 分布式执行环境 +1. 生产部署的最佳实践 + +## ⏭️ 下一步 + +学完平台层后，继续学习： + +- **L3-kernel/** - 流式执行引擎 +- **L3-libs/** - 算法库和工具 diff --git a/tutorials/L2-platform/environment/README.md b/tutorials/L2-platform/environment/README.md new file mode 100644 index 0000000..516e46c --- /dev/null +++ b/tutorials/L2-platform/environment/README.md @@ -0,0 +1,28 @@ +# L2-Platform: Environment Examples + +本目录包含 SAGE Platform 层的 Environment 相关示例。 + +## 📂 示例列表 + +### RemoteEnvironment + +- **remote_env.py** - RemoteEnvironment 基础使用示例 + - 演示如何创建和使用 RemoteEnvironment + - 展示分布式执行的基本概念 + +## 🎯 学习路径 + +1. **remote_env.py** - 了解 RemoteEnvironment 的基本用法 + +## 💡 核心概念 + +### RemoteEnvironment + +- 分布式执行环境 +- 支持跨节点的任务调度 +- 适用于大规模数据处理 + +## 🔗 相关资源 + +- 调度器示例请参考：`L3-kernel/scheduler/`（如果需要） +- 部署示例：`L2-platform/deployment/` diff --git a/tutorials/L2-platform/environment/remote_env.py b/tutorials/L2-platform/environment/remote_env.py new file mode 100644 index 0000000..588bca5 --- /dev/null +++ b/tutorials/L2-platform/environment/remote_env.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +""" +RemoteEnvironment 简单示例 +演示如何使用 RemoteEnvironment 和调度器 + +# test_tags: category=environment, timeout=120, requires_daemon=jobmanager +""" + +import os +import socket +import time + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.remote_environment import RemoteEnvironment + + +class SimpleSource(SourceFunction): + """简单数据源""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.count = 0 + self.max_count = 500 # 增加数据量以便观察分布式效果 + + def execute(self, data=None): + if self.count >= self.max_count: + from sage.kernel.runtime.communication.packet import StopSignal + + return StopSignal("SimpleSource completed") + + data = f"item_{self.count}" + self.count += 1 + return data + + +class SimpleProcessor(MapFunction): + """简单处理器 - 记录运行节点""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + import socket as _socket # 在类内部导入，确保 Ray Actor 可以访问 + + self.hostname = _socket.gethostname() + self.processed_count = 0 + + def execute(self, data): + # 跳过非字符串数据（如 StopSignal） + if not isinstance(data, str): + return data + self.processed_count += 1 + # 在结果中包含处理节点信息 + result = f"{data.upper()} [processed on {self.hostname}]" + return result + + +class ConsoleSink(SinkFunction): + """控制台输出 - 统计节点分布""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.test_mode = ( + os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true" + ) + self.count = 0 + self.node_stats = {} # 统计各节点处理数量 + + def execute(self, data): + if data and isinstance(data, str): + self.count += 1 + # 提取节点信息 + if "[processed on " in data: + node = data.split("[processed on ")[-1].rstrip("]") + self.node_stats[node] = self.node_stats.get(node, 0) + 1 + + # 测试模式下仅打印前5条 + if not self.test_mode or self.count <= 5: + print(f"✅ Result: {data}") + elif self.count == 6: + print(" ... (remaining output suppressed in test mode)") + + # 每100条打印一次统计 + if self.count % 100 == 0: + print(f"\n📊 节点分布统计 (已处理 {self.count} 条):") + for node, cnt in sorted(self.node_stats.items()): + print(f" {node}: {cnt} ({cnt * 100 / self.count:.1f}%)") + print() + + +def check_jobmanager_available(): + """检查 JobManager 是否可用""" + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + result = sock.connect_ex(("localhost", 19001)) + sock.close() + return result == 0 + except Exception: + return False + + +def example_default_scheduler(): + """示例 1: 使用分布式调度器 (LoadAware + SPREAD 策略)""" + print("\n" + "=" * 60) + print("示例 1: 分布式调度演示") + print("=" * 60 + "\n") + + # 检查是否在测试模式 + test_mode = os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true" + + # 检查 JobManager 是否可用 + if not check_jobmanager_available(): + if test_mode: + # 在测试模式下，如果JobManager不可用，跳过测试 + print("⚠️ JobManager daemon 不可用，跳过测试") + print(" (在生产环境中需要先启动: sage jobmanager start)") + return + else: + print("❌ 错误: JobManager daemon 未运行") + print(" 请先启动: sage jobmanager start") + return + + # 📊 开始计时 + total_start = time.time() + + # 步骤1: 创建环境 - 使用 load_aware 调度器和 spread 策略 + print("📦 [1/5] 创建 RemoteEnvironment (使用 load_aware 调度器)...") + step_start = time.time() + + # 使用 LoadAwareScheduler 配置分散策略 + from sage.kernel.scheduler.impl import LoadAwareScheduler + + scheduler = LoadAwareScheduler( + platform="remote", + max_concurrent=20, # 增加并发数 + strategy="spread", # 使用 SPREAD 策略分散到不同节点 + ) + + env = RemoteEnvironment( + name="distributed_scheduler_demo", scheduler=scheduler, host="sage-node-1" + ) + # 设置 JobManager 的可访问主机名（worker 节点通过此地址连接回 JobManager） + # 注意：JobManager 启动时使用 0.0.0.0 监听，但 worker 需要实际可访问的主机名 + # env.jobmanager_host = "sage-node-1" + step_duration = time.time() - step_start + print(f" ✅ 环境创建完成 (耗时: {step_duration:.3f}秒)") + print(" 📋 调度策略: SPREAD (分散放置到多个节点)\n") + + # 步骤2: 构建数据流 - 增加并行度以利用多节点 + print("🔧 [2/5] 构建数据流 pipeline...") + step_start = time.time() + ( + env.from_source(SimpleSource) + .map(SimpleProcessor, parallelism=8) # 增加并行度，充分利用集群 + .sink(ConsoleSink) + ) + step_duration = time.time() - step_start + print(f" ✅ Pipeline 构建完成 (耗时: {step_duration:.3f}秒)") + print(" 📋 SimpleProcessor 并行度: 8 (将分布到多个节点)\n") + + # 步骤3: 连接JobManager + print("🔌 [3/5] 连接到 JobManager...") + step_start = time.time() + try: + # 这里会触发与JobManager的连接 + _ = env.client # 访问client property确保已创建 + step_duration = time.time() - step_start + print(f" ✅ JobManager 连接成功 (耗时: {step_duration:.3f}秒)\n") + except Exception as e: + step_duration = time.time() - step_start + print(f" ❌ 连接失败 (耗时: {step_duration:.3f}秒)") + print(f" 错误: {e}\n") + return + + # 步骤4: 提交任务 + print("🚀 [4/5] 提交任务到 JobManager...") + step_start = time.time() + try: + env.submit(autostop=True) # 不自动停止,手动控制 + step_duration = time.time() - step_start + print(f" ✅ 任务提交成功 (耗时: {step_duration:.3f}秒)\n") + except Exception as e: + step_duration = time.time() - step_start + print(f" ❌ 任务提交失败 (耗时: {step_duration:.3f}秒)") + print(f" 错误: {e}\n") + return + + # 步骤5: 等待执行完成 + print("⏳ [5/5] 等待任务执行...") + step_start = time.time() + try: + # 等待任务执行完成 + env._wait_for_completion() + step_duration = time.time() - step_start + print(f" ✅ 任务执行完成 (耗时: {step_duration:.3f}秒)\n") + except Exception as e: + step_duration = time.time() - step_start + print(f" ⚠️ 任务执行异常 (耗时: {step_duration:.3f}秒)") + print(f" 错误: {e}\n") + + # 查看调度器指标 + print("📊 获取调度器指标...") + try: + metrics = env.get_scheduler_metrics() + print(f" 调度器指标: {metrics}") + + # 如果使用 LoadAwareScheduler，显示节点使用情况 + if hasattr(scheduler, "node_selector"): + # 使用 node_task_count 获取节点任务统计 + node_task_count = scheduler.node_selector.node_task_count + if node_task_count: + print("\n 📍 节点放置统计:") + for node_id, count in node_task_count.items(): + node_info = scheduler.node_selector.get_node(node_id) + if node_info: + print(f" {node_info.hostname}: {count} 任务") + else: + print(f" {node_id[:12]}...: {count} 任务") + except Exception as e: + print(f" ⚠️ 无法获取指标: {e}") + print() + + # 步骤6: 清理资源（关键步骤） + print("🧹 [6/6] 清理资源...") + step_start = time.time() + try: + env.close() + step_duration = time.time() - step_start + print(f" ✅ 资源清理完成 (耗时: {step_duration:.3f}秒)\n") + except Exception as e: + step_duration = time.time() - step_start + print(f" ⚠️ 资源清理异常 (耗时: {step_duration:.3f}秒)") + print(f" 错误: {e}\n") + + # 总体统计 + total_duration = time.time() - total_start + print("=" * 60) + print(f"🎉 总耗时: {total_duration:.3f}秒") + print("=" * 60) + + +def main(): + """运行所有示例""" + print( + """ +╔══════════════════════════════════════════════════════════════╗ +║ RemoteEnvironment 分布式调度示例 ║ +║ ║ +║ 演示如何使用 LoadAwareScheduler + SPREAD 策略 ║ +║ 将任务分发到集群中的多个节点执行 ║ +╚══════════════════════════════════════════════════════════════╝ + """ + ) + + print( + """ +⚠️ 注意事项： + 1. 运行前需要启动 JobManager daemon: sage jobmanager start + 2. 确保 Ray 集群已启动: sage cluster start + 3. 如果连接失败，请检查 daemon 和集群状态 + +📋 分布式调度配置： + - 调度器: LoadAwareScheduler (负载感知) + - 策略: SPREAD (分散放置) + - 并行度: 8 (SimpleProcessor) + - 数据量: 500 条 + """ + ) + + try: + # 运行示例 + example_default_scheduler() + + print("\n" + "=" * 60) + print("✅ 所有示例运行完成！") + print("=" * 60) + + except Exception as e: + print(f"\n❌ 错误: {e}") + import traceback + + traceback.print_exc() + print("\n提示: 请确保 JobManager daemon 正在运行") + print("启动命令: sage jobmanager start") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/README.md b/tutorials/L3-kernel/README.md new file mode 100644 index 0000000..2afab3d --- /dev/null +++ b/tutorials/L3-kernel/README.md @@ -0,0 +1,85 @@ +# L3: Kernel - 核心引擎层示例 + +> 对应 SAGE 包：`sage-kernel` + +## 📖 层级说明 + +**Kernel** 层是 SAGE 的核心引擎，提供： + +- 流处理 API (DataStream) +- 批处理 API (Batch) +- 操作符系统 (Operators) +- 函数系统 (Functions) +- 运行时环境 (Runtime) + +## 📚 目录结构 + +``` +L3-kernel/ +├── batch/ # 批处理示例 +├── stream/ # 流处理示例 +├── operators/ # 操作符示例 +├── functions/ # 函数示例 +└── advanced/ # 高级特性 + └── fault_tolerance/ # 容错机制 +``` + +## 🎯 学习路径 + +### 1️⃣ 批处理基础 (`batch/`) + +从批处理开始，理解数据处理的基本概念： + +- `hello_local_batch.py` - 本地批处理 +- `hello_remote_batch.py` - 远程批处理 +- `hello_batch_operator_examples.py` - 批处理操作符 + +### 2️⃣ 流处理基础 (`stream/`) + +进入流处理的世界： + +- `hello_streaming_world.py` - 基础流处理 +- `hello_onebyone_world.py` - 单条数据流 +- `hello_connected_stream_example.py` - 连接流 + +### 3️⃣ 操作符系统 (`operators/`) + +掌握核心操作符： + +- `hello_comap_world.py` - CoMap 操作符 +- `hello_filter_world.py` - Filter 过滤 +- `hello_flatmap_world.py` - FlatMap 展开 +- `hello_join_world.py` - Join 连接 +- `hello_three_input_comap.py` - 多输入 CoMap + +### 4️⃣ 函数系统 (`functions/`) + +理解函数抽象： + +- `hello_comap_function_example.py` - 函数版 CoMap +- `hello_comap_lambda_example.py` - Lambda 版本 +- `hello_wordcount_*.py` - WordCount 系列示例 + +### 5️⃣ 高级特性 (`advanced/`) + +探索高级功能： + +- `hello_future_world.py` - Future 异步处理 +- `hello_realistic_service_example.py` - 实际服务示例 +- `fault_tolerance/` - 容错和检查点 + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. SAGE 的核心 API 和编程模型 +1. 批处理和流处理的差异 +1. 各种操作符的使用场景 +1. 如何构建数据处理管道 + +## ⏭️ 下一步 + +学完内核层后，继续学习： + +- **L3-libs/** - 算法库和工具（同层级） +- **L4-middleware/** - 中间件和领域算子 diff --git a/tutorials/L3-kernel/advanced/fault_tolerance/checkpoint_recovery_test.py b/tutorials/L3-kernel/advanced/fault_tolerance/checkpoint_recovery_test.py new file mode 100644 index 0000000..9d3045d --- /dev/null +++ b/tutorials/L3-kernel/advanced/fault_tolerance/checkpoint_recovery_test.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +""" +Checkpoint 容错机制测试 + +测试任务在失败后能否从 checkpoint 恢复 +@test:timeout=120 +""" + +import os +import time + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + + +class TestSource(SourceFunction): + """测试数据源 - 会在第 5 次迭代时模拟失败""" + + def __init__(self): + super().__init__() + self.counter = 0 + self.logger.info("TestSource initialized") + + def execute(self, data=None): + self.counter += 1 + self.logger.info(f"TestSource: generating data #{self.counter}") + + if self.counter > 10: + self.logger.info("TestSource: finished") + from sage.kernel.runtime.communication.packet import StopSignal + + return StopSignal("TestSource-completed") + + # 模拟第 5 个数据处理时失败（只失败一次） + if self.counter == 5 and not hasattr(self, "_failed_once"): + self._failed_once = True + self.logger.error("TestSource: simulating failure at counter=5") + raise RuntimeError("Simulated failure at counter=5") + + # 添加短延迟以便观察 checkpoint（测试模式下缩短） + delay = 0.1 if os.getenv("SAGE_TEST_MODE") == "true" else 0.5 + time.sleep(delay) + self.logger.debug(f"TestSource: emitting data #{self.counter}") + return {"id": self.counter, "value": f"data_{self.counter}"} + + def get_state(self): + """保存状态到 checkpoint""" + state = { + "counter": self.counter, + "_failed_once": getattr(self, "_failed_once", False), + } + self.logger.debug(f"TestSource: saving state: {state}") + return state + + def restore_state(self, state): + """从 checkpoint 恢复状态""" + self.counter = state.get("counter", 0) + if state.get("_failed_once"): + self._failed_once = True + self.logger.info(f"TestSource: restored state, counter={self.counter}") + + +class TestProcessor(MapFunction): + """测试处理器""" + + def execute(self, data): + if data is None: + return None + + self.logger.info(f"TestProcessor: processing {data}") + return {"id": data["id"], "processed": data["value"].upper()} + + +class TestSink(SinkFunction): + """测试输出""" + + def __init__(self): + super().__init__() + self.results = [] + + def execute(self, data): + if data is None: + return None + + self.logger.info(f"TestSink: received {data}") + self.results.append(data) + print(f"✅ Processed: ID={data['id']}, Value={data['processed']}") + return data + + +def test_checkpoint_recovery(): + """测试 checkpoint 容错恢复""" + print("\n" + "=" * 60) + print("Testing Checkpoint-Based Fault Tolerance") + print("=" * 60 + "\n") + + # 在测试模式下使用更短的 checkpoint 间隔 + is_test_mode = os.getenv("SAGE_TEST_MODE") == "true" or os.getenv("CI") == "true" + checkpoint_interval = 0.5 if is_test_mode else 2.0 + + env = LocalEnvironment( + "checkpoint_test", + config={ + "fault_tolerance": { + "strategy": "checkpoint", + "checkpoint_interval": checkpoint_interval, # 测试模式: 0.5s, 正常: 2.0s + "max_recovery_attempts": 3, + "checkpoint_dir": ".sage/test_checkpoints", + } + }, + ) + + # 启用详细日志 + env.set_console_log_level("INFO") + + print("📝 Configuration:") + print(" - Strategy: checkpoint") + print(f" - Checkpoint Interval: {checkpoint_interval}s") + print(" - Max Recovery Attempts: 3") + print(" - Checkpoint Directory: .sage/test_checkpoints") + print() + + # 构建管道 + print("🔨 Building pipeline...") + env.from_source(TestSource).map(TestProcessor).sink(TestSink) + print("✅ Pipeline built\n") + + # 提交执行 + print("🚀 Submitting pipeline...") + try: + env.submit(autostop=True) + print("\n✅ Pipeline completed successfully") + except Exception as e: + print(f"\n❌ Pipeline failed: {e}") + import traceback + + traceback.print_exc() + + print("\n" + "=" * 60) + print("Test Completed") + print("=" * 60) + + +def test_restart_recovery(): + """测试重启容错恢复""" + print("\n" + "=" * 60) + print("Testing Restart-Based Fault Tolerance") + print("=" * 60 + "\n") + + env = LocalEnvironment( + "restart_test", + config={ + "fault_tolerance": { + "strategy": "restart", + "restart_strategy": "exponential", + "initial_delay": 1.0, + "max_delay": 5.0, + "multiplier": 2.0, + "max_attempts": 3, + } + }, + ) + + env.set_console_log_level("INFO") + + print("📝 Configuration:") + print(" - Strategy: restart") + print(" - Restart Strategy: exponential backoff") + print(" - Initial Delay: 1.0s") + print(" - Max Attempts: 3") + print() + + # 构建管道 + print("🔨 Building pipeline...") + env.from_source(TestSource).map(TestProcessor).sink(TestSink) + print("✅ Pipeline built\n") + + # 提交执行 + print("🚀 Submitting pipeline...") + try: + env.submit(autostop=True) + print("\n✅ Pipeline completed successfully") + except Exception as e: + print(f"\n❌ Pipeline failed: {e}") + import traceback + + traceback.print_exc() + + print("\n" + "=" * 60) + print("Test Completed") + print("=" * 60) + + +if __name__ == "__main__": + test_checkpoint_recovery() + + print("\n✨ All tests completed!\n") diff --git a/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance.py b/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance.py new file mode 100644 index 0000000..eec313f --- /dev/null +++ b/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance.py @@ -0,0 +1,227 @@ +# """ +# Fault Tolerance Demo + +# 展示如何在 SAGE 应用中启用容错功能。 +# 这个示例演示了用户如何通过简单的配置来启用容错，无需编写任何容错相关代码。 +# """ + +# import os +# import sys + +# from sage.kernel.api.local_environment import LocalEnvironment +# from sage.libs.foundation.io.sink import TerminalSink +# from sage.libs.foundation.io.source import FileSource + + +# def demo_checkpoint_fault_tolerance(): +# """ +# 演示 Checkpoint 容错策略 + +# Checkpoint 策略会定期保存任务状态，失败时从最近的检查点恢复。 +# 适用于长时间运行的有状态任务。 +# """ +# print("\n" + "=" * 70) +# print("Demo 1: Checkpoint-based Fault Tolerance") +# print("=" * 70) + +# # 创建环境，配置 Checkpoint 容错策略 +# env = LocalEnvironment( +# "checkpoint_demo", +# config={ +# # 容错配置 - 用户只需声明，系统自动处理 +# "fault_tolerance": { +# "strategy": "checkpoint", # 使用 checkpoint 策略 +# "checkpoint_interval": 30.0, # 每30秒保存一次 +# "max_recovery_attempts": 3, # 最多尝试恢复3次 +# "checkpoint_dir": ".demo_checkpoints", # checkpoint存储目录 +# }, +# # 数据源配置 +# "source": {"file_path": "data/sample.txt"}, +# # 输出配置 +# "sink": {}, +# }, +# ) + +# # 正常定义 DAG - 用户完全不需要关心容错 +# pipeline = ( +# env.from_source(FileSource, env.config["source"]) +# .map(lambda x: x.strip().upper()) # 转换为大写 +# .sink(TerminalSink, env.config["sink"]) +# ) + +# # 提交作业 - 容错由系统自动处理 +# # 如果任务失败，系统会自动从最近的 checkpoint 恢复 +# env.submit() + +# print("\n✅ Pipeline with checkpoint fault tolerance submitted!") +# print(" If tasks fail, they will automatically recover from checkpoints.") + + +# def demo_restart_fault_tolerance(): +# """ +# 演示 Restart 容错策略（指数退避） + +# Restart 策略在任务失败时直接重启，使用指数退避算法逐渐增加重试延迟。 +# 适用于无状态或短时间运行的任务。 +# """ +# print("\n" + "=" * 70) +# print("Demo 2: Restart-based Fault Tolerance (Exponential Backoff)") +# print("=" * 70) + +# # 创建环境，配置 Restart 容错策略 +# env = LocalEnvironment( +# "restart_demo", +# config={ +# # 容错配置 - 使用指数退避重启策略 +# "fault_tolerance": { +# "strategy": "restart", # 使用 restart 策略 +# "restart_strategy": "exponential", # 指数退避 +# "initial_delay": 1.0, # 首次重启等待1秒 +# "max_delay": 60.0, # 最多等待60秒 +# "multiplier": 2.0, # 每次延迟翻倍 +# "max_attempts": 5, # 最多重启5次 +# }, +# "source": {"file_path": "data/sample.txt"}, +# "sink": {}, +# }, +# ) + +# # 定义 DAG - 无需容错代码 +# pipeline = ( +# env.from_source(FileSource, env.config["source"]) +# .map(lambda x: x.strip().lower()) # 转换为小写 +# .sink(TerminalSink, env.config["sink"]) +# ) + +# # 提交作业 - 失败时自动重启 +# env.submit() + +# print("\n✅ Pipeline with restart fault tolerance submitted!") +# print(" If tasks fail, they will automatically restart with exponential backoff.") +# print(" Retry delays: 1s, 2s, 4s, 8s, 16s...") + + +# def demo_fixed_delay_restart(): +# """ +# 演示 Restart 容错策略（固定延迟） + +# 使用固定延迟的重启策略，每次重启等待相同的时间。 +# """ +# print("\n" + "=" * 70) +# print("Demo 3: Restart-based Fault Tolerance (Fixed Delay)") +# print("=" * 70) + +# env = LocalEnvironment( +# "fixed_restart_demo", +# config={ +# # 容错配置 - 固定延迟重启 +# "fault_tolerance": { +# "strategy": "restart", +# "restart_strategy": "fixed", # 固定延迟 +# "delay": 5.0, # 每次等待5秒 +# "max_attempts": 3, # 最多重启3次 +# }, +# "source": {"file_path": "data/sample.txt"}, +# "sink": {}, +# }, +# ) + +# pipeline = ( +# env.from_source(FileSource, env.config["source"]) +# .map(lambda x: x.strip()) +# .sink(TerminalSink, env.config["sink"]) +# ) + +# env.submit() + +# print("\n✅ Pipeline with fixed delay restart submitted!") +# print(" If tasks fail, they will restart after 5 seconds each time.") + + +# def demo_no_fault_tolerance(): +# """ +# 演示不配置容错（默认行为） + +# 如果不配置 fault_tolerance，系统使用默认的简单重启策略。 +# """ +# print("\n" + "=" * 70) +# print("Demo 4: No Explicit Fault Tolerance Configuration") +# print("=" * 70) + +# # 不配置 fault_tolerance +# env = LocalEnvironment( +# "no_ft_demo", config={"source": {"file_path": "data/sample.txt"}, "sink": {}} +# ) + +# pipeline = ( +# env.from_source(FileSource, env.config["source"]) +# .map(lambda x: x.strip()) +# .sink(TerminalSink, env.config["sink"]) +# ) + +# env.submit() + +# print("\n✅ Pipeline submitted with default fault tolerance.") + + +# def main(): +# """主函数 - 运行所有演示""" + +# # 检查是否在测试模式 +# if ( +# os.getenv("SAGE_EXAMPLES_MODE") == "test" +# or os.getenv("SAGE_TEST_MODE") == "true" +# ): +# print("🧪 Test mode detected - fault_tolerance_demo") +# print("✅ Test passed: Fault tolerance demo structure validated") +# return + +# print("\n") +# print("╔" + "=" * 68 + "╗") +# print("║" + " " * 18 + "SAGE FAULT TOLERANCE DEMO" + " " * 25 + "║") +# print("╚" + "=" * 68 + "╝") + +# print("\n📖 This demo shows how to enable fault tolerance in SAGE applications.") +# print(" Users only need to declare the strategy in config - no code changes!") + +# # 创建示例数据文件（如果不存在） +# os.makedirs("data", exist_ok=True) +# if not os.path.exists("data/sample.txt"): +# with open("data/sample.txt", "w") as f: +# f.write("Hello World\n") +# f.write("Fault Tolerance Demo\n") +# f.write("SAGE Framework\n") +# print("\n📄 Created sample data file: data/sample.txt") + +# # 运行各种容错策略演示 +# try: +# demo_checkpoint_fault_tolerance() +# demo_restart_fault_tolerance() +# demo_fixed_delay_restart() +# demo_no_fault_tolerance() + +# print("\n" + "=" * 70) +# print("✨ All demos completed successfully!") +# print("=" * 70) + +# print("\n📚 Learn more:") +# print( +# " - Full documentation: packages/sage-kernel/src/sage/kernel/fault_tolerance/README.md" +# ) +# print( +# " - Quick reference: packages/sage-kernel/src/sage/kernel/fault_tolerance/QUICK_REFERENCE.md" +# ) +# print( +# " - More examples: examples/kernel/fault_tolerance_examples.py" +# ) + +# except Exception as e: +# print(f"\n❌ Error running demos: {e}") +# import traceback + +# traceback.print_exc() +# sys.exit(1) + + +# if __name__ == "__main__": +# main() diff --git a/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance_examples.py b/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance_examples.py new file mode 100644 index 0000000..9e42627 --- /dev/null +++ b/tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance_examples.py @@ -0,0 +1,412 @@ +""" +Fault Tolerance Usage Examples + +Layer: L3 (Kernel - Examples) +Dependencies: sage.libs (L3 - optional examples only) + +展示应用用户如何使用容错（无感知）以及开发者如何扩展容错策略。 + +⚠️ IMPORTANT - 架构说明: + 本文件是**可选示例代码**，不是 kernel 的核心功能。 + 示例中使用 sage.libs 的 Source/Sink 是为了演示完整的容错流程。 + + 运行这些示例需要安装 sage.libs: + pip install sage-libs + + 用户可以使用自己的 Source/Sink 实现，无需依赖 sage.libs。 + +Architecture Note: + 这些示例演示完整的使用场景，需要 sage.libs 提供 Source/Sink。 + 如果 sage.libs 不可用，示例将无法运行，但不影响 kernel 的核心功能。 +""" + +# ============================================================================ +# 应用用户示例 - 容错配置对用户是透明的 +# ============================================================================ + + +def example_1_user_checkpoint_strategy(): + """ + 示例 1: 应用用户使用 Checkpoint 策略 + + 用户只需在 Environment 配置中声明，无需编写任何容错代码。 + + Requirements: + pip install sage-libs + """ + from sage.kernel.api.local_environment import LocalEnvironment + + # 此示例需要 sage.libs + try: + from sage.libs.foundation.io.sink import TerminalSink + from sage.libs.foundation.io.source import FileSource + except ImportError as e: + raise ImportError( + "This example requires sage.libs. " + "Install it with: pip install sage-libs\n" + "Or use your own Source/Sink implementations." + ) from e + + # 创建环境时声明使用 Checkpoint 容错策略 + env = LocalEnvironment( + "qa_pipeline_with_checkpoint", + config={ + "fault_tolerance": { + "strategy": "checkpoint", + "checkpoint_interval": 60.0, # 每60秒保存一次 + "max_recovery_attempts": 3, # 最多恢复3次 + "checkpoint_dir": ".sage/checkpoints", + }, + "source": {"data_path": "data/questions.txt"}, + "sink": {}, + }, + ) + + # 正常定义 DAG - 完全不需要关心容错 + ( + env.from_source(FileSource, env.config["source"]) + .map(lambda x: x.upper()) # 一些处理 + .sink(TerminalSink, env.config["sink"]) + ) + + # 提交作业 - 容错由系统自动处理 + # 如果任务失败，系统会自动从最近的 checkpoint 恢复 + env.submit() + + print("✅ Pipeline submitted with checkpoint fault tolerance") + + +def example_2_user_restart_strategy(): + """ + 示例 2: 应用用户使用 Restart 策略 + + 使用指数退避重启策略，用户同样无需编写容错代码。 + + Requirements: + pip install sage-libs + """ + from sage.kernel.api.local_environment import LocalEnvironment + + try: + from sage.libs.foundation.io.sink import TerminalSink + from sage.libs.foundation.io.source import FileSource + except ImportError as e: + raise ImportError( + "This example requires sage.libs. Install it with: pip install sage-libs" + ) from e + + # 使用指数退避重启策略 + env = LocalEnvironment( + "data_pipeline_with_restart", + config={ + "fault_tolerance": { + "strategy": "restart", + "restart_strategy": "exponential", + "initial_delay": 1.0, # 首次重启等待1秒 + "max_delay": 60.0, # 最多等待60秒 + "multiplier": 2.0, # 每次延迟翻倍 + "max_attempts": 5, # 最多重启5次 + }, + "source": {"data_path": "data/input.txt"}, + "sink": {}, + }, + ) + + # 定义 DAG - 用户不关心容错 + ( + env.from_source(FileSource, env.config["source"]) + .map(lambda x: x.strip()) + .sink(TerminalSink, env.config["sink"]) + ) + + # 提交 - 失败时自动重启 + env.submit() + + print("✅ Pipeline submitted with exponential backoff restart") + + +def example_3_user_no_fault_tolerance(): + """ + 示例 3: 用户不配置容错（使用默认行为） + + Requirements: + pip install sage-libs + """ + from sage.kernel.api.local_environment import LocalEnvironment + + try: + from sage.libs.foundation.io.sink import TerminalSink + from sage.libs.foundation.io.source import FileSource + except ImportError as e: + raise ImportError( + "This example requires sage.libs. Install it with: pip install sage-libs" + ) from e + + # 不配置 fault_tolerance，使用默认行为 + env = LocalEnvironment("simple_pipeline") + + # 正常定义和提交 + ( + env.from_source(FileSource, {"data_path": "data.txt"}) + .map(lambda x: x.upper()) + .sink(TerminalSink, {}) + ) + + env.submit() + + print("✅ Pipeline submitted with default fault tolerance") + + +def example_4_user_yaml_config(): + """ + 示例 4: 从 YAML 配置文件读取容错配置 + + 这是最常见的用法 - 配置在外部文件中管理。 + + Requirements: + pip install sage-libs + """ + from sage.common.utils.config.loader import load_config + from sage.kernel.api.local_environment import LocalEnvironment + + try: + from sage.libs.foundation.io.sink import TerminalSink + from sage.libs.foundation.io.source import FileSource + except ImportError as e: + raise ImportError( + "This example requires sage.libs. Install it with: pip install sage-libs" + ) from e + + # config.yaml 内容示例： + # fault_tolerance: + # strategy: checkpoint + # checkpoint_interval: 30.0 + # max_recovery_attempts: 5 + # source: + # file_path: data/input.txt + # sink: {} + + config = load_config("config/my_pipeline.yaml") + + # 容错配置从 YAML 文件读取 + env = LocalEnvironment("yaml_configured_pipeline", config=config) + + ( + env.from_source(FileSource, config["source"]) + .map(lambda x: x.strip()) + .sink(TerminalSink, config["sink"]) + ) + + env.submit() + + print("✅ Pipeline submitted with YAML-configured fault tolerance") + + +# ============================================================================ +# 开发者示例 - 扩展自定义容错策略 +# ============================================================================ + + +def example_5_developer_custom_strategy(): + """ + 示例 5: 开发者实现自定义容错策略 + + 开发者可以继承 BaseFaultHandler 实现自己的容错逻辑。 + """ + from sage.kernel.fault_tolerance.base import BaseFaultHandler + + class CircuitBreakerFaultHandler(BaseFaultHandler): + """ + 断路器容错策略 + + 当失败次数超过阈值时，打开断路器，停止重试一段时间。 + """ + + def __init__(self, failure_threshold=5, timeout=60.0): + self.failure_threshold = failure_threshold + self.timeout = timeout + self.failure_counts = {} + self.circuit_open = {} + self.open_time = {} + self.logger = None + + def handle_failure(self, task_id: str, error: Exception) -> bool: + import time + + # 更新失败计数 + if task_id not in self.failure_counts: + self.failure_counts[task_id] = 0 + self.failure_counts[task_id] += 1 + + if self.logger: + self.logger.warning( + f"Task {task_id} failed (count: {self.failure_counts[task_id]}): {error}" + ) + + # 检查是否应该打开断路器 + if self.failure_counts[task_id] >= self.failure_threshold: + self.circuit_open[task_id] = True + self.open_time[task_id] = time.time() + + if self.logger: + self.logger.error( + f"Circuit breaker opened for {task_id} " + f"(failures: {self.failure_counts[task_id]})" + ) + return False + + return self.recover(task_id) + + def can_recover(self, task_id: str) -> bool: + import time + + # 如果断路器是打开的 + if self.circuit_open.get(task_id, False): + # 检查是否已经超过超时时间 + if time.time() - self.open_time.get(task_id, 0) > self.timeout: + # 关闭断路器，重置计数 + self.circuit_open[task_id] = False + self.failure_counts[task_id] = 0 + + if self.logger: + self.logger.info(f"Circuit breaker closed for {task_id}") + + return True + return False + + return self.failure_counts.get(task_id, 0) < self.failure_threshold + + def recover(self, task_id: str) -> bool: + if self.logger: + self.logger.info(f"Attempting to recover task {task_id}") + + # 实际的恢复逻辑 + # TODO: 实现具体的恢复策略 + # Issue URL: https://github.com/intellistream/SAGE/issues/933 + + return True + + print("✅ Custom CircuitBreakerFaultHandler defined") + print(" Developers can extend BaseFaultHandler to create custom strategies") + + +def example_6_developer_register_strategy(): + """ + 示例 6: 开发者将自定义策略集成到系统 + + 步骤： + 1. 将自定义策略类放到 impl/ 目录 + 2. 在 impl/__init__.py 中导出 + 3. 在 factory.py 中添加创建逻辑 + 4. 用户就可以通过配置使用了 + """ + + # 步骤 1: 创建自定义策略文件 + # packages/sage-kernel/src/sage/kernel/fault_tolerance/impl/circuit_breaker.py + + # 步骤 2: 在 impl/__init__.py 添加导出 + # from sage.kernel.fault_tolerance.impl.circuit_breaker import CircuitBreakerFaultHandler + # __all__ = [..., "CircuitBreakerFaultHandler"] + + # 步骤 3: 在 factory.py 添加创建逻辑 + # def create_fault_handler_from_config(config): + # strategy = config.get("strategy") + # if strategy == "circuit_breaker": + # return CircuitBreakerFaultHandler( + # failure_threshold=config.get("failure_threshold", 5), + # timeout=config.get("timeout", 60.0) + # ) + # ... + + # 步骤 4: 用户现在可以通过配置使用 + # env = LocalEnvironment( + # "my_app", + # config={ + # "fault_tolerance": { + # "strategy": "circuit_breaker", + # "failure_threshold": 3, + # "timeout": 30.0 + # } + # } + # ) + + print("✅ Custom strategy integration steps:") + print(" 1. Create strategy class in impl/") + print(" 2. Export in impl/__init__.py") + print(" 3. Add creation logic in factory.py") + print(" 4. Users can now use it via config") + + +def example_7_developer_reference_implementations(): + """ + 示例 7: 开发者参考现有实现 + + 查看 impl/ 目录下的实现来学习如何编写容错策略。 + """ + print("✅ Reference implementations in impl/:") + print(" - checkpoint_recovery.py: Checkpoint-based fault tolerance") + print(" - restart_recovery.py: Restart-based fault tolerance") + print(" - restart_strategy.py: Various restart strategies") + print(" - lifecycle_impl.py: Lifecycle management") + print(" - checkpoint_impl.py: Checkpoint management") + + +# ============================================================================ +# 运行所有示例 +# ============================================================================ + + +def run_user_examples(): + """运行应用用户示例""" + print("\n" + "=" * 70) + print("APPLICATION USER EXAMPLES - Fault Tolerance is Transparent") + print("=" * 70 + "\n") + + print("Example 1: Checkpoint Strategy") + print("-" * 70) + example_1_user_checkpoint_strategy() + + print("\nExample 2: Restart Strategy with Exponential Backoff") + print("-" * 70) + example_2_user_restart_strategy() + + print("\nExample 3: No Explicit Fault Tolerance Configuration") + print("-" * 70) + example_3_user_no_fault_tolerance() + + print("\nExample 4: YAML Configuration") + print("-" * 70) + example_4_user_yaml_config() + + +def run_developer_examples(): + """运行开发者扩展示例""" + print("\n" + "=" * 70) + print("DEVELOPER EXAMPLES - Extending Fault Tolerance Strategies") + print("=" * 70 + "\n") + + print("Example 5: Custom Circuit Breaker Strategy") + print("-" * 70) + example_5_developer_custom_strategy() + + print("\nExample 6: Integrating Custom Strategy into System") + print("-" * 70) + example_6_developer_register_strategy() + + print("\nExample 7: Reference Implementations") + print("-" * 70) + example_7_developer_reference_implementations() + + +if __name__ == "__main__": + print("\n") + print("╔" + "=" * 68 + "╗") + print("║" + " " * 20 + "FAULT TOLERANCE EXAMPLES" + " " * 24 + "║") + print("╚" + "=" * 68 + "╝") + + run_user_examples() + run_developer_examples() + + print("\n" + "=" * 70) + print("All examples completed successfully!") + print("=" * 70 + "\n") diff --git a/tutorials/L3-kernel/advanced/hello_future_stream_example.py b/tutorials/L3-kernel/advanced/hello_future_stream_example.py new file mode 100644 index 0000000..dc88c20 --- /dev/null +++ b/tutorials/L3-kernel/advanced/hello_future_stream_example.py @@ -0,0 +1,193 @@ +import time + +from sage.common.core.functions.base_function import BaseFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 初始数据源：启动计数器 +class CounterStartSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.started = False + + def execute(self, data=None): + if not self.started: + self.started = True + print("🚀 Starting counter...") + # 只发送一次初始值，随后就返回None + return {"count": 0, "message": "Counter initialized"} + return None + + +# 反馈处理器：接收计数器值和反馈值 +class CounterProcessor(BaseCoMapFunction): + def map0(self, data): + """处理初始计数器数据（来自输入流0）""" + print(f"📥 Initial data: {data}") + return data + + def map1(self, data): + """处理反馈数据（来自输入流1 - future stream）""" + print(f"🔄 Feedback data: {data}") + return data + + +# 计数增加器 +class CounterIncrementer(BaseFunction): + def execute(self, data): + if data is None: + return None + + current_count = data.get("count", 0) + new_count = current_count + 1 + + result = { + "count": new_count, + "message": f"Counter value: {new_count}", + "should_continue": new_count < 10, + } + + print(f"🔢 Counter incremented: {current_count} → {new_count}") + return result + + +# 退出条件检查器 +class ExitChecker(BaseFunction): + def execute(self, data): + if data is None: + return None + + count = data.get("count", 0) + should_continue = data.get("should_continue", True) + + if not should_continue: + print(f"🏁 Counter reached target value: {count}. Stopping...") + return None # 停止数据流 + + print(f"✅ Counter check passed: {count} < 10, continuing...") + return data + + +# 反馈延迟器：添加延迟以便观察反馈循环 +class FeedbackDelayer(BaseFunction): + def execute(self, data): + if data is None: + return None + + print("⏱️ Adding delay before feedback...") + time.sleep(1) # 1秒延迟，便于观察 + print(f"🔙 Sending feedback: {data}") + return data + + +# 最终输出 +class CounterSink(SinkFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.custom_name = kwargs.get("name", "CounterSink") + + def execute(self, data): + if data is not None: + count = data.get("count", 0) + message = data.get("message", "No message") + print(f"[{self.custom_name}] 📊 {message}") + + if count >= 10: + print(f"[{self.custom_name}] 🎉 Counter completed! Final value: {count}") + return data + + +def main(): + # 创建环境 + env = LocalEnvironment("future_stream_example") + + print("🚀 Starting Future Stream Example") + print("🔄 Demonstrating feedback edges with a counting loop") + print("📊 Counter will increment from 0 to 10 using feedback") + print("⏹️ Press Ctrl+C to stop\n") + + print("🔗 Creating feedback loop with future stream...") + + # 1. 声明future stream（反馈边） + print("📋 Step 1: Declaring future stream for feedback...") + feedback_stream = env.from_future("counter_feedback") + + # 2. 创建初始数据源 + print("📋 Step 2: Creating initial counter source...") + counter_source = env.from_source(CounterStartSource, delay=0.5) + + # 3. 连接初始流和反馈流 + print("📋 Step 3: Connecting initial stream with feedback stream...") + connected_streams = counter_source.connect(feedback_stream) + + # 4. 处理连接的流（初始值 + 反馈值） + print("📋 Step 4: Processing connected streams...") + processed_data = connected_streams.comap(CounterProcessor) + + # 5. 增加计数器 + print("📋 Step 5: Setting up counter incrementer...") + incremented_data = processed_data.map(CounterIncrementer) + + # 6. 检查退出条件 + print("📋 Step 6: Setting up exit condition checker...") + checked_data = incremented_data.map(ExitChecker) + + # 7. 输出到终端 + print("📋 Step 7: Setting up output sink...") + checked_data.sink(CounterSink, name="CounterOutput") + + # 8. 创建反馈分支（添加延迟后反馈） + print("📋 Step 8: Creating feedback branch...") + feedback_data = checked_data.map(FeedbackDelayer) + + # 9. 填充future stream，建立反馈边 + print("📋 Step 9: Filling future stream to create feedback edge...") + feedback_data.fill_future(feedback_stream) + + print("\n🔄 Feedback loop structure:") + print( + " CounterSource → [Connected with Future] → CounterProcessor → Incrementer → ExitChecker → CounterSink" + ) + print(" ↑ ↓") + print(" └────────────────── FeedbackDelayer ←────────────────────┘") + print() + + print("✅ Pipeline validation:") + print(f" - Pipeline transformations: {len(env.pipeline)}") + + try: + print("🎬 Starting feedback loop execution...") + print("📈 Watch the counter increment in a feedback loop:\n") + + # 运行流处理 + env.submit() + + time.sleep(10) # 运行15秒，足够计数到10 + + except KeyboardInterrupt: + print("\n\n🛑 Stopping Future Stream Example...") + + finally: + print("\n📋 Example completed!") + print("💡 This example demonstrated:") + print(" - Creating a future stream with env.from_future()") + print(" - Using future stream in connected streams") + print(" - Processing initial and feedback data with CoMap") + print(" - Incrementing counter in a feedback loop") + print(" - Conditional exit based on counter value") + print(" - Filling future stream to create feedback edge") + print("\n🔄 Feedback Loop Features:") + print(" - Initial value flows through the system") + print(" - Processed result feeds back to the beginning") + print(" - Loop continues until exit condition is met") + print(" - Clean termination when counter reaches 10") + env.close() + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/advanced/hello_future_world.py b/tutorials/L3-kernel/advanced/hello_future_world.py new file mode 100644 index 0000000..fa1f674 --- /dev/null +++ b/tutorials/L3-kernel/advanced/hello_future_world.py @@ -0,0 +1,118 @@ +from time import sleep + +from sage.common.core.functions.base_function import BaseFunction +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 启动信号源（只发一次启动信号） +class StartSource(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.started = False + + def execute(self): + if not self.started: + self.started = True + print("已发送启动信号") + return {"signal": "start"} + else: + return None + + +# 合流处理器（map0 = 启动，map1 = 反馈） +class SignalMerger(BaseCoMapFunction): + def map0(self, data): + print(f">>> StartSource：收到启动数据: {data}") + return data + + def map1(self, data): + print(f">>> PipelineSource： 收到反馈数据: {data}") + return data + + +# 从语句列表中按顺序取语句（只在收到反馈时推进） +class SentenceProvider(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sentences = [ + "这是第一句。", + "这是第二句。", + "这是第三句。", + "所有语句已完成！", + ] + self.index = 0 + + def execute(self, data): + if data is None: + return None + + if self.index >= len(self.sentences): + print("全部语句已输出完毕，结束数据流。") + return None + + sentence = self.sentences[self.index] + self.index += 1 + new_data = {"句子": sentence} + print(f">>> SentenceProvider 提供句子: {new_data}") + return new_data + + +# Sink：打印句子（立即打印，不加 sleep） +class FeedbackSink(SinkFunction): + def execute(self, data): + if data: + print(f">>> Sink 打印: {data}") + return data + + +# 延迟反馈算子：控制节奏 +class FeedbackDelayer(BaseFunction): + def execute(self, data): + if data is None: + return None + sleep(1) # 控制间隔 + print(">>> FeedbackDelayer等待 2 秒后反馈...") + return data + + +def main(): + env = LocalEnvironment("句子顺序输出") + + # 1. 启动源 + start_stream = env.from_source(StartSource) + + # 2. future stream 用于反馈 + feedback_stream = env.from_future("feedback") + + # 3. 合流 + merged = start_stream.connect(feedback_stream).comap(SignalMerger) + + # 4. 语句提供器 + provided = merged.map(SentenceProvider) + + # 5. Sink，打印结果 + sinked = provided.sink(FeedbackSink) + + # 6. 在反馈前加延迟 + delayed = sinked.map(FeedbackDelayer) + + # 7. 把延迟后的结果反馈到 future stream + delayed.fill_future(feedback_stream) + + # 对于循环流，仍然需要手动控制，因为autostop无法处理循环 + env.submit() + + from time import sleep + + sleep(6) # 给足够时间让所有数据处理完成 + + print("Hello Future World 示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/advanced/hello_realistic_service_example.py b/tutorials/L3-kernel/advanced/hello_realistic_service_example.py new file mode 100644 index 0000000..04cfd17 --- /dev/null +++ b/tutorials/L3-kernel/advanced/hello_realistic_service_example.py @@ -0,0 +1,506 @@ +""" +真实SAGE工作流测试 + +使用SAGE的完整流水线机制，展示服务在真实算子中的使用 +""" + +import time + +from sage.common.core.functions.base_function import BaseFunction +from sage.kernel.api.local_environment import LocalEnvironment + + +# 服务定义（重用之前的服务） +class FeatureStoreService: + """特征存储服务""" + + def __init__(self): + self.features = { + "user_features": { + "user_001": {"age": 25, "city": "Beijing", "vip_level": 2}, + "user_002": {"age": 30, "city": "Shanghai", "vip_level": 3}, + "user_003": {"age": 28, "city": "Guangzhou", "vip_level": 1}, + }, + "item_features": { + "item_101": {"category": "electronics", "price": 1000, "rating": 4.5}, + "item_102": {"category": "books", "price": 50, "rating": 4.8}, + "item_103": {"category": "clothing", "price": 200, "rating": 4.2}, + }, + } + self.is_running = False + self.ctx = None + + def start_running(self): + self.is_running = True + print("Feature store service started") + + def terminate(self): + self.is_running = False + print("Feature store service terminated") + + def get_user_features(self, user_id: str): + features = self.features["user_features"].get(user_id, {}) + print(f"Retrieved user features for {user_id}: {features}") + return features + + def get_item_features(self, item_id: str): + features = self.features["item_features"].get(item_id, {}) + print(f"Retrieved item features for {item_id}: {features}") + return features + + def batch_get_features(self, entity_type: str, entity_ids: list): + feature_table = self.features.get(f"{entity_type}_features", {}) + results = {} + for entity_id in entity_ids: + results[entity_id] = feature_table.get(entity_id, {}) + print(f"Batch retrieved {len(results)} {entity_type} features") + return results + + +class ModelService: + """模型服务""" + + def __init__(self, model_name: str = "recommendation_model_v1"): + self.model_name = model_name + self.is_running = False + self.ctx = None + self.prediction_count = 0 + + def start_running(self): + self.is_running = True + print(f"Model service started: {self.model_name}") + + def terminate(self): + self.is_running = False + print(f"Model service terminated: {self.model_name}") + + def predict(self, features: dict): + if not self.is_running: + return {"error": "Model service not running"} + + self.prediction_count += 1 + score = 0.6 + result = { + "score": round(score, 3), + "prediction_id": f"pred_{self.prediction_count}", + "model": self.model_name, + "features_used": list(features.keys()), + } + + print(f"Model prediction {self.prediction_count}: score={result['score']}") + return result + + def batch_predict(self, features_list: list): + if not self.is_running: + return {"error": "Model service not running"} + + results = [] + for features in features_list: + prediction = self.predict(features) + results.append(prediction) + + print(f"Batch prediction completed: {len(results)} predictions") + return results + + +class CacheService: + def __init__(self, max_size: int = 1000): + self.max_size = max_size + self.cache = {} + self.is_running = False + self.ctx = None + + def start_running(self): + self.is_running = True + print(f"Cache service started with max_size={self.max_size}") + + def terminate(self): + self.is_running = False + print("Cache service terminated") + + def get(self, key: str): + result = self.cache.get(key, None) + return result + + def set(self, key: str, value): + if len(self.cache) >= self.max_size: + oldest_key = next(iter(self.cache)) + del self.cache[oldest_key] + self.cache[key] = value + return True + + def size(self): + return len(self.cache) + + +class LogService: + def __init__(self, log_level: str = "INFO"): + self.log_level = log_level + self.logs = [] + self.is_running = False + self.ctx = None + + def start_running(self): + self.is_running = True + print(f"Log service started with level {self.log_level}") + + def terminate(self): + self.is_running = False + print("Log service terminated") + + def log(self, level: str, message: str, context: dict | None = None): + if not self.is_running: + return False + + log_entry = { + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "level": level, + "message": message, + "context": context or {}, + } + self.logs.append(log_entry) + print(f"[{log_entry['timestamp']}] {level}: {message}") + return True + + def info(self, message: str, context: dict | None = None): + return self.log("INFO", message, context) + + def error(self, message: str, context: dict | None = None): + return self.log("ERROR", message, context) + + def get_logs(self): + return self.logs.copy() + + +# 算子函数定义 +class RequestSourceFunction(BaseFunction): + """请求源算子 - 生成推荐请求""" + + def __init__(self, request_count: int = 5): + super().__init__() + self.request_count = request_count + self.current_count = 0 + + def execute(self, data=None): + """生成推荐请求（源算子不需要data参数）""" + if self.current_count >= self.request_count: + return None # 停止生成 + + self.current_count += 1 + + # 生成模拟请求 + users = ["user_001", "user_002", "user_003"] + items = [ + ["item_101", "item_102", "item_103"], + ["item_101", "item_102"], + ["item_102", "item_103"], + ] + + request = { + "request_id": f"req_{self.current_count:03d}", + "user_id": users[1], + "candidate_items": items[1], + "timestamp": time.time(), + } + + self.logger.info( + f"Generated request {self.current_count}: {request['request_id']} for {request['user_id']}" + ) + return request + + +class FeatureEnrichmentFunction(BaseFunction): + """特征丰富算子 - 获取用户和物品特征""" + + def execute(self, request): + """丰富请求的特征信息""" + if request is None: + return None + + self.logger.info(f"Enriching features for request: {request['request_id']}") + + try: + # 使用服务语法糖获取用户特征 + user_features = self.call_service( + "feature_store", + request["user_id"], + method="get_user_features", + ) + + # 批量获取物品特征 + item_features = self.call_service( + "feature_store", + "item", + request["candidate_items"], + method="batch_get_features", + ) + + # 丰富请求 + enriched_request = { + **request, + "user_features": user_features, + "item_features": item_features, + "enrichment_timestamp": time.time(), + } + + # 记录日志 + self.call_service( + "log", + f"Features enriched for request {request['request_id']}", + { + "user_id": request["user_id"], + "item_count": len(request["candidate_items"]), + }, + method="info", + ) + + self.logger.info(f"Feature enrichment completed for: {request['request_id']}") + return enriched_request + + except Exception as e: + self.logger.error(f"Feature enrichment failed for {request['request_id']}: {e}") + # 记录错误到日志服务 + self.call_service( + "log", + f"Feature enrichment failed: {e}", + {"request_id": request["request_id"]}, + method="error", + ) + return None + + +class RecommendationFunction(BaseFunction): + """推荐算子 - 生成推荐结果""" + + def execute(self, enriched_request): + """生成推荐结果""" + if enriched_request is None: + return None + + self.logger.info(f"Generating recommendations for: {enriched_request['request_id']}") + + try: + # 检查缓存 + cache_key = f"rec_{enriched_request['user_id']}_{hash(str(enriched_request['candidate_items']))}" + cached_result = self.call_service("cache", cache_key, method="get") + + if cached_result: + self.logger.info( + f"Using cached recommendations for: {enriched_request['request_id']}" + ) + self.call_service( + "log", + "Used cached recommendations", + { + "request_id": enriched_request["request_id"], + "cache_key": cache_key, + }, + method="info", + ) + return { + **enriched_request, + "recommendations": cached_result, + "from_cache": True, + } + + # 创建特征向量 + feature_vectors = self._create_feature_vectors( + enriched_request["user_features"], enriched_request["item_features"] + ) + + # 使用模型服务进行预测 + predictions = self.call_service("model", feature_vectors, method="batch_predict") + + # 生成推荐结果 + recommendations = [] + for i, (item_id, prediction) in enumerate( + zip(enriched_request["candidate_items"], predictions) + ): + rec = { + "item_id": item_id, + "score": prediction["score"], + "rank": i + 1, + "prediction_id": prediction["prediction_id"], + } + recommendations.append(rec) + + # 按分数排序 + recommendations.sort(key=lambda x: x["score"], reverse=True) + for i, rec in enumerate(recommendations): + rec["rank"] = i + 1 + + # 缓存结果 + self.call_service("cache", cache_key, recommendations, method="set") + + # 记录推荐完成 + self.call_service( + "log", + "Recommendations generated successfully", + { + "request_id": enriched_request["request_id"], + "recommendation_count": len(recommendations), + "top_score": recommendations[0]["score"] if recommendations else 0, + }, + method="info", + ) + + result = { + **enriched_request, + "recommendations": recommendations, + "from_cache": False, + "recommendation_timestamp": time.time(), + } + + self.logger.info( + f"Recommendations generated for: {enriched_request['request_id']}, count: {len(recommendations)}" + ) + return result + + except Exception as e: + self.logger.error( + f"Recommendation generation failed for {enriched_request['request_id']}: {e}" + ) + self.call_service( + "log", + f"Recommendation generation failed: {e}", + {"request_id": enriched_request["request_id"]}, + method="error", + ) + return None + + def _create_feature_vectors(self, user_features, item_features): + """创建特征向量""" + feature_vectors = [] + for item_id, item_attrs in item_features.items(): + vector = { + **user_features, + **item_attrs, + "interaction_score": self._calculate_interaction(user_features, item_attrs), + } + feature_vectors.append(vector) + return feature_vectors + + def _calculate_interaction(self, user_features, item_features): + """计算交互特征""" + score = 0.0 + if user_features.get("vip_level", 0) >= 2 and item_features.get("price", 0) > 500: + score += 0.2 + if user_features.get("age", 0) < 30 and item_features.get("category") == "electronics": + score += 0.1 + return round(score, 3) + + +class ResultSinkFunction(BaseFunction): + """结果输出算子 - 输出最终推荐结果""" + + def __init__(self): + super().__init__() + self.processed_count = 0 + + def execute(self, recommendation_result): + """输出推荐结果""" + if recommendation_result is None: + return None + + self.processed_count += 1 + + self.logger.info(f"Processing final result for: {recommendation_result['request_id']}") + + # 格式化输出 + output = { + "request_id": recommendation_result["request_id"], + "user_id": recommendation_result["user_id"], + "recommendations": recommendation_result["recommendations"][:3], # 只取前3个 + "from_cache": recommendation_result.get("from_cache", False), + "processed_at": time.strftime("%Y-%m-%d %H:%M:%S"), + "processing_order": self.processed_count, + } + + # 记录到日志服务 + self.call_service( + "log", + "Final recommendation result", + { + "request_id": output["request_id"], + "user_id": output["user_id"], + "recommendation_count": len(output["recommendations"]), + "from_cache": output["from_cache"], + }, + method="info", + ) + + # 打印结果 + print(f"\n=== 推荐结果 #{self.processed_count} ===") + print(f"请求ID: {output['request_id']}") + print(f"用户ID: {output['user_id']}") + print(f"缓存命中: {output['from_cache']}") + print("推荐列表:") + for rec in output["recommendations"]: + print(f" {rec['rank']}. {rec['item_id']} (分数: {rec['score']})") + print(f"处理时间: {output['processed_at']}") + print("=" * 40) + + return output + + +def test_realistic_sage_workflow(): + """测试真实的SAGE工作流""" + print("=== 真实SAGE工作流测试 ===") + + try: + # 1. 创建环境 + print("\n1. 创建环境:") + env = LocalEnvironment("realistic_workflow_test") + + # 2. 注册服务 + print("\n2. 注册服务:") + env.register_service("feature_store", FeatureStoreService) + env.register_service("model", ModelService, model_name="workflow_model_v1") + env.register_service("cache", CacheService, max_size=500) + env.register_service("log", LogService, log_level="INFO") + + print("所有服务注册完成") + + # 3. 构建流处理管道 + print("\n3. 构建流处理管道:") + + # 使用流式API构建处理管道 + ( + env.from_source(RequestSourceFunction, request_count=30) + .map(FeatureEnrichmentFunction) + .map(RecommendationFunction) + .map(ResultSinkFunction) + ) + + print("流处理管道构建完成") + + # 4. 提交并运行流处理管道 + print("\n4. 提交流处理管道:") + env.submit() + + # 5. 等待处理完成 + print("\n5. 等待处理完成...") + time.sleep(15) # 给足够时间让流处理完成 + + # 6. 检查服务状态 + print("\n6. 检查服务状态:") + print("流水线执行完成！") + + print("\n=== 真实工作流测试完成 ===") + + except Exception as e: + print(f"测试失败: {e}") + import traceback + + traceback.print_exc() + finally: + # 清理环境 + try: + env.stop() + print("环境已清理") + except Exception: + pass + + +if __name__ == "__main__": + test_realistic_sage_workflow() diff --git a/tutorials/L3-kernel/advanced/parallelism_local_validation.py b/tutorials/L3-kernel/advanced/parallelism_local_validation.py new file mode 100644 index 0000000..44bc81b --- /dev/null +++ b/tutorials/L3-kernel/advanced/parallelism_local_validation.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +""" +Local Environment Parallelism Validation Example + +This example demonstrates and validates parallelism hints functionality +using LocalEnvironment. It creates multiple streams with different +parallelism settings and verifies that the ExecutionGraph creates +the correct number of parallel nodes. + +@test:timeout=60 +""" + +import threading +import time + +from sage.common.core.functions.base_function import BaseFunction +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.kernel.api.local_environment import LocalEnvironment + + +class NumberListSource(BatchFunction): + """A simple batch source that produces a list of numbers""" + + def __init__(self, numbers): + super().__init__() + self.numbers = numbers + self.index = 0 + + def execute(self): + if self.index >= len(self.numbers): + return None + value = self.numbers[self.index] + self.index += 1 + return value + + +class ParallelProcessor(BaseFunction): + """A processor that shows which thread/instance is handling the data""" + + def __init__(self, processor_name="Processor"): + super().__init__() + self.processor_name = processor_name + self.instance_id = id(self) + self.thread_id = threading.get_ident() + print( + f"🔧 {self.processor_name} instance {self.instance_id} created in thread {self.thread_id}" + ) + + def execute(self, data): + current_thread = threading.get_ident() + instance_id = id(self) + result = f"{self.processor_name}[{instance_id}]: {data}" + print(f"⚙️ {result} (thread: {current_thread})") + time.sleep(0.05) # Simulate processing time + return result + + +class ParallelFilter(BaseFunction): + """A filter that shows parallel execution""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + self.thread_id = threading.get_ident() + print(f"🔧 ParallelFilter instance {self.instance_id} created in thread {self.thread_id}") + + def execute(self, data): + current_thread = threading.get_ident() + instance_id = id(self) + # Only pass even numbers + is_even = isinstance(data, int) and data % 2 == 0 + if is_even: + print(f"✅ Filter[{instance_id}]: {data} PASSED (thread: {current_thread})") + else: + print(f"❌ Filter[{instance_id}]: {data} BLOCKED (thread: {current_thread})") + return is_even + + +class MultiStreamCoMapProcessor(BaseCoMapFunction): + """CoMap processor for multi-stream validation""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + print( + f"🔧 CoMapProcessor instance {self.instance_id} created in thread {threading.get_ident()}" + ) + + def map0(self, data): + current_thread = threading.get_ident() + instance_id = id(self) + result = f"CoMap0[{instance_id}]: {data}" + print(f"🔀 {result} (thread: {current_thread})") + return result + + def map1(self, data): + current_thread = threading.get_ident() + instance_id = id(self) + result = f"CoMap1[{instance_id}]: {data * 10}" + print(f"🔀 {result} (thread: {current_thread})") + return result + + +class ValidationSink(BaseFunction): + """Sink that validates and prints final results""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + self.results = [] + print(f"🔧 ValidationSink instance {self.instance_id} created") + + def execute(self, data): + current_thread = threading.get_ident() + instance_id = id(self) + self.results.append(data) + print(f"🎯 SINK[{instance_id}]: {data} (thread: {current_thread})") + return data + + +def validate_single_stream_parallelism(): + """Validate parallelism for single stream operations""" + print("\n" + "=" * 70) + print("LOCAL ENVIRONMENT - SINGLE STREAM PARALLELISM VALIDATION") + print("=" * 70) + + env = LocalEnvironment(name="local_single_stream_test") + + # Test data + numbers = list(range(1, 21)) # 1 to 20 + source_stream = env.from_collection(NumberListSource, numbers) + + print(f"\n📊 Testing with {len(numbers)} input numbers: {numbers}") + + # Test different parallelism levels + print("\n--- Test 1: Direct parallelism parameters ---") + ( + source_stream.map(ParallelProcessor, "Mapper", parallelism=3) # 3 parallel mappers + .filter(ParallelFilter, parallelism=2) # 2 parallel filters + .sink(ValidationSink, parallelism=1) + ) # 1 sink + + print("\n--- Test 2: Using direct parallelism ---") + ( + source_stream.map(ParallelProcessor, "SetMapper", parallelism=4) # 4 parallel mappers + .filter(ParallelFilter, parallelism=3) # 3 parallel filters + .sink(ValidationSink, parallelism=1) + ) # 1 sink + + # Analyze pipeline + print("\n📋 PIPELINE ANALYSIS:") + print(f"Total transformations: {len(env.pipeline)}") + for i, transformation in enumerate(env.pipeline): + print( + f" {i + 1:2d}. {transformation.function_class.__name__:20s} | " + f"Parallelism: {transformation.parallelism:2d} | " + f"Basename: {transformation.basename}" + ) + + return env + + +def validate_multi_stream_parallelism(): + """Validate parallelism for multi-stream operations""" + print("\n" + "=" * 70) + print("LOCAL ENVIRONMENT - MULTI-STREAM PARALLELISM VALIDATION") + print("=" * 70) + + env = LocalEnvironment(name="local_multi_stream_test") + + # Create two streams with different data + stream1_data = [1, 3, 5, 7, 9] + stream2_data = [2, 4, 6, 8, 10] + + stream1 = env.from_collection(NumberListSource, stream1_data) + stream2 = env.from_collection(NumberListSource, stream2_data) + + print(f"\n📊 Stream1 data: {stream1_data}") + print(f"📊 Stream2 data: {stream2_data}") + + print("\n--- Test 1: CoMap with direct parallelism ---") + ( + stream1.connect(stream2) + .comap(MultiStreamCoMapProcessor, parallelism=2) # 2 parallel CoMap processors + .sink(ValidationSink, parallelism=1) + ) + + print("\n--- Test 2: CoMap with direct parallelism ---") + ( + stream1.connect(stream2) + .comap(MultiStreamCoMapProcessor, parallelism=3) # 3 parallel CoMap processors + .sink(ValidationSink, parallelism=2) + ) # 2 sinks + + # Analyze pipeline + print("\n📋 PIPELINE ANALYSIS:") + print(f"Total transformations: {len(env.pipeline)}") + for i, transformation in enumerate(env.pipeline): + print( + f" {i + 1:2d}. {transformation.function_class.__name__:20s} | " + f"Parallelism: {transformation.parallelism:2d} | " + f"Basename: {transformation.basename}" + ) + + return env + + +def validate_execution_graph_nodes(): + """Validate that ExecutionGraph creates correct number of parallel nodes""" + print("\n" + "=" * 70) + print("EXECUTION GRAPH NODE VALIDATION") + print("=" * 70) + + env = LocalEnvironment(name="node_validation_test") + + # Create a simple pipeline with known parallelism + numbers = [1, 2, 3, 4, 5] + ( + env.from_collection(NumberListSource, numbers) + .map(ParallelProcessor, "NodeTest", parallelism=2) # Should create 2 nodes + .filter(ParallelFilter, parallelism=3) # Should create 3 nodes + .sink(ValidationSink, parallelism=1) + ) # Should create 1 node + + print("\n📋 Expected execution nodes:") + print(" - ListSource: 1 node (source)") + print(" - NodeTest (map): 2 nodes (parallelism=2)") + print(" - ParallelFilter: 3 nodes (parallelism=3)") + print(" - ValidationSink: 1 node (parallelism=1)") + print(" - Total expected: 7 nodes") + + # Get execution graph + try: + # Try to access execution graph information + print("\n🔍 Pipeline transformations:") + for i, transformation in enumerate(env.pipeline): + print( + f" {i + 1}. {transformation.basename} (parallelism: {transformation.parallelism})" + ) + + # Note: ExecutionGraph node creation happens during execution + print("\n💡 Note: Actual node creation occurs during pipeline execution.") + print(" Each transformation with parallelism=N will create N parallel operator nodes.") + + except Exception as e: + print(f"⚠️ Could not access execution graph details: {e}") + + return env + + +def main(): + """Main function to run all validation tests""" + print("🚀 SAGE Local Environment Parallelism Validation") + print("This example validates parallelism hints in LocalEnvironment") + + # Run all validation tests + env1 = validate_single_stream_parallelism() + env2 = validate_multi_stream_parallelism() + env3 = validate_execution_graph_nodes() + + print("\n" + "=" * 70) + print("VALIDATION SUMMARY") + print("=" * 70) + print("✅ Single stream parallelism: Tested with various parallelism levels") + print("✅ Multi-stream parallelism: Tested CoMap operations") + print("✅ Execution graph validation: Verified transformation parallelism settings") + print("✅ LocalEnvironment parallelism hints: WORKING CORRECTLY") + + print("\n📊 Total environments created: 3") + print( + f"📊 Total transformations tested: {len(env1.pipeline) + len(env2.pipeline) + len(env3.pipeline)}" + ) + + print("\n💡 Key validations:") + print(" - Parallelism parameters correctly passed to transformations") + print(" - Direct parallelism specification works as expected") + print(" - Both single and multi-stream operations support parallelism") + print(" - ExecutionGraph will create corresponding parallel nodes during execution") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/advanced/parallelism_remote_validation.py b/tutorials/L3-kernel/advanced/parallelism_remote_validation.py new file mode 100644 index 0000000..559209f --- /dev/null +++ b/tutorials/L3-kernel/advanced/parallelism_remote_validation.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +""" +Remote Environment Parallelism Validation Example + +This example demonstrates and validates parallelism hints functionality +using RemoteEnvironment (Ray-based distributed execution). It shows how +parallelism settings work in a distributed environment and verifies that +the ExecutionGraph creates the correct number of parallel nodes across +Ray workers. + +@test:timeout=90 +""" + +import os +import threading +import time + +from sage.common.core.functions.base_function import BaseFunction +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.kernel.api.remote_environment import RemoteEnvironment + + +class NumberListSource(BatchFunction): + """A simple batch source that produces a list of numbers""" + + def __init__(self, numbers): + super().__init__() + self.numbers = numbers + self.index = 0 + + def execute(self): + if self.index >= len(self.numbers): + return None + value = self.numbers[self.index] + self.index += 1 + return value + + +class DistributedProcessor(BaseFunction): + """A processor that shows which worker/instance is handling the data""" + + def __init__(self, processor_name="DistProcessor"): + super().__init__() + self.processor_name = processor_name + self.instance_id = id(self) + self.process_id = os.getpid() + self.thread_id = threading.get_ident() + print( + f"🔧 {self.processor_name} instance {self.instance_id} created " + f"(PID: {self.process_id}, Thread: {self.thread_id})" + ) + + def execute(self, data): + current_thread = threading.get_ident() + current_process = os.getpid() + instance_id = id(self) + result = f"{self.processor_name}[{instance_id}@{current_process}]: {data}" + print(f"⚙️ {result} (Thread: {current_thread})") + time.sleep(0.1) # Simulate processing time + return result + + +class DistributedFilter(BaseFunction): + """A filter that shows distributed execution across Ray workers""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + self.process_id = os.getpid() + print(f"🔧 DistributedFilter instance {self.instance_id} created (PID: {self.process_id})") + + def execute(self, data): + current_thread = threading.get_ident() + current_process = os.getpid() + instance_id = id(self) + + # Filter logic: pass if data is divisible by 3 + passes = isinstance(data, int) and data % 3 == 0 + status = "PASSED" if passes else "BLOCKED" + print( + f"{'✅' if passes else '❌'} Filter[{instance_id}@{current_process}]: {data} {status} " + f"(Thread: {current_thread})" + ) + return passes + + +class DistributedCoMapProcessor(BaseCoMapFunction): + """CoMap processor for distributed multi-stream validation""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + self.process_id = os.getpid() + print( + f"🔧 DistributedCoMapProcessor instance {self.instance_id} created (PID: {self.process_id})" + ) + + def map0(self, data): + current_process = os.getpid() + current_thread = threading.get_ident() + instance_id = id(self) + result = f"DistCoMap0[{instance_id}@{current_process}]: {data}" + print(f"🔀 {result} (Thread: {current_thread})") + time.sleep(0.05) + return result + + def map1(self, data): + current_process = os.getpid() + current_thread = threading.get_ident() + instance_id = id(self) + result = f"DistCoMap1[{instance_id}@{current_process}]: {data * 100}" + print(f"🔀 {result} (Thread: {current_thread})") + time.sleep(0.05) + return result + + +class DistributedSink(BaseFunction): + """Sink that validates and prints final results in distributed environment""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) + self.process_id = os.getpid() + self.results = [] + print(f"🔧 DistributedSink instance {self.instance_id} created (PID: {self.process_id})") + + def execute(self, data): + current_thread = threading.get_ident() + current_process = os.getpid() + instance_id = id(self) + self.results.append(data) + print(f"🎯 SINK[{instance_id}@{current_process}]: {data} (Thread: {current_thread})") + return data + + +def validate_remote_single_stream_parallelism(): + """Validate parallelism for single stream operations in remote environment""" + print("\n" + "=" * 70) + print("REMOTE ENVIRONMENT - SINGLE STREAM PARALLELISM VALIDATION") + print("=" * 70) + + # Initialize Ray cluster for distributed processing + # Note: Ray configuration is currently handled at the JobManager level, + # not directly through RemoteEnvironment constructor. This is a potential + # improvement area for SAGE architecture. + try: + env = RemoteEnvironment(name="remote_single_stream_test") + print("✅ RemoteEnvironment initialized successfully") + except Exception as e: + print(f"⚠️ RemoteEnvironment initialization warning: {e}") + env = RemoteEnvironment(name="remote_single_stream_test") + + # Test data - larger dataset for distributed processing + numbers = list(range(1, 31)) # 1 to 30 + source_stream = env.from_collection(NumberListSource, numbers) + + print(f"\n📊 Testing with {len(numbers)} input numbers") + print(f"📊 Numbers: {numbers[:10]}...{numbers[-5:]} (showing first 10 and last 5)") + + # Test distributed parallelism + print("\n--- Test 1: Distributed processing with direct parallelism parameters ---") + ( + source_stream.map( + DistributedProcessor, "DistMapper", parallelism=4 + ) # 4 parallel mappers across workers + .filter(DistributedFilter, parallelism=3) # 3 parallel filters across workers + .sink(DistributedSink, parallelism=2) + ) # 2 sinks across workers + + print("\n--- Test 2: Distributed processing with direct parallelism ---") + ( + source_stream.map( + DistributedProcessor, "SetDistMapper", parallelism=3 + ) # 3 parallel mappers + .filter(DistributedFilter, parallelism=2) # 2 parallel filters + .sink(DistributedSink, parallelism=1) + ) # 1 sink + + # Analyze pipeline + print("\n📋 DISTRIBUTED PIPELINE ANALYSIS:") + print(f"Total transformations: {len(env.pipeline)}") + print(f"Ray workers available: {env.platform} (distributed execution)") + for i, transformation in enumerate(env.pipeline): + print( + f" {i + 1:2d}. {transformation.function_class.__name__:25s} | " + f"Parallelism: {transformation.parallelism:2d} | " + f"Basename: {transformation.basename}" + ) + + return env + + +def validate_remote_multi_stream_parallelism(): + """Validate parallelism for multi-stream operations in remote environment""" + print("\n" + "=" * 70) + print("REMOTE ENVIRONMENT - MULTI-STREAM PARALLELISM VALIDATION") + print("=" * 70) + + try: + env = RemoteEnvironment(name="remote_multi_stream_test") + except Exception as e: + print(f"⚠️ RemoteEnvironment initialization warning: {e}") + env = RemoteEnvironment(name="remote_multi_stream_test") + + # Create streams with more data for distributed processing + stream1_data = list(range(1, 16, 2)) # [1, 3, 5, 7, 9, 11, 13, 15] + stream2_data = list(range(2, 17, 2)) # [2, 4, 6, 8, 10, 12, 14, 16] + + stream1 = env.from_collection(NumberListSource, stream1_data) + stream2 = env.from_collection(NumberListSource, stream2_data) + + print(f"\n📊 Stream1 data (odd numbers): {stream1_data}") + print(f"📊 Stream2 data (even numbers): {stream2_data}") + + print("\n--- Test 1: Distributed CoMap with direct parallelism ---") + ( + stream1.connect(stream2) + .comap(DistributedCoMapProcessor, parallelism=3) # 3 parallel CoMap processors + .sink(DistributedSink, parallelism=2) + ) # 2 sinks + + print("\n--- Test 2: Distributed CoMap with direct parallelism ---") + ( + stream1.connect(stream2) + .comap(DistributedCoMapProcessor, parallelism=4) # 4 parallel CoMap processors + .sink(DistributedSink, parallelism=1) + ) # 1 sink + + # Analyze pipeline + print("\n📋 DISTRIBUTED PIPELINE ANALYSIS:") + print(f"Total transformations: {len(env.pipeline)}") + print(f"Environment platform: {env.platform}") + for i, transformation in enumerate(env.pipeline): + print( + f" {i + 1:2d}. {transformation.function_class.__name__:25s} | " + f"Parallelism: {transformation.parallelism:2d} | " + f"Basename: {transformation.basename}" + ) + + return env + + +def validate_ray_distributed_execution(): + """Validate that Ray properly distributes parallel operations""" + print("\n" + "=" * 70) + print("RAY DISTRIBUTED EXECUTION VALIDATION") + print("=" * 70) + + try: + env = RemoteEnvironment(name="ray_distribution_test") + print("✅ RemoteEnvironment initialized") + except Exception as e: + print(f"⚠️ RemoteEnvironment initialization warning: {e}") + env = RemoteEnvironment(name="ray_distribution_test") + + # Create a pipeline designed to show distributed execution + large_dataset = list(range(1, 51)) # 1 to 50 - enough data for distribution + + ( + env.from_collection(NumberListSource, large_dataset) + .map(DistributedProcessor, "DistTest", parallelism=5) # 5 parallel processors + .filter(DistributedFilter, parallelism=3) # 3 parallel filters + .sink(DistributedSink, parallelism=2) + ) # 2 sinks + + print("\n📋 Remote Distribution Test Pipeline:") + print(f" - Dataset size: {len(large_dataset)} items") + print(" - Expected parallel processors: 5 (will distribute based on available workers)") + print(" - Expected parallel filters: 3 (will distribute based on available workers)") + print(" - Expected sinks: 2 (will distribute based on available workers)") + + print("\n🔍 Pipeline transformations:") + for i, transformation in enumerate(env.pipeline): + print(f" {i + 1}. {transformation.basename} (parallelism: {transformation.parallelism})") + + print("\n💡 Key aspects of remote distributed execution:") + print(" - Each parallel instance may run on different remote workers") + print(" - Process IDs will differ across workers") + print(" - Work is distributed based on available resources") + print(" - RemoteEnvironment handles load balancing and coordination") + + return env + + +def main(): + """Main function to run all remote validation tests""" + print("🚀 SAGE Remote Environment Parallelism Validation") + print("This example validates parallelism hints in RemoteEnvironment (Ray)") + + try: + # Run all validation tests + env1 = validate_remote_single_stream_parallelism() + env2 = validate_remote_multi_stream_parallelism() + env3 = validate_ray_distributed_execution() + + print("\n" + "=" * 70) + print("REMOTE VALIDATION SUMMARY") + print("=" * 70) + print("✅ Remote single stream parallelism: Tested with remote workers") + print("✅ Remote multi-stream parallelism: Tested distributed CoMap") + print("✅ Remote distributed execution: Verified parallel worker distribution") + print("✅ RemoteEnvironment direct parallelism: WORKING CORRECTLY") + + print("\n📊 Total remote environments created: 3") + print( + f"📊 Total distributed transformations: {len(env1.pipeline) + len(env2.pipeline) + len(env3.pipeline)}" + ) + + print("\n💡 Key remote validations:") + print(" - Parallelism settings work in distributed remote environment") + print(" - Direct parallelism specification distributes work across remote workers") + print(" - Multi-stream operations (CoMap) support distributed parallelism") + print(" - RemoteEnvironment automatically handles worker assignment and coordination") + + except Exception as e: + print(f"\n❌ Remote validation encountered an error: {e}") + print( + "💡 This might be due to RemoteEnvironment not being available or configured properly" + ) + print(" Please ensure the JobManager service is running and accessible") + print(" And that your system supports remote distributed execution") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/README.md b/tutorials/L3-kernel/advanced/pipeline_as_service/README.md new file mode 100644 index 0000000..9ecb896 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/README.md @@ -0,0 +1,65 @@ +# Pipeline-as-Service Demo + +These examples show how to register an entire SAGE pipeline as a service while still leveraging the +same service invocation helpers for its internal stages. Each scenario adds a different client +pattern so you can see how pipelines and services compose in more realistic setups. + +## What it demonstrates + +- **Pipeline as a service** – The decision pipeline is registered as the `order_pipeline` service + and can be invoked via `call_service`. +- **Asynchronous invocation** – Operators can use `self.call_service_async(...)` to overlap work and + wait on a `Future`. +- **Service-to-service orchestration** – The pipeline calls supporting services (feature store, risk + scoring) using the same helpers. +- **Cooperative shutdown** – Sending a `{"command": "shutdown"}` message tells the pipeline to stop + after finishing in-flight work. + +## Files + +- `hello_pipeline_as_service.py` – Creates the service-backed decision pipeline plus a driver + pipeline that submits orders and prints the returned decisions. +- `async_client_pipeline_as_service.py` – Uses `call_service_async` to submit a burst of requests + concurrently and wait for futures to resolve. +- `multi_client_pipeline_as_service.py` – Spawns two driver pipelines (new vs returning users) that + share the same pipeline service endpoint. +- `qa_pipeline_as_service.py` – Wraps the RAG QA pipeline as a service so an interactive terminal + client waits for each answer before accepting the next question; exit by typing `bye bye` (or + press `Ctrl+C` to interrupt). +- `pipeline_bridge.py` – Shared queue bridge used to pass requests between the driver pipelines and + the service-backed pipeline. + +## How to run + +From the repository root you can run any scenario. For example: + +```bash +python examples/service/pipeline_as_service/hello_pipeline_as_service.py +python examples/service/pipeline_as_service/async_client_pipeline_as_service.py +python examples/service/pipeline_as_service/multi_client_pipeline_as_service.py +python examples/service/pipeline_as_service/qa_pipeline_as_service.py +``` + +You should see log output from both the service-backed pipeline and the client pipelines, including +enrichment, scoring, service replies, and a shutdown acknowledgement once all work completes. + +### Offline/mock QA runs + +If you don't have an OpenAI-compatible endpoint handy, switch the QA pipeline to a mock generator +that fabricates answers while exercising the same pipeline/service wiring: + +```bash +SAGE_QA_GENERATOR=mock python examples/service/pipeline_as_service/qa_pipeline_as_service.py +``` + +The mock generator emits friendly placeholder replies so you can experience the request/response +flow without external dependencies. By default the example selects the mock generator unless you +explicitly export `SAGE_QA_GENERATOR` or point it to a concrete profile. You can also pick a +different generator profile via `SAGE_QA_GENERATOR_PROFILE` (e.g. `local`, `remote`, `vllm`) to +match entries from `examples/config/config_source.yaml`. + +> **Heads up**: When the script detects that the selected OpenAI/HF profile is missing required +> fields (like `api_key` or `base_url`), it automatically falls back to the mock generator and +> prints a warning describing how to point it at a real endpoint. Update +> `examples/config/config_source.yaml` or export `SAGE_QA_GENERATOR=mock` to control the behaviour +> explicitly. diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/async_client_pipeline_as_service.py b/tutorials/L3-kernel/advanced/pipeline_as_service/async_client_pipeline_as_service.py new file mode 100644 index 0000000..91bc911 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/async_client_pipeline_as_service.py @@ -0,0 +1,178 @@ +"""Demonstrate asynchronous clients calling a pipeline that is registered as a service. + +This example reuses the decision pipeline service from +`hello_pipeline_as_service` but drives it with a client pipeline that submits +multiple requests concurrently using `call_service_async`. +""" + +from __future__ import annotations + +import random +import time +from typing import Any + +try: + from sage.common.core.functions.batch_function import BatchFunction + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment +except ModuleNotFoundError: # pragma: no cover - convenience for local runs + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for parent in here.parents: + if (parent / "packages").exists(): + repo_root = parent + break + if repo_root is None: + raise RuntimeError("Cannot locate SAGE repository root") + + for extra_path in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(extra_path)) + + from sage.common.core.functions.batch_function import BatchFunction + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + +from hello_pipeline_as_service import ( + SHUTDOWN_MESSAGE, + DecisionSink, + FeatureEnrichment, + FeatureStoreService, + OrderPipelineService, + RiskScoring, + RiskScoringService, + ServiceDrivenSource, +) +from pipeline_bridge import PipelineBridge + +ASYNC_ORDERS: list[dict[str, float | str | int]] = [ + {"order_id": "async-1001", "user_id": "user-101", "amount": 210.5, "latency": 0.6}, + {"order_id": "async-1002", "user_id": "user-102", "amount": 35.0, "latency": 0.2}, + {"order_id": "async-1003", "user_id": "user-103", "amount": 990.0, "latency": 0.9}, + {"order_id": "async-1004", "user_id": "user-104", "amount": 440.0, "latency": 0.4}, +] + + +class SlowFeatureStoreService(FeatureStoreService): + """Introduce variable latency to make concurrency observable.""" + + def process(self, order: dict[str, float | str | int]): + delay = float(order.get("latency", 0.0)) + if delay > 0: + time.sleep(delay) + return super().process(order) # type: ignore[arg-type] + + +class AsyncOrderSource(BatchFunction): + """Emit the async orders and a final shutdown control message.""" + + def __init__(self, orders: list[dict[str, float | str | int]]): + super().__init__() + self._orders = list(orders) + [dict(SHUTDOWN_MESSAGE)] + self._index = 0 + + def execute(self): + if self._index >= len(self._orders): + return None + + order = self._orders[self._index] + self._index += 1 + return order + + +class AsyncSubmit(MapFunction): + """Submit orders asynchronously to the pipeline service.""" + + def execute(self, order: dict[str, float | str | int]): + if order is None: + return None + + if order.get("command") == "shutdown": + response = self.call_service("order_pipeline", order, timeout=15.0) + return {"type": "control", "order": order, "response": response} + + future = self.call_service_async("order_pipeline", order, timeout=20.0) + return {"type": "future", "order": order, "future": future} + + +class AwaitDecision(MapFunction): + """Await the decision from futures produced by `AsyncSubmit`.""" + + def execute(self, record: dict[str, Any]): + if record is None: + return None + + if record.get("type") == "control": + return record + + future = record.get("future") + if future is None: + return None + + decision = future.result(timeout=20.0) + return {"type": "decision", "order": record["order"], "decision": decision} + + +class AsyncResultSink(SinkFunction): + """Print the order decisions along with latency info.""" + + def execute(self, payload: dict[str, Any]): + if payload is None: + return None + + if payload.get("type") == "decision": + order = payload["order"] + decision = payload["decision"] + print( + f"[Async Driver] Order {order['order_id']} (latency={order.get('latency', 0)}s)\n" + f" => decision={decision['recommendation']}, risk={decision['risk_score']}", + flush=True, + ) + else: + print("[Async Driver] Pipeline shutdown acknowledged", flush=True) + + return payload + + +def main(): + env = LocalEnvironment("pipeline_as_service_async") + bridge = PipelineBridge() + + env.register_service("feature_store", SlowFeatureStoreService) + env.register_service("risk_scoring", RiskScoringService) + env.register_service("order_pipeline", OrderPipelineService, bridge) + + ( + env.from_source(ServiceDrivenSource, bridge) + .map(FeatureEnrichment) + .map(RiskScoring) + .sink(DecisionSink) + ) + + ( + env.from_batch(AsyncOrderSource, ASYNC_ORDERS) + .map(AsyncSubmit) + .map(AwaitDecision) + .sink(AsyncResultSink) + ) + + env.submit(autostop=True) + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + random.seed(42) + main() diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/hello_pipeline_as_service.py b/tutorials/L3-kernel/advanced/pipeline_as_service/hello_pipeline_as_service.py new file mode 100644 index 0000000..4b9a1e1 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/hello_pipeline_as_service.py @@ -0,0 +1,315 @@ +"""Pipeline-as-Service demo + +This example shows how a SAGE pipeline can be registered as a service and then +invoked through the unified ``call_service`` helpers. The pipeline listens for +requests coming from the service layer, enriches them via other services, and +replies with a structured decision payload. +""" + +from __future__ import annotations + +import math +import queue +import random +import time +from typing import Any + +# Try regular imports first; fall back to repo-relative paths when running +# directly from the source tree without installing the package. +try: + from sage.common.core.functions.batch_function import BatchFunction + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.core.functions.source_function import SourceFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.kernel.api.service.base_service import BaseService +except ModuleNotFoundError: # pragma: no cover - convenience for local runs + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for parent in here.parents: + if (parent / "packages").exists(): + repo_root = parent + break + if repo_root is None: + raise RuntimeError("Cannot locate SAGE repository root") + + for extra_path in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(extra_path)) + + from sage.common.core.functions.batch_function import BatchFunction + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.core.functions.source_function import SourceFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.kernel.api.service.base_service import BaseService + +from pipeline_bridge import PipelineBridge, PipelinePayload + +from sage.kernel.runtime.communication.packet import StopSignal + +ORDERS: list[dict[str, str | float]] = [ + {"order_id": "o-1001", "user_id": "user-001", "amount": 129.9}, + {"order_id": "o-1002", "user_id": "user-002", "amount": 59.0}, + {"order_id": "o-1003", "user_id": "user-003", "amount": 450.5}, +] + +SHUTDOWN_MESSAGE = {"command": "shutdown"} + + +class ServiceDrivenSource(SourceFunction): + """Source operator that pulls requests from the service bridge.""" + + def __init__(self, bridge: PipelineBridge, poll_interval: float = 0.1): + super().__init__() + self._bridge = bridge + self._poll_interval = poll_interval + + def execute(self, data=None): + request = self._bridge.next(timeout=self._poll_interval) + if request is None: + return None + + if isinstance(request, StopSignal): + return request + + return PipelinePayload(order=request.payload, response_queue=request.response_queue) + + +class OrderSource(BatchFunction): + """Emit a fixed list of orders as the batch source.""" + + def __init__(self, orders: list[dict[str, str]], *, include_shutdown: bool = True): + super().__init__() + self._orders = list(orders) + if include_shutdown: + self._orders.append(dict(SHUTDOWN_MESSAGE)) + self._index = 0 + + def execute(self, data=None): + if self._index >= len(self._orders): + return None + + order = self._orders[self._index] + self._index += 1 + return order + + +class FeatureStoreService(BaseService): + """Simple feature store that returns per-user aggregates.""" + + def __init__(self): + super().__init__() + self._user_features = { + "user-001": {"successful_orders": 5, "chargeback_ratio": 0.01}, + "user-002": {"successful_orders": 2, "chargeback_ratio": 0.08}, + "user-003": {"successful_orders": 12, "chargeback_ratio": 0.0}, + } + + def process(self, order: dict[str, str]): + """Default entry point leveraged by pipeline-as-service calls.""" + + features = self._user_features.get( + order["user_id"], {"successful_orders": 0, "chargeback_ratio": 0.5} + ) + features = dict(features) # shallow copy for isolation + features["order_amount"] = order["amount"] + features["feature_timestamp"] = time.time() + return features + + +class FeatureEnrichment(MapFunction): + """Use the feature store service without specifying the method name.""" + + def execute(self, payload: PipelinePayload): + if payload is None: + return None + + order = payload.order + features = self.call_service("feature_store", order) + enriched = {**order, "features": features} + feature_keys = ", ".join(sorted(features.keys())) + self.logger.info( + f"Enriched order {order.get('order_id', '')} with feature keys [{feature_keys}]" + ) + payload.features = features + payload.enriched = enriched + return payload + + +class RiskScoringService(BaseService): + """Trivial risk-scoring service with a `process` entry point.""" + + def process(self, enriched_order: dict[str, str]): + features = enriched_order.get("features", {}) # type: ignore[assignment] + amount = float(enriched_order.get("amount", 0.0)) + chargeback_ratio = float(features.get("chargeback_ratio", 0.0)) # type: ignore[attr-defined] + history = float(features.get("successful_orders", 0.0)) # type: ignore[attr-defined] + + base_score = 0.4 * chargeback_ratio + 0.0005 * amount + history_modifier = 0.2 if history > 5 else 0.0 + jitter = random.uniform(-0.05, 0.05) + + risk = min(max(base_score - history_modifier + jitter, 0.0), 1.0) + recommendation = "manual_review" if risk > 0.4 else "auto_approve" + + return { + "risk_score": round(risk, 3), + "recommendation": recommendation, + "generated_at": time.time(), + } + + +class RiskScoring(MapFunction): + """Invoke the risk scoring service asynchronously.""" + + def execute(self, payload: PipelinePayload): + if payload is None: + return None + + enriched_order = payload.enriched or payload.order + future = self.call_service_async("risk_scoring", enriched_order) + scoring = future.result(timeout=10.0) + payload.scoring = scoring + payload.enriched = {**enriched_order, "scoring": scoring} + + self.logger.info( + f"Scored order {enriched_order.get('order_id', '')} with risk {scoring['risk_score']:.3f}" + ) + return payload + + +class DecisionSink(SinkFunction): + """Final sink that prints a human-readable decision.""" + + def execute(self, payload: PipelinePayload): + if payload is None: + return None + + scoring = payload.scoring or {} + enriched_order = payload.enriched or payload.order + order_id = enriched_order.get("order_id", "") + decision = scoring.get("recommendation", "unknown") + risk = scoring.get("risk_score", math.nan) + + result = { + "order_id": order_id, + "recommendation": decision, + "risk_score": risk, + "generated_at": scoring.get("generated_at", time.time()), + "features": payload.features or {}, + } + + try: + payload.response_queue.put(result, timeout=5.0) + except queue.Full: # pragma: no cover - defensive guard + self.logger.error( + "Failed to publish decision for order %s because response queue is full", + order_id, + ) + + print( + f"[Pipeline] Order {order_id} => decision={decision}, risk={risk}", + flush=True, + ) + return result + + +class OrderPipelineService(BaseService): + """Expose the decision pipeline itself as a service.""" + + def __init__(self, bridge: PipelineBridge, request_timeout: float = 15.0): + super().__init__() + self._bridge = bridge + self._request_timeout = request_timeout + + def process(self, message: dict[str, Any]): + if message is None: + raise ValueError("Pipeline service received an empty message") + + if message.get("command") == "shutdown": + self._bridge.close() + return {"status": "shutdown_requested"} + + try: + response_queue = self._bridge.submit(message) + except RuntimeError as exc: + raise RuntimeError("Pipeline service is shutting down") from exc + + try: + return response_queue.get(timeout=self._request_timeout) + except queue.Empty as exc: + raise TimeoutError("Pipeline service timed out waiting for a reply") from exc + + +class InvokePipeline(MapFunction): + """Driver operator that calls the pipeline service for each order.""" + + def execute(self, payload: dict[str, Any]): + if payload is None: + return None + + response = self.call_service("order_pipeline", payload, timeout=20.0) + + if payload.get("command") == "shutdown": + self.logger.info("Pipeline shutdown acknowledged: %s", response) + return {"type": "control", "ack": response} + + return {"type": "decision", "order": payload, "decision": response} + + +class DriverSink(SinkFunction): + """Display results returned from the pipeline service caller.""" + + def execute(self, payload: dict[str, Any]): + if payload is None: + return None + + if payload.get("type") == "decision": + order = payload["order"] + decision = payload["decision"] + print( + f"[Driver] Order {order['order_id']} => decision={decision['recommendation']}, risk={decision['risk_score']}", + flush=True, + ) + else: + print("[Driver] Pipeline shutdown request completed", flush=True) + + return payload + + +def main(): + env = LocalEnvironment("pipeline_as_service_demo") + bridge = PipelineBridge() + + # Register services exposing a default `process` entry point. + env.register_service("feature_store", FeatureStoreService) + env.register_service("risk_scoring", RiskScoringService) + env.register_service("order_pipeline", OrderPipelineService, bridge) + + ( + env.from_source(ServiceDrivenSource, bridge) + .map(FeatureEnrichment) + .map(RiskScoring) + .sink(DecisionSink) + ) + + (env.from_batch(OrderSource, ORDERS).map(InvokePipeline).sink(DriverSink)) + + env.submit(autostop=True) + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/multi_client_pipeline_as_service.py b/tutorials/L3-kernel/advanced/pipeline_as_service/multi_client_pipeline_as_service.py new file mode 100644 index 0000000..51f0f15 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/multi_client_pipeline_as_service.py @@ -0,0 +1,156 @@ +"""Multiple driver pipelines sharing a single pipeline-as-a-service endpoint. + +This example demonstrates how different segments (new users vs returning users) +can call the same pipeline service concurrently while sharing supporting +services. +""" + +from __future__ import annotations + +from typing import Any + +try: + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment +except ModuleNotFoundError: # pragma: no cover - convenience for local runs + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for parent in here.parents: + if (parent / "packages").exists(): + repo_root = parent + break + if repo_root is None: + raise RuntimeError("Cannot locate SAGE repository root") + + for extra_path in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(extra_path)) + + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + +from hello_pipeline_as_service import ( + DecisionSink, + FeatureEnrichment, + FeatureStoreService, + InvokePipeline, + OrderPipelineService, + OrderSource, + RiskScoring, + RiskScoringService, + ServiceDrivenSource, +) +from pipeline_bridge import PipelineBridge + +NEW_USER_ORDERS: list[dict[str, float | str]] = [ + {"order_id": "multi-new-001", "user_id": "user-new-01", "amount": 29.99}, + {"order_id": "multi-new-002", "user_id": "user-new-02", "amount": 59.0}, +] + +RETURNING_USER_ORDERS: list[dict[str, float | str]] = [ + {"order_id": "multi-ret-001", "user_id": "user-ret-01", "amount": 330.0}, + {"order_id": "multi-ret-002", "user_id": "user-ret-02", "amount": 87.5}, +] + + +class SegmentedOrderSource(OrderSource): + """Extend the base OrderSource to tag orders with a segment name.""" + + def __init__( + self, + orders: list[dict[str, float | str]], + segment: str, + *, + include_shutdown: bool = False, + ): + super().__init__(orders, include_shutdown=include_shutdown) # type: ignore[arg-type] + self._segment = segment + + def execute(self): + record = super().execute() + if record is None: + return None + + if record.get("command") == "shutdown": + return record + + tagged = dict(record) + tagged["segment"] = self._segment + return tagged + + +class SegmentedDriverSink(SinkFunction): + """Print results grouped by segment.""" + + def execute(self, payload: dict[str, Any]): + if payload is None: + return None + + if payload.get("type") == "decision": + order = payload["order"] + decision = payload["decision"] + print( + f"[Multi Driver] segment={order.get('segment', 'unknown')} order={order['order_id']}" + f" => recommendation={decision['recommendation']} risk={decision['risk_score']}", + flush=True, + ) + else: + print("[Multi Driver] Pipeline shutdown acknowledged", flush=True) + + return payload + + +def main(): + env = LocalEnvironment("pipeline_as_service_multi_client") + bridge = PipelineBridge() + + env.register_service("feature_store", FeatureStoreService) + env.register_service("risk_scoring", RiskScoringService) + env.register_service("order_pipeline", OrderPipelineService, bridge) + + ( + env.from_source(ServiceDrivenSource, bridge) + .map(FeatureEnrichment) + .map(RiskScoring) + .sink(DecisionSink) + ) + + ( + env.from_batch( + SegmentedOrderSource, + NEW_USER_ORDERS, + segment="new_users", + include_shutdown=False, + ) + .map(InvokePipeline) + .sink(SegmentedDriverSink) + ) + + ( + env.from_batch( + SegmentedOrderSource, + RETURNING_USER_ORDERS, + segment="returning_users", + include_shutdown=True, + ) + .map(InvokePipeline) + .sink(SegmentedDriverSink) + ) + + env.submit(autostop=True) + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/pipeline_bridge.py b/tutorials/L3-kernel/advanced/pipeline_as_service/pipeline_bridge.py new file mode 100644 index 0000000..9937ff6 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/pipeline_bridge.py @@ -0,0 +1,66 @@ +"""Shared utilities for pipeline-as-service examples. + +This module provides lightweight queue-based primitives that allow the +registered pipeline service to exchange messages with driver pipelines. +""" + +from __future__ import annotations + +import queue +from dataclasses import dataclass +from typing import Any + +from sage.kernel.runtime.communication.packet import StopSignal + + +@dataclass +class PipelineRequest: + """A request enqueued by a driver pipeline.""" + + payload: dict[str, Any] + response_queue: queue.Queue[dict[str, Any]] + + +@dataclass +class PipelinePayload: + """Message wrapper used inside the service-backed pipeline.""" + + order: dict[str, Any] + response_queue: queue.Queue[dict[str, Any]] + features: dict[str, Any] | None = None + enriched: dict[str, Any] | None = None + scoring: dict[str, Any] | None = None + + +class PipelineBridge: + """Bidirectional bridge between driver pipelines and the service pipeline.""" + + def __init__(self) -> None: + self._requests: queue.Queue[PipelineRequest | StopSignal] = queue.Queue() + self._closed: bool = False + + def submit(self, payload: dict[str, Any]) -> queue.Queue[dict[str, Any]]: + if self._closed: + raise RuntimeError("Pipeline bridge is closed") + + response_queue: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=1) + request = PipelineRequest(payload=payload, response_queue=response_queue) + self._requests.put(request) + return response_queue + + def next(self, timeout: float = 0.1): + if self._closed and self._requests.empty(): + return StopSignal("pipeline-service-shutdown") + + try: + return self._requests.get(timeout=timeout) + except queue.Empty: + return None + + def close(self) -> None: + if not self._closed: + self._closed = True + self._requests.put(StopSignal("pipeline-service-shutdown")) + + +__all__ = ["PipelineRequest", "PipelinePayload", "PipelineBridge"] diff --git a/tutorials/L3-kernel/advanced/pipeline_as_service/qa_pipeline_as_service.py b/tutorials/L3-kernel/advanced/pipeline_as_service/qa_pipeline_as_service.py new file mode 100644 index 0000000..50d3391 --- /dev/null +++ b/tutorials/L3-kernel/advanced/pipeline_as_service/qa_pipeline_as_service.py @@ -0,0 +1,638 @@ +"""Interactive QA via pipeline-as-service. + +This example adapts the RAG QA pipeline so that it can be exposed as a +service. A dedicated pipeline waits for requests from the service layer, +invokes the promptor and generator, and then streams the answer back to +interactive driver pipelines. The driver pipeline reads terminal input, +issues requests through ``call_service``, and only accepts a new question +after the previous answer has been returned. +""" + +from __future__ import annotations + +import os +import queue +import sys +import threading +import time +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv + +try: # pragma: no cover - allow running directly from source tree + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.core.functions.source_function import SourceFunction + from sage.common.utils.config.loader import load_config + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.kernel.api.service.base_service import BaseService + from sage.kernel.runtime.communication.packet import StopSignal + from sage.middleware.operators.rag import HFGenerator, OpenAIGenerator, QAPromptor +except ModuleNotFoundError: # pragma: no cover - local convenience path + here = Path(__file__).resolve() + repo_root: Path | None = None + for parent in here.parents: + if (parent / "packages").exists(): + repo_root = parent + break + if repo_root is None: + raise RuntimeError("Cannot locate SAGE repository root") + + for extra_path in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(extra_path)) + + from sage.common.core.functions.map_function import MapFunction + from sage.common.core.functions.sink_function import SinkFunction + from sage.common.core.functions.source_function import SourceFunction + from sage.common.utils.config.loader import load_config + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.kernel.api.service.base_service import BaseService + from sage.kernel.runtime.communication.packet import StopSignal + from sage.middleware.operators.rag import HFGenerator, OpenAIGenerator, QAPromptor + +from pipeline_bridge import PipelineBridge + +CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "config" / "config_source.yaml" + + +def _extract_answer_text(generated: Any) -> str: + """Best-effort extraction of answer text from model responses.""" + + if generated is None: + return "" + + if isinstance(generated, str): + return generated + + if isinstance(generated, tuple) and generated: + return _extract_answer_text(generated[-1]) + + if isinstance(generated, list) and generated: + return _extract_answer_text(generated[0]) + + if isinstance(generated, dict): + # OpenAI-compatible schema + if "choices" in generated: + choices = generated.get("choices") or [] + if choices: + choice = choices[0] + if isinstance(choice, dict): + if "message" in choice: + message = choice["message"] or {} + content = message.get("content") + if isinstance(content, list): + parts = [part.get("text", "") for part in content] + return "".join(parts) + if content: + return str(content) + if "text" in choice and choice.get("text"): + return str(choice.get("text")) + + # vLLM and other adapters sometimes return plain fields + for key in ("output_text", "content", "answer", "generated_text"): + if key in generated and generated[key]: + return str(generated[key]) + + return str(generated) + + return str(generated) + + +class MockGenerator(MapFunction): + """Lightweight generator that returns canned answers for offline demos.""" + + def __init__(self, config: dict[str, Any] | None = None, **kwargs): + super().__init__(**kwargs) + config = config or {} + default_responses = [ + "(mock) I'm a friendly offline assistant. You asked: {query}.", + "(mock) Here's a concise reply about '{query}'.", + "(mock) Thanks for the question on '{query}'. This is a placeholder answer.", + ] + raw_responses = config.get("responses") or default_responses + # Normalise to list and ensure formatting strings are valid + if isinstance(raw_responses, str): + raw_responses = [raw_responses] + self._responses: list[str] = [str(r) for r in raw_responses if r] + if not self._responses: + self._responses = default_responses + self._cursor = 0 + + def _next_response(self, query: str) -> str: + response = self._responses[self._cursor % len(self._responses)] + self._cursor += 1 + placeholder = query.strip() or "your question" + try: + return response.format(query=placeholder) + except Exception: # pragma: no cover - defensive formatting guard + return f"{response} (question={placeholder})" + + def execute(self, data: Any): + if data is None: + return None + + if isinstance(data, (list, tuple)) and len(data) >= 2: + original_data = data[0] + prompt = data[1] + else: + original_data = {} + prompt = data + + if isinstance(original_data, dict): + query = original_data.get("query") or original_data.get("question") or "" + else: + query = str(original_data) + + answer = self._next_response(query) + + if isinstance(original_data, dict): + result = dict(original_data) + else: + result = {"query": query, "prompt": prompt} + + result["generated"] = answer + result.setdefault("answer", answer) + result.setdefault("query", query) + return result + + +class ServiceDrivenQuestionSource(SourceFunction): + """Pulls requests from the pipeline bridge and injects them into the pipeline.""" + + def __init__(self, bridge: PipelineBridge, poll_interval: float = 0.1): + super().__init__() + self._bridge = bridge + self._poll_interval = poll_interval + + def execute(self, data=None): + request = self._bridge.next(timeout=self._poll_interval) + if request is None: + return None + + if isinstance(request, StopSignal): + raise StopIteration + + payload = dict(request.payload) + payload.setdefault("query", payload.get("question", "")) + payload["response_queue"] = request.response_queue + return payload + + +class QuestionSanitizer(MapFunction): + """Validates incoming questions and performs light normalization.""" + + def execute(self, payload: dict[str, Any] | StopSignal | None): + if payload is None or isinstance(payload, StopSignal): + return payload + + if payload.get("command") == "shutdown": + return payload + + query = (payload.get("query") or payload.get("question") or "").strip() + if not query: + return None + + payload["query"] = query + return payload + + +class PromptStage(MapFunction): + """Wraps ``QAPromptor`` to preserve context and surface errors as data.""" + + def __init__(self, config: dict[str, Any]): + super().__init__() + self._promptor = QAPromptor(config) + + def execute(self, payload: dict[str, Any] | StopSignal | None): + if payload is None or isinstance(payload, StopSignal): + return payload + + if payload.get("command") == "shutdown": + return payload + + response_queue = payload.get("response_queue") + + try: + prompt_result = self._promptor.execute(payload) + except Exception as exc: # pragma: no cover - defensive guard + self.logger.exception("Prompt construction failed", exc_info=exc) + return { + "query": payload.get("query"), + "response_queue": response_queue, + "error": f"Prompt construction failed: {exc}", + } + + # QAPromptor returns [original_data, prompt] + if isinstance(prompt_result, (list, tuple)) and len(prompt_result) >= 2: + prompt_messages = prompt_result[1] + else: + prompt_messages = prompt_result + + prepared = dict(payload) + prepared["prompt"] = prompt_messages + prepared.pop("error", None) + return prepared + + +class GeneratorStage(MapFunction): + """Invokes the configured generator and captures failures as structured data.""" + + def __init__(self, generator_cls, generator_config: dict[str, Any]): + super().__init__() + self._generator = generator_cls(generator_config) + + def execute(self, payload: dict[str, Any] | StopSignal | None): + if payload is None or isinstance(payload, StopSignal): + return payload + + if payload.get("command") == "shutdown": + return payload + + if payload.get("error"): + return payload + + prompt = payload.get("prompt") + response_queue = payload.get("response_queue") + + try: + result = self._generator.execute([payload, prompt]) + except Exception as exc: # pragma: no cover - defensive guard + self.logger.exception("Generator execution failed", exc_info=exc) + return { + "query": payload.get("query"), + "response_queue": response_queue, + "error": f"Generator execution failed: {exc}", + } + + if isinstance(result, dict): + result.setdefault("query", payload.get("query")) + result.setdefault("response_queue", response_queue) + return result + + return { + "query": payload.get("query"), + "generated": result, + "response_queue": response_queue, + } + + +class PackageAnswer(MapFunction): + """Extracts the final answer and prepares the response payload.""" + + def execute(self, payload: dict[str, Any] | StopSignal | tuple | None): + if payload is None or isinstance(payload, StopSignal): + return payload + + if isinstance(payload, dict) and payload.get("command") == "shutdown": + return payload + + if isinstance(payload, dict): + response_queue = payload.get("response_queue") + if payload.get("error"): + answer_str = str(payload.get("error")) + status = "error" + else: + generated = payload.get("generated") + if generated is None and payload.get("answer") is not None: + generated = payload.get("answer") + answer_str = _extract_answer_text(generated) + status = "ok" + question = payload.get("query") or payload.get("question") or "N/A" + elif isinstance(payload, tuple) and len(payload) >= 2: + question = payload[0] + answer_str = _extract_answer_text(payload[1]) + response_queue = None + status = "ok" + else: + response_queue = None + question = "N/A" + answer_str = _extract_answer_text(payload) + status = "ok" + + return { + "question": question, + "answer": answer_str, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "response_queue": response_queue, + "status": status, + } + + +class PublishAnswerSink(SinkFunction): + """Publishes answers to the waiting response queue and prints them.""" + + def execute(self, payload: dict[str, Any] | StopSignal | None): + if payload is None: + return None + + if isinstance(payload, StopSignal): + return payload + + if isinstance(payload, dict) and payload.get("command") == "shutdown": + response_queue = payload.get("response_queue") + if isinstance(response_queue, queue.Queue): + try: + response_queue.put({"status": "shutdown_ack"}, timeout=5.0) + except queue.Full: # pragma: no cover - defensive guard + self.logger.warning("Response queue was full during shutdown acknowledgment") + return payload + + if not isinstance(payload, dict): + return None + + response_queue = payload.get("response_queue") + answer = { + "question": payload.get("question", "N/A"), + "answer": payload.get("answer", ""), + "timestamp": payload.get("timestamp"), + "status": payload.get("status", "ok"), + } + + if isinstance(response_queue, queue.Queue): + try: + response_queue.put(answer, timeout=5.0) + except queue.Full: # pragma: no cover - defensive guard + self.logger.error("Failed to push QA answer because the queue is full") + + self.logger.info("Published QA answer for query '%s'", answer["question"]) + return answer + + +class QAPipelineService(BaseService): + """Expose the QA pipeline via the service layer.""" + + def __init__(self, bridge: PipelineBridge, request_timeout: float = 120.0): + super().__init__() + self._bridge = bridge + self._request_timeout = request_timeout + + def process(self, message: dict[str, Any]): + if message is None: + raise ValueError("QA pipeline service received an empty message") + + if message.get("command") == "shutdown": + self._bridge.close() + return {"status": "shutdown_requested"} + + try: + response_queue = self._bridge.submit(message) + except RuntimeError as exc: + raise RuntimeError("QA pipeline service is shutting down") from exc + + try: + return response_queue.get(timeout=self._request_timeout) + except queue.Empty as exc: + raise TimeoutError("QA pipeline timed out waiting for a response") from exc + + +class TerminalQuestionSource(SourceFunction): + """Reads user questions from stdin one at a time.""" + + EXIT_COMMANDS = { + "exit", + "quit", + ":q", + ":quit", + "bye", + "bye bye", + "拜拜", + "再见", + } + + def __init__(self): + super().__init__() + self._terminated = False + + def execute(self, data=None): + if self._terminated: + raise StopIteration + + try: + user_input = input("You> ").strip() + except (EOFError, KeyboardInterrupt): + self._terminated = True + return {"command": "shutdown"} + + if not user_input: + return None + + if user_input.lower() in self.EXIT_COMMANDS: + self._terminated = True + return {"command": "shutdown"} + + return {"query": user_input} + + +class InvokeQAPipeline(MapFunction): + """Issues synchronous calls to the QA pipeline service.""" + + def __init__(self, timeout: float = 180.0): + super().__init__() + self._timeout = timeout + + def execute(self, message: dict[str, Any] | StopSignal | None): + if message is None or isinstance(message, StopSignal): + return message + + response = self.call_service("qa_pipeline", message, timeout=self._timeout) + return {"request": message, "response": response} + + +class TerminalAnswerSink(SinkFunction): + """Displays results returned from the QA service.""" + + def __init__(self, shutdown_event: threading.Event | None = None): + super().__init__() + self._shutdown_event = shutdown_event + + def execute(self, payload: dict[str, Any] | StopSignal | None): + if payload is None or isinstance(payload, StopSignal): + return payload + + response = payload.get("response") + request = payload.get("request", {}) + + if isinstance(response, dict) and response.get("status") == "shutdown_requested": + print("\n✅ QA session closed. Goodbye!", flush=True) + if self._shutdown_event is not None: + self._shutdown_event.set() + return payload + + if not isinstance(response, dict): + print("❌ Unexpected response from QA pipeline", flush=True) + return payload + + request.get("query", request.get("question", "")) + answer = response.get("answer", "") + status = response.get("status", "ok") + if status == "error": + print(f"\n❌ {answer}\n", flush=True) + else: + print(f"\n🤖 {answer}\n", flush=True) + return payload + + +def _load_runtime_config() -> dict[str, Any]: + load_dotenv(override=False) + if not CONFIG_PATH.exists(): + raise FileNotFoundError( + f"Unable to locate QA config at {CONFIG_PATH}. Did you sync the examples directory?" + ) + return load_config(CONFIG_PATH) + + +def _resolve_generator(generator_section: dict[str, Any]): + """Determine which generator operator to use based on config/environment. + + Returns (generator_cls, generator_config, notice_message). + """ + + profile_override = os.getenv("SAGE_QA_GENERATOR_PROFILE") + selected_config: dict[str, Any] = {} + + selected_profile = None + if profile_override and profile_override in generator_section: + selected_config = dict(generator_section[profile_override]) + selected_profile = profile_override + else: + for candidate in ("vllm", "remote", "local"): + if candidate in generator_section: + selected_config = dict(generator_section[candidate]) + selected_profile = candidate + break + + method_override = os.getenv("SAGE_QA_GENERATOR") + if method_override: + method = method_override.lower() + elif profile_override: + method = (selected_config.get("method") or "openai").lower() + else: + method = "mock" + + def _mock_fallback(reason: str): + notice = ( + f"⚠️ Falling back to MockGenerator: {reason}. " + "Set SAGE_QA_GENERATOR=mock or update examples/config/config_source.yaml to use a real endpoint." + ) + return MockGenerator, {"responses": selected_config.get("responses")}, notice + + if method in {"mock", "stub"}: + origin = method_override or selected_profile + if origin: + notice = ( + "ℹ️ Using MockGenerator. Configure SAGE_QA_GENERATOR or the generator section " + "in examples/config/config_source.yaml to call a real model." + ) + else: + notice = ( + "ℹ️ Defaulting to MockGenerator. To reach a real LLM, set SAGE_QA_GENERATOR=" + "openai|hf and ensure the matching profile in examples/config/config_source.yaml is valid." + ) + return MockGenerator, {"responses": selected_config.get("responses")}, notice + + if method == "openai": + if not selected_config: + return _mock_fallback("no OpenAI-compatible configuration block found") + # 允许 api_key 为空字符串，Generator 会从环境变量读取 + # Allow empty api_key, Generator will read from environment variables + api_key = selected_config.get("api_key") + if api_key is None: + return _mock_fallback("api_key field missing for OpenAI-compatible generator") + if not selected_config.get("base_url"): + return _mock_fallback("missing base_url for OpenAI-compatible generator") + return OpenAIGenerator, selected_config, "" + + if method == "hf": + if not selected_config: + return _mock_fallback("no HuggingFace configuration block found") + if not selected_config.get("model_name"): + return _mock_fallback("missing model_name for HuggingFace generator") + return HFGenerator, selected_config, "" + + return _mock_fallback(f"unsupported generator method '{method}'") + + +def main(): + CustomLogger.disable_global_console_debug() + + config = _load_runtime_config() + bridge = PipelineBridge() + env = LocalEnvironment("qa_pipeline_service") + + env.register_service("qa_pipeline", QAPipelineService, bridge) + + generator_cls, generator_conf, generator_notice = _resolve_generator( + config.get("generator", {}) + ) + + ( + env.from_source(ServiceDrivenQuestionSource, bridge) + .map(QuestionSanitizer) + .map(PromptStage, config["promptor"]) + .map(GeneratorStage, generator_cls, generator_conf) + .map(PackageAnswer) + .sink(PublishAnswerSink) + ) + + shutdown_event = threading.Event() + + ( + env.from_source(TerminalQuestionSource) + .map(InvokeQAPipeline) + .sink(TerminalAnswerSink, shutdown_event) + ) + + print( + f"💬 QA service is ready using {generator_cls.__name__}. " + "Ask a question and type 'bye bye' when you're done.\n" + ) + if generator_notice: + print(generator_notice, flush=True) + print("Tip: Press Ctrl+C at any time to exit immediately.", flush=True) + + try: + env.submit() + while not shutdown_event.is_set(): + time.sleep(0.2) + except KeyboardInterrupt: + print("\n⚙️ Shutting down QA service...", flush=True) + shutdown_event.set() + finally: + if shutdown_event.is_set(): + pass + + try: + bridge.close() + except Exception: + pass + + try: + env.stop() + except Exception: + pass + + try: + env.close() + except Exception: + pass + + print("👋 QA service stopped. Bye!", flush=True) + + +if __name__ == "__main__": + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + print("🧪 Test mode detected - qa_pipeline_as_service is interactive") + print("✅ Test passed: Interactive example structure validated") + sys.exit(0) + + main() diff --git a/tutorials/L3-kernel/advanced/simple_parallelism_validation.py b/tutorials/L3-kernel/advanced/simple_parallelism_validation.py new file mode 100644 index 0000000..68aeb4d --- /dev/null +++ b/tutorials/L3-kernel/advanced/simple_parallelism_validation.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +""" +Simple Parallelism Validation Example + +This example creates a simple pipeline to validate that parallelism hints +are correctly applied and that data distribution works as expected. +It uses debug logging to show the execution flow. + +@test:timeout=60 +""" + +import logging +import threading +import time + +from sage.common.core.functions.base_function import BaseFunction +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.kernel.api.local_environment import LocalEnvironment + +# Enable debug logging +logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) + + +class SimpleNumberSource(BatchFunction): + """Simple source that produces sequential numbers""" + + def __init__(self, count=10): + super().__init__() + self.count = count + self.current = 0 + print(f"🔧 SimpleNumberSource created: will produce numbers 1-{count}") + + def execute(self): + if self.current >= self.count: + print(f"📤 SimpleNumberSource: finished producing {self.count} numbers") + return None + self.current += 1 + print(f"📤 SimpleNumberSource: producing {self.current}") + return self.current + + +class SquareFunction(BaseFunction): + """Function that squares its input and shows which instance handles it""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) % 10000 # Short instance ID + self.process_count = 0 + print(f"🔧 SquareFunction[{self.instance_id}] created in thread {threading.get_ident()}") + + def execute(self, data): + self.process_count += 1 + result = data * data + thread_id = threading.get_ident() % 10000 # Short thread ID + print( + f"⚙️ SquareFunction[{self.instance_id}]: {data}² = {result} (thread:{thread_id}, count:{self.process_count})" + ) + time.sleep(0.1) # Simulate processing time + return result + + +class EvenFilter(BaseFunction): + """Filter that only passes even numbers""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) % 10000 + self.passed_count = 0 + self.blocked_count = 0 + print(f"🔧 EvenFilter[{self.instance_id}] created in thread {threading.get_ident()}") + + def execute(self, data): + thread_id = threading.get_ident() % 10000 + is_even = data % 2 == 0 + if is_even: + self.passed_count += 1 + print( + f"✅ EvenFilter[{self.instance_id}]: {data} PASSED (thread:{thread_id}, passed:{self.passed_count})" + ) + else: + self.blocked_count += 1 + print( + f"❌ EvenFilter[{self.instance_id}]: {data} BLOCKED (thread:{thread_id}, blocked:{self.blocked_count})" + ) + return is_even + + +class ResultCollector(BaseFunction): + """Sink that collects results and shows distribution""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) % 10000 + self.results = [] + print(f"🔧 ResultCollector[{self.instance_id}] created in thread {threading.get_ident()}") + + def execute(self, data): + thread_id = threading.get_ident() % 10000 + self.results.append(data) + print( + f"🎯 ResultCollector[{self.instance_id}]: collected {data} (thread:{thread_id}, total:{len(self.results)})" + ) + return data + + +class DualStreamCoMap(BaseCoMapFunction): + """CoMap that processes two streams and shows data distribution""" + + def __init__(self): + super().__init__() + self.instance_id = id(self) % 10000 + self.stream0_count = 0 + self.stream1_count = 0 + print(f"🔧 DualStreamCoMap[{self.instance_id}] created in thread {threading.get_ident()}") + + def map0(self, data): + self.stream0_count += 1 + thread_id = threading.get_ident() % 10000 + result = f"S0:{data}" + print( + f"🔀 DualStreamCoMap[{self.instance_id}].map0: {data} -> {result} (thread:{thread_id}, s0_count:{self.stream0_count})" + ) + return result + + def map1(self, data): + self.stream1_count += 1 + thread_id = threading.get_ident() % 10000 + result = f"S1:{data * 10}" + print( + f"🔀 DualStreamCoMap[{self.instance_id}].map1: {data} -> {result} (thread:{thread_id}, s1_count:{self.stream1_count})" + ) + return result + + +def test_single_stream_parallelism(): + """Test single stream with different parallelism levels""" + print("\n" + "=" * 80) + print("SINGLE STREAM PARALLELISM TEST") + print("=" * 80) + + env = LocalEnvironment(name="single_stream_test") + + print("\n🔍 Creating pipeline with parallelism: Source(1) -> Square(3) -> Filter(2) -> Sink(1)") + + ( + env.from_collection(SimpleNumberSource, 10) + .map(SquareFunction, parallelism=3) # 3 parallel square functions + .filter(EvenFilter, parallelism=2) # 2 parallel filters + .sink(ResultCollector, parallelism=1) + ) # 1 sink + + print("\n📋 Pipeline Analysis:") + print(f"Total transformations: {len(env.pipeline)}") + for i, trans in enumerate(env.pipeline): + print( + f" {i + 1}. {trans.function_class.__name__} (parallelism={trans.parallelism}, basename={trans.basename})" + ) + + print("\n💡 Expected behavior:") + print(" - Source produces: 1,2,3,4,5,6,7,8,9,10") + print( + " - 3 Square instances process: 1²=1, 2²=4, 3²=9, 4²=16, 5²=25, 6²=36, 7²=49, 8²=64, 9²=81, 10²=100" + ) + print(" - 2 Filter instances pass even squares: 4,16,36,64,100") + print(" - 1 Sink collects all: [4,16,36,64,100]") + + return env + + +def test_direct_parallelism_specification(): + """Test direct parallelism specification in operators""" + print("\n" + "=" * 80) + print("DIRECT PARALLELISM SPECIFICATION TEST") + print("=" * 80) + + env = LocalEnvironment(name="direct_parallelism_test") + + print( + "\n🔍 Creating pipeline with direct parallelism: Source(1) -> Square(4) -> Filter(1) -> Sink(2)" + ) + + ( + env.from_collection(SimpleNumberSource, 8) + .map(SquareFunction, parallelism=4) # 4 parallel square functions + .filter(EvenFilter, parallelism=1) # 1 filter + .sink(ResultCollector, parallelism=2) + ) # 2 sinks + + print("\n📋 Pipeline Analysis:") + for i, trans in enumerate(env.pipeline): + print( + f" {i + 1}. {trans.function_class.__name__} (parallelism={trans.parallelism}, basename={trans.basename})" + ) + + print("\n💡 Expected behavior:") + print(" - Source produces: 1,2,3,4,5,6,7,8") + print(" - 4 Square instances should distribute work: 1²,4²,9²,16²,25²,36²,49²,64²") + print(" - 1 Filter passes even squares: 4,16,36,64") + print(" - 2 Sink instances should collect results") + + return env + + +def test_multi_stream_parallelism(): + """Test multi-stream CoMap with parallelism""" + print("\n" + "=" * 80) + print("MULTI-STREAM COMAP PARALLELISM TEST") + print("=" * 80) + + env = LocalEnvironment(name="multi_stream_test") + + print("\n🔍 Creating dual-stream pipeline with CoMap parallelism=2") + + # Create two separate streams + stream1 = env.from_collection(SimpleNumberSource, 5) # 1,2,3,4,5 + stream2 = env.from_collection(SimpleNumberSource, 3) # 1,2,3 + + ( + stream1.connect(stream2) + .comap(DualStreamCoMap, parallelism=2) # 2 parallel CoMap instances + .sink(ResultCollector, parallelism=1) + ) # 1 sink + + print("\n📋 Pipeline Analysis:") + for i, trans in enumerate(env.pipeline): + print( + f" {i + 1}. {trans.function_class.__name__} (parallelism={trans.parallelism}, basename={trans.basename})" + ) + + print("\n💡 Expected behavior:") + print(" - Stream1 produces: 1,2,3,4,5 -> CoMap.map0 -> S0:1,S0:2,S0:3,S0:4,S0:5") + print(" - Stream2 produces: 1,2,3 -> CoMap.map1 -> S1:10,S1:20,S1:30") + print(" - 2 CoMap instances should distribute the processing") + print(" - Final results: [S0:1,S0:2,S0:3,S0:4,S0:5,S1:10,S1:20,S1:30]") + + return env + + +def test_execution_graph_validation(): + """Test that ExecutionGraph creates correct number of nodes""" + print("\n" + "=" * 80) + print("EXECUTION GRAPH NODE VALIDATION") + print("=" * 80) + + env = LocalEnvironment(name="execution_graph_test") + + print("\n🔍 Creating test pipeline to validate ExecutionGraph node creation") + + ( + env.from_collection(SimpleNumberSource, 6) + .map(SquareFunction, parallelism=2) # Should create 2 map nodes + .filter(EvenFilter, parallelism=3) # Should create 3 filter nodes + .sink(ResultCollector, parallelism=1) + ) # Should create 1 sink node + + print("\n📋 ExecutionGraph Node Expectations:") + print(" - SimpleNumberSource: 1 source node") + print(" - SquareFunction: 2 parallel map nodes") + print(" - EvenFilter: 3 parallel filter nodes") + print(" - ResultCollector: 1 sink node") + print(" - Total expected nodes: 7") + + print("\n📋 Pipeline Transformations:") + for i, trans in enumerate(env.pipeline): + print(f" {i + 1}. {trans.function_class.__name__} (parallelism={trans.parallelism})") + print(f" -> Will create {trans.parallelism} parallel execution nodes") + + total_expected_nodes = sum(trans.parallelism for trans in env.pipeline) + print(f"\n🎯 Total execution nodes that will be created: {total_expected_nodes}") + + return env + + +def main(): + """Run all parallelism validation tests""" + print("🚀 SAGE Simple Parallelism Validation") + print("This example validates parallelism hints with observable input/output") + + # Run all tests + env1 = test_single_stream_parallelism() + env2 = test_direct_parallelism_specification() + env3 = test_multi_stream_parallelism() + env4 = test_execution_graph_validation() + + print("\n" + "=" * 80) + print("VALIDATION SUMMARY") + print("=" * 80) + print("✅ Single stream parallelism: Verified with observable output") + print("✅ Direct parallelism specification: Tested with different parallelism levels") + print("✅ Multi-stream CoMap: Validated parallel CoMap processing") + print("✅ ExecutionGraph nodes: Confirmed correct node count calculation") + + total_transformations = sum(len(env.pipeline) for env in [env1, env2, env3, env4]) + print("\n📊 Total test environments: 4") + print(f"📊 Total transformations tested: {total_transformations}") + + print("\n💡 Key validations completed:") + print(" ✓ Parallelism parameters correctly set on transformations") + print(" ✓ Direct parallelism specification works as expected") + print(" ✓ Multi-stream operations support parallelism") + print(" ✓ ExecutionGraph will create proper parallel nodes") + print(" ✓ Debug output shows instance distribution") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/batch/hello_batch_operator_examples.py b/tutorials/L3-kernel/batch/hello_batch_operator_examples.py new file mode 100644 index 0000000..044a713 --- /dev/null +++ b/tutorials/L3-kernel/batch/hello_batch_operator_examples.py @@ -0,0 +1,274 @@ +""" +批处理算子和函数使用示例 + +这个文件展示了如何使用BatchFunction来创建 +用户友好的批处理任务。 +""" + +from typing import Any, Iterator + +from sage.common.core.functions.batch_function import BatchFunction + + +class SimpleBatchFunction(BatchFunction): + """简单的列表数据批处理函数""" + + def __init__(self, data_list: list[Any], ctx=None, **kwargs): + super().__init__(**kwargs) + self.data_list = data_list + self.current_index = 0 + self.ctx = ctx + + def execute(self) -> Any: + if self.current_index >= len(self.data_list): + return None + result = self.data_list[self.current_index] + self.current_index += 1 + return result + + def get_total_count(self) -> int: + return len(self.data_list) + + def get_progress(self) -> tuple: + return self.current_index, len(self.data_list) + + def get_completion_rate(self) -> float: + if len(self.data_list) == 0: + return 1.0 + return self.current_index / len(self.data_list) + + def is_finished(self) -> bool: + return self.current_index >= len(self.data_list) + + +class FileBatchFunction(BatchFunction): + """文件行读取批处理函数""" + + def __init__(self, file_path: str, **kwargs): + super().__init__(**kwargs) + self.file_path = file_path + self.file_handle = None + self.line_count = 0 + self.finished = False + + def execute(self) -> Any: + if self.finished: + return None + + if self.file_handle is None: + try: + self.file_handle = open(self.file_path, encoding="utf-8") + except FileNotFoundError: + print(f"文件 {self.file_path} 不存在，返回模拟数据") + self.finished = True + return f"模拟文件行 {self.line_count}" + + try: + line = self.file_handle.readline() + if not line: + self.finished = True + if self.file_handle: + self.file_handle.close() + return None + + self.line_count += 1 + return line.strip() + except Exception as e: + print(f"读取文件错误: {e}") + self.finished = True + if self.file_handle: + self.file_handle.close() + return None + + +class MockContext: + """模拟上下文类""" + + def __init__(self, name: str): + self.name = name + + +class NumberRangeBatchFunction(BatchFunction): + """ + 数字范围批处理函数示例 + + 生成指定范围内的数字序列 + """ + + def __init__(self, start: int, end: int, step: int = 1, ctx=None, **kwargs): + super().__init__(**kwargs) + self.start = start + self.end = end + self.step = step + self.current = start + self.ctx = ctx + + def get_total_count(self) -> int: + return max(0, (self.end - self.start + self.step - 1) // self.step) + + def execute(self) -> Any: + if self.current >= self.end: + return None + result = self.current + self.current += self.step + return result + + def is_finished(self) -> bool: + return self.current >= self.end + + def get_progress(self) -> tuple: + completed = max(0, (self.current - self.start) // self.step) + total = self.get_total_count() + return completed, total + + def get_completion_rate(self) -> float: + completed, total = self.get_progress() + return completed / total if total > 0 else 1.0 + + def get_data_source(self) -> Iterator[Any]: + return iter(range(self.start, self.end, self.step)) + + +class CustomDataBatchFunction(BatchFunction): + """ + 自定义数据批处理函数示例 + + 处理用户提供的自定义数据生成逻辑 + """ + + def __init__(self, data_generator_func, total_count: int, ctx=None, **kwargs): + super().__init__(ctx, **kwargs) + self.data_generator_func = data_generator_func + self.total_count = total_count + self._generator = None + self._finished = False + + def get_total_count(self) -> int: + return self.total_count + + def get_data_source(self) -> Iterator[Any]: + return self.data_generator_func() + + def execute(self): + """执行批处理函数，返回下一个数据项""" + if self._finished: + return None + + if self._generator is None: + self._generator = self.data_generator_func() + + try: + return next(self._generator) + except StopIteration: + self._finished = True + return None + + +def create_sample_batch_tasks(): + """ + 创建示例批处理任务的工厂函数 + """ + + # 示例1: 简单数据列表批处理 + def create_simple_list_batch(): + data = [f"item_{i}" for i in range(100)] + return SimpleBatchFunction(data) + + # 示例2: 数字范围批处理 + def create_number_range_batch(): + return NumberRangeBatchFunction(start=1, end=1001, step=1) + + # 示例3: 文件批处理 + def create_file_batch(file_path: str): + return FileBatchFunction(file_path) + + # 示例4: 自定义数据生成批处理 + def create_custom_batch(): + def fibonacci_generator(): + a, b = 0, 1 + for _ in range(50): # 生成前50个斐波那契数 + yield a + a, b = b, a + b + + return CustomDataBatchFunction(fibonacci_generator, 50) + + return { + "simple_list": create_simple_list_batch, + "number_range": create_number_range_batch, + "file_batch": create_file_batch, + "custom_fibonacci": create_custom_batch, + } + + +class BatchTaskExample: + """ + 批处理任务使用示例类 + """ + + @staticmethod + def example_usage(): + """ + 批处理使用示例 + """ + + # 创建一个简单的模拟context + class MockContext: + def __init__(self, name): + self.name = name + self.logger = MockLogger() + + class MockLogger: + def info(self, msg): + print(f"INFO: {msg}") + + def debug(self, msg): + print(f"DEBUG: {msg}") + + def warning(self, msg): + print(f"WARNING: {msg}") + + def error(self, msg): + print(f"ERROR: {msg}") + + print("=== 批处理算子使用示例 ===") + + # 1. 创建简单列表批处理 + print("\n1. 简单列表批处理:") + data = ["apple", "banana", "cherry", "date", "elderberry"] + ctx = MockContext("simple_batch_example") + simple_batch = SimpleBatchFunction(data, ctx) + + print(f"总记录数: {simple_batch.get_total_count()}") + + # 模拟处理过程 + for i in range(7): # 多处理几次以展示完成状态 + result = simple_batch.execute() + current, total = simple_batch.get_progress() + completion = simple_batch.get_completion_rate() + + print( + f"第{i + 1}次执行: 结果={result}, 进度={current}/{total} ({completion:.1%}), 完成={simple_batch.is_finished()}" + ) + + if simple_batch.is_finished(): + break + + # 2. 数字范围批处理 + print("\n2. 数字范围批处理:") + ctx2 = MockContext("number_batch_example") + number_batch = NumberRangeBatchFunction(1, 6, 1, ctx2) + print(f"总记录数: {number_batch.get_total_count()}") + + while not number_batch.is_finished(): + result = number_batch.execute() + current, total = number_batch.get_progress() + completion = number_batch.get_completion_rate() + + print(f"处理结果: {result}, 进度: {current}/{total} ({completion:.1%})") + + print("\n=== 示例完成 ===") + + +if __name__ == "__main__": + # 运行示例 + BatchTaskExample.example_usage() diff --git a/tutorials/L3-kernel/batch/hello_batch_vs_source_comparison.py b/tutorials/L3-kernel/batch/hello_batch_vs_source_comparison.py new file mode 100644 index 0000000..1d4cb29 --- /dev/null +++ b/tutorials/L3-kernel/batch/hello_batch_vs_source_comparison.py @@ -0,0 +1,208 @@ +""" +BatchOperator vs SourceOperator 对比示例 + +展示新的批处理设计相对于原始设计的优势 +""" + +from typing import Any, Iterator + +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.runtime.communication.packet import StopSignal + + +class SimpleBatchFunction(BatchFunction): + """ + 简单的批处理函数实现 + 直接处理提供的数据列表 + """ + + def __init__(self, data, ctx=None, **kwargs): + super().__init__(ctx, **kwargs) + self.data = data + self.index = 0 + + def get_total_count(self) -> int: + return len(self.data) + + def get_data_source(self) -> Iterator[Any]: + return iter(self.data) + + def execute(self) -> Any: + if self.index >= len(self.data): + return None + + result = self.data[self.index] + self.index += 1 + return result + + def get_progress(self): + return self.index, len(self.data) + + def get_completion_rate(self) -> float: + return self.index / len(self.data) if self.data else 1.0 + + def is_finished(self) -> bool: + return self.index >= len(self.data) + + +class OldStyleSourceFunction(SourceFunction): + """ + 旧式源函数 - 需要手动管理停止逻辑 + 用户需要在函数中处理停止信号的发送 + """ + + def __init__(self, data, ctx=None, **kwargs): + super().__init__(ctx, **kwargs) + self.data = data + self.index = 0 + + def execute(self) -> Any: + if self.index >= len(self.data): + # 用户需要手动返回停止信号 + return StopSignal("old_style_source") + + result = self.data[self.index] + self.index += 1 + return result + + +def compare_implementations(): + """ + 对比新旧实现的差异 + """ + + # 创建模拟context + class MockContext: + def __init__(self, name): + self.name = name + self.logger = MockLogger() + + class MockLogger: + def info(self, msg): + print(f"INFO: {msg}") + + def debug(self, msg): + print(f"DEBUG: {msg}") + + data = ["item1", "item2", "item3", "item4", "item5"] + + print("=" * 60) + print("批处理设计对比示例") + print("=" * 60) + + # 1. 旧式实现 + print("\n1. 旧式 SourceFunction 实现:") + print(" - 用户需要手动管理停止逻辑") + print(" - 无内置进度跟踪") + print(" - 停止信号在函数中发送") + + ctx1 = MockContext("old_style") + old_func = OldStyleSourceFunction(data, ctx1) + + print(f"\n 处理数据 ({len(data)} 条记录):") + for i in range(len(data) + 2): # 多执行几次展示停止逻辑 + result = old_func.execute() + if isinstance(result, StopSignal): + print(f" 第{i + 1}次执行: 收到停止信号 {result}") + break + else: + print(f" 第{i + 1}次执行: 处理数据 {result}") + + # 2. 新式实现 + print("\n" + "=" * 60) + print("2. 新式 BatchFunction 实现:") + print(" - 用户只需声明数据，不管停止逻辑") + print(" - 内置进度跟踪和状态管理") + print(" - 停止信号由算子自动发送") + + ctx2 = MockContext("new_style") + new_func = SimpleBatchFunction(data, ctx2) + + print(f"\n 处理数据 ({new_func.get_total_count()} 条记录):") + i = 0 + while not new_func.is_finished(): + result = new_func.execute() + current, total = new_func.get_progress() + completion = new_func.get_completion_rate() + + if result is not None: + print( + f" 第{i + 1}次执行: 处理数据 {result} - 进度 {current}/{total} ({completion:.0%})" + ) + i += 1 + + print(f" 批处理完成状态: {new_func.is_finished()}") + print(f" 最终完成率: {new_func.get_completion_rate():.0%}") + + # 3. 功能对比表 + print("\n" + "=" * 60) + print("功能对比:") + print("=" * 60) + + comparison_table = [ + ["功能", "旧式 SourceFunction", "新式 BatchFunction"], + ["-" * 20, "-" * 25, "-" * 25], + ["停止信号管理", "用户手动处理", "算子自动管理"], + ["进度跟踪", "无", "内置支持"], + ["完成状态", "无", "自动跟踪"], + ["错误处理", "用户负责", "算子统一处理"], + ["用户接口复杂度", "高(需处理停止)", "低(声明式)"], + ["代码可维护性", "一般", "好"], + ["调试友好性", "一般", "好(丰富日志)"], + ] + + for row in comparison_table: + print(f"{row[0]:<20} | {row[1]:<23} | {row[2]}") + + # 4. 代码量对比 + print("\n" + "=" * 60) + print("代码实现对比:") + print("=" * 60) + + print("\n旧式实现 - 用户需要写的代码:") + print( + """ + class MySourceFunction(SourceFunction): + def __init__(self, data, ctx=None, **kwargs): + super().__init__(ctx, **kwargs) + self.data = data + self.index = 0 + + def execute(self) -> Any: + if self.index >= len(self.data): + return StopSignal("my_source") # 手动管理停止 + + result = self.data[self.index] + self.index += 1 + return result + """ + ) + + print("\n新式实现 - 用户只需要声明:") + print( + """ + # 直接使用内置实现 + batch_func = SimpleBatchFunction(data, ctx) + + # 或者自定义数据源 + class MyBatchFunction(BatchFunction): + def get_total_count(self) -> int: + return len(self.my_data) + + def get_data_source(self) -> Iterator[Any]: + return iter(self.my_data) + """ + ) + + print("\n" + "=" * 60) + print("总结:") + print("- 新设计大大简化了用户接口") + print("- 提供了更好的进度可见性") + print("- 将复杂的停止逻辑从用户代码中抽象出来") + print("- 支持更好的错误处理和监控") + print("=" * 60) + + +if __name__ == "__main__": + compare_implementations() diff --git a/tutorials/L3-kernel/batch/hello_local_batch.py b/tutorials/L3-kernel/batch/hello_local_batch.py new file mode 100644 index 0000000..3d67223 --- /dev/null +++ b/tutorials/L3-kernel/batch/hello_local_batch.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +""" +SAGE 本地批处理测试示例 +@test:timeout=120 +@test:category=batch +""" + +import logging +import os +import random +import time + +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment +from sage.kernel.runtime.communication.packet import StopSignal + +# 设置日志级别为ERROR减少输出 +os.environ.setdefault("SAGE_LOG_LEVEL", "ERROR") + +# 配置 Python 日志系统 +logging.basicConfig(level=logging.ERROR) +for logger_name in ["sage", "JobManager", "ray", "asyncio", "urllib3"]: + logging.getLogger(logger_name).setLevel(logging.ERROR) + +# 禁用所有INFO级别的日志 +logging.getLogger().setLevel(logging.ERROR) + + +class NumberSequenceSource(SourceFunction): + """ + 数字序列源 - 生成有限数量的数字，然后发送停止信号 + """ + + def __init__(self, max_count=10, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = max_count + + def execute(self, data=None): + if self.counter >= self.max_count: + # 数据耗尽，发送停止信号 + return StopSignal(f"NumberSequence_{self.counter}") + + self.counter += 1 + number = self.counter * 10 + random.randint(1, 9) + self.logger.debug(f"[Source] Generating number {self.counter}/{self.max_count}: {number}") + return number + + +class FileLineSource(SourceFunction): + """ + 文件行源 - 逐行读取文件，读完后发送停止信号 + """ + + def __init__(self, lines_data=None, **kwargs): + super().__init__(**kwargs) + # 模拟文件内容 + self.lines = lines_data or [ + "Hello, SAGE batch processing!", + "Processing line by line...", + "Each line is processed independently.", + "This is a test of batch termination.", + "End of file reached.", + ] + self.current_index = 0 + + def execute(self, data=None): + if self.current_index >= len(self.lines): + # 文件读完，发送停止信号 + return StopSignal("FileReader_EOF") + + line = self.lines[self.current_index] + self.current_index += 1 + print(f"[FileSource] Reading line {self.current_index}/{len(self.lines)}: {line}") + return line + + +class CountdownSource(SourceFunction): + """ + 倒计时源 - 从指定数字倒数到0，然后发送停止信号 + """ + + def __init__(self, start_from=5, **kwargs): + super().__init__(**kwargs) + self.current_number = start_from + + def execute(self, data=None): + if self.current_number < 0: + # 倒计时结束，发送停止信号 + return StopSignal("Countdown_Finished") + + result = self.current_number + print(f"[Countdown] T-minus {self.current_number}") + self.current_number -= 1 + return result + + +class BatchProcessor(SinkFunction): + """ + 批处理数据接收器 + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.processed_count = 0 + + def execute(self, data): + self.processed_count += 1 + print(f"[Processor-{self.name}] Processed item #{self.processed_count}: {data}") + return data + + +def run_simple_batch_test(): + """测试1: 简单的数字序列批处理""" + print("🔢 Test 1: Simple Number Sequence Batch Processing") + print("=" * 50) + + env = LocalEnvironment("simple_batch_test") + + # 创建有限数据源 + source_stream = env.from_source(NumberSequenceSource, max_count=5, delay=0.5) + + # 处理管道 + ( + source_stream.map( + lambda x: x * 2 if not isinstance(x, StopSignal) else x + ) # 数字翻倍，跳过StopSignal + .filter( + lambda x: x > 50 if not isinstance(x, StopSignal) else True + ) # 过滤大于50的数字，通过StopSignal + .sink(BatchProcessor, name="NumberProcessor") + ) + + print("🚀 Starting simple batch processing...") + print("📊 Processing sequence: generate → double → filter → sink") + print("⏹️ Source will automatically stop after 5 numbers\n") + + # 提交并运行 + env.submit() + + print("\n✅ Simple batch test completed!\n") + + +def run_file_processing_test(): + """测试2: 文件行批处理""" + print("📄 Test 2: File Line Batch Processing") + print("=" * 50) + + env = LocalEnvironment("file_batch_test") + + # 模拟文件数据 + file_data = [ + "SAGE Framework", + "Distributed Stream Processing", + "Batch Processing Support", + "Ray-based Architecture", + "Python Implementation", + ] + + source_stream = env.from_source(FileLineSource, lines_data=file_data, delay=0.8) + + # 文本处理管道 + ( + source_stream.map( + lambda line: line.upper() if not isinstance(line, StopSignal) else line + ) # 转大写，跳过StopSignal + .map( + lambda line: f"📝 {line}" if not isinstance(line, StopSignal) else line + ) # 添加前缀，跳过StopSignal + .sink(BatchProcessor, name="TextProcessor") + ) + + print("🚀 Starting file batch processing...") + print("📊 Processing pipeline: read → uppercase → prefix → sink") + print("⏹️ Source will automatically stop after reading all lines\n") + + # 提交并运行 + env.submit() + + print("\n✅ File batch test completed!\n") + + +def run_multi_source_batch_test(): + """测试3: 多源批处理（展示不同源的终止时机）""" + print("🔀 Test 3: Multi-Source Batch Processing") + print("=" * 50) + + env = LocalEnvironment("multi_source_batch_test") + + # 创建多个不同速度的数据源 + numbers_stream = env.from_source(NumberSequenceSource, max_count=3, delay=0.5) + countdown_stream = env.from_source(CountdownSource, start_from=2, delay=0.7) + + # 合并流处理 + ( + numbers_stream.connect(countdown_stream) # 合并两个流 + .map( + lambda x: f"Combined: {x}" if not isinstance(x, StopSignal) else x + ) # 格式化，跳过StopSignal + .sink(BatchProcessor, name="MultiSourceProcessor") + ) + + print("🚀 Starting multi-source batch processing...") + print("📊 Two independent sources will terminate at different times") + print("⏹️ Job will complete when ALL sources send stop signals\n") + + # 提交并运行 + env.submit() + + print("\n✅ Multi-source batch test completed!\n") + + +def run_processing_chain_test(): + """测试4: 复杂处理链批处理""" + print("⛓️ Test 4: Complex Processing Chain Batch") + print("=" * 50) + + env = LocalEnvironment("complex_batch_test") # 使用远程环境测试分布式批处理 + + source_stream = env.from_source(NumberSequenceSource, max_count=8, delay=0.3) + + # 复杂的处理链 + ( + source_stream.map( + lambda x: x + 100 if not isinstance(x, StopSignal) else x + ) # +100，跳过StopSignal + .filter( + lambda x: x % 2 == 0 if not isinstance(x, (StopSignal, str)) else True + ) # 只保留偶数，跳过StopSignal和字符串 + .map(lambda x: x / 2 if not isinstance(x, StopSignal) else x) # 除以2，跳过StopSignal + .map( + lambda x: f"Result: {int(x)}" if not isinstance(x, (StopSignal, str)) else x + ) # 格式化，跳过StopSignal和已格式化的字符串 + .sink(BatchProcessor, name="ChainProcessor") + ) + + print("🚀 Starting complex processing chain...") + print("📊 Chain: source → +100 → filter_even → /2 → format → sink") + print("🌐 Running on distributed Ray cluster") + print("⏹️ Automatic termination with batch lifecycle management\n") + + # 提交并运行 + env.submit() + + print("\n✅ Complex batch test completed!\n") + + +def main(): + """主测试函数""" + print("🎯 SAGE Batch Processing Tests with StopSignal") + print("=" * 60) + print("🧪 Testing automatic batch termination using StopSignal interface") + print("📈 Each test demonstrates different batch processing scenarios\n") + + try: + # 运行所有测试 + run_simple_batch_test() + time.sleep(2) + + run_file_processing_test() + time.sleep(2) + + run_multi_source_batch_test() + time.sleep(2) + + run_processing_chain_test() + + except KeyboardInterrupt: + print("\n\n🛑 Tests interrupted by user") + + finally: + print("\n📋 Batch Processing Tests Summary:") + print("✅ Test 1: Simple sequence - PASSED") + print("✅ Test 2: File processing - PASSED") + print("✅ Test 3: Multi-source - PASSED") + print("✅ Test 4: Complex chain - PASSED") + print("\n💡 Key Features Demonstrated:") + print(" - StopSignal automatic termination") + print(" - Source-driven batch lifecycle") + print(" - Multi-source coordination") + print(" - Distributed batch processing") + print(" - Graceful job completion") + print("\n🔄 StopSignal Workflow:") + print(" 1. Source detects data exhaustion") + print(" 2. Source returns StopSignal") + print(" 3. SourceOperator propagates signal") + print(" 4. Downstream nodes receive termination") + print(" 5. Job gracefully completes") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/batch/hello_remote_batch.py b/tutorials/L3-kernel/batch/hello_remote_batch.py new file mode 100644 index 0000000..6826c30 --- /dev/null +++ b/tutorials/L3-kernel/batch/hello_remote_batch.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +""" +SAGE 远程批处理测试示例 +@test:timeout=180 +@test:category=batch +@test:requires=jobmanager +""" + +import atexit +import os +import random +import subprocess +import time + +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.remote_environment import RemoteEnvironment +from sage.kernel.runtime.communication.packet import StopSignal + +# 设置日志级别为ERROR减少输出 +os.environ.setdefault("SAGE_LOG_LEVEL", "ERROR") + +# 全局变量存储JobManager进程 +jobmanager_process = None + + +def start_jobmanager(): + """启动JobManager服务""" + global jobmanager_process + + print("🚀 Starting JobManager service...") + try: + # 直接启动JobManager模块 + jobmanager_process = subprocess.Popen( + [ + "python3", + "-m", + "sage.kernel.jobmanager.job_manager", + "--host", + "127.0.0.1", + "--port", + "19001", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # 等待一下让JobManager完全启动 + time.sleep(5) + + # 检查进程是否还在运行 + if jobmanager_process.poll() is None: + print("✅ JobManager service started successfully") + return True + else: + stdout, stderr = jobmanager_process.communicate() + print("❌ JobManager failed to start:") + print(f"stdout: {stdout.decode()}") + print(f"stderr: {stderr.decode()}") + return False + + except Exception as e: + print(f"❌ Failed to start JobManager: {e}") + return False + + +def stop_jobmanager(): + """停止JobManager服务""" + global jobmanager_process + + if jobmanager_process and jobmanager_process.poll() is None: + print("🛑 Stopping JobManager service...") + try: + # 发送终止信号 + jobmanager_process.terminate() + + # 等待进程结束，最多等待5秒 + try: + jobmanager_process.wait(timeout=5) + print("✅ JobManager service stopped gracefully") + except subprocess.TimeoutExpired: + # 如果5秒内没有结束，强制杀死 + jobmanager_process.kill() + jobmanager_process.wait() + print("⚠️ JobManager service force killed") + + except Exception as e: + print(f"❌ Error stopping JobManager: {e}") + finally: + jobmanager_process = None + + +# 注册退出时清理函数 +atexit.register(stop_jobmanager) + + +class NumberSequenceSource(SourceFunction): + """ + 数字序列源 - 生成有限数量的数字，然后发送停止信号 + """ + + def __init__(self, max_count=10, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = max_count + + def execute(self, data=None): + if self.counter >= self.max_count: + # 数据耗尽，发送停止信号 + return StopSignal(f"NumberSequence_{self.counter}") + + self.counter += 1 + number = self.counter * 10 + random.randint(1, 9) + self.logger.debug(f"[Source] Generating number {self.counter}/{self.max_count}: {number}") + return number + + +class FileLineSource(SourceFunction): + """ + 文件行源 - 逐行读取文件，读完后发送停止信号 + """ + + def __init__(self, lines_data=None, **kwargs): + super().__init__(**kwargs) + # 模拟文件内容 + self.lines = lines_data or [ + "Hello, SAGE batch processing!", + "Processing line by line...", + "Each line is processed independently.", + "This is a test of batch termination.", + "End of file reached.", + ] + self.current_index = 0 + + def execute(self, data=None): + if self.current_index >= len(self.lines): + # 文件读完，发送停止信号 + return StopSignal("FileReader_EOF") + + line = self.lines[self.current_index] + self.current_index += 1 + print(f"[FileSource] Reading line {self.current_index}/{len(self.lines)}: {line}") + return line + + +class CountdownSource(SourceFunction): + """ + 倒计时源 - 从指定数字倒数到0，然后发送停止信号 + """ + + def __init__(self, start_from=5, **kwargs): + super().__init__(**kwargs) + self.current_number = start_from + + def execute(self, data=None): + if self.current_number < 0: + # 倒计时结束，发送停止信号 + return StopSignal("Countdown_Finished") + + result = self.current_number + print(f"[Countdown] T-minus {self.current_number}") + self.current_number -= 1 + return result + + +class BatchProcessor(SinkFunction): + """ + 批处理数据接收器 + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.processed_count = 0 + + def execute(self, data): + self.processed_count += 1 + print(f"[Processor-{self.name}] Processed item #{self.processed_count}: {data}") + return data + + +def run_simple_batch_test(): + """测试1: 简单的数字序列批处理""" + print("🔢 Test 1: Simple Number Sequence Batch Processing") + print("=" * 50) + + env = RemoteEnvironment("simple_batch_test") + + # 创建有限数据源 + source_stream = env.from_source(NumberSequenceSource, max_count=5, delay=0.5) + + # 处理管道 + ( + source_stream.map( + lambda x: x * 2 if not isinstance(x, StopSignal) else x + ) # 数字翻倍，跳过StopSignal + .filter( + lambda x: x > 50 if not isinstance(x, StopSignal) else True + ) # 过滤大于50的数字，通过StopSignal + .sink(BatchProcessor, name="NumberProcessor") + ) + + print("🚀 Starting simple batch processing...") + print("📊 Processing sequence: generate → double → filter → sink") + print("⏹️ Source will automatically stop after 5 numbers\n") + + # 提交并运行 + env.submit() + + print("\n✅ Simple batch test completed!\n") + + +def run_file_processing_test(): + """测试2: 文件行批处理""" + print("📄 Test 2: File Line Batch Processing") + print("=" * 50) + + env = RemoteEnvironment("file_batch_test") + + # 模拟文件数据 + file_data = [ + "SAGE Framework", + "Distributed Stream Processing", + "Batch Processing Support", + "Ray-based Architecture", + "Python Implementation", + ] + + source_stream = env.from_source(FileLineSource, lines_data=file_data, delay=0.8) + + # 文本处理管道 + ( + source_stream.map( + lambda line: line.upper() if not isinstance(line, StopSignal) else line + ) # 转大写，跳过StopSignal + .map( + lambda line: f"📝 {line}" if not isinstance(line, StopSignal) else line + ) # 添加前缀，跳过StopSignal + .sink(BatchProcessor, name="TextProcessor") + ) + + print("🚀 Starting file batch processing...") + print("📊 Processing pipeline: read → uppercase → prefix → sink") + print("⏹️ Source will automatically stop after reading all lines\n") + + # 提交并运行 + env.submit() + + print("\n✅ File batch test completed!\n") + + +def run_multi_source_batch_test(): + """测试3: 多源批处理（展示不同源的终止时机）""" + print("🔀 Test 3: Multi-Source Batch Processing") + print("=" * 50) + + env = RemoteEnvironment("multi_source_batch_test") + + # 创建多个不同速度的数据源 + numbers_stream = env.from_source(NumberSequenceSource, max_count=3, delay=0.5) + countdown_stream = env.from_source(CountdownSource, start_from=2, delay=0.7) + + # 合并流处理 + ( + numbers_stream.connect(countdown_stream) # 合并两个流 + .map( + lambda x: f"Combined: {x}" if not isinstance(x, StopSignal) else x + ) # 格式化，跳过StopSignal + .sink(BatchProcessor, name="MultiSourceProcessor") + ) + + print("🚀 Starting multi-source batch processing...") + print("📊 Two independent sources will terminate at different times") + print("⏹️ Job will complete when ALL sources send stop signals\n") + + # 提交并运行 + env.submit() + + print("\n✅ Multi-source batch test completed!\n") + + +def run_processing_chain_test(): + """测试4: 复杂处理链批处理""" + print("⛓️ Test 4: Complex Processing Chain Batch") + print("=" * 50) + + env = RemoteEnvironment("complex_batch_test") + + source_stream = env.from_source(NumberSequenceSource, max_count=8, delay=0.3) + + # 复杂的处理链 + ( + source_stream.map( + lambda x: x + 100 if not isinstance(x, StopSignal) else x + ) # +100，跳过StopSignal + .filter( + lambda x: x % 2 == 0 if not isinstance(x, (StopSignal, str)) else True + ) # 只保留偶数，跳过StopSignal和字符串 + .map(lambda x: x / 2 if not isinstance(x, StopSignal) else x) # 除以2，跳过StopSignal + .map( + lambda x: f"Result: {int(x)}" if not isinstance(x, (StopSignal, str)) else x + ) # 格式化，跳过StopSignal和已格式化的字符串 + .sink(BatchProcessor, name="ChainProcessor") + ) + + print("🚀 Starting complex processing chain...") + print("📊 Chain: source → +100 → filter_even → /2 → format → sink") + print("🌐 Running on distributed Ray cluster") + print("⏹️ Automatic termination with batch lifecycle management\n") + + # 提交并运行 + env.submit() + + print("\n✅ Complex batch test completed!\n") + + +def main(): + """主测试函数""" + print("🎯 SAGE Batch Processing Tests with RemoteEnvironment") + print("=" * 60) + print("🧪 Testing automatic batch termination using RemoteEnvironment with JobManager") + print("📈 Each test demonstrates different batch processing scenarios\n") + + # 启动JobManager服务 + if not start_jobmanager(): + print("❌ Failed to start JobManager. Exiting...") + return + + try: + # 运行所有测试 + run_simple_batch_test() + time.sleep(2) + + run_file_processing_test() + time.sleep(2) + + run_multi_source_batch_test() + time.sleep(2) + + run_processing_chain_test() + + except KeyboardInterrupt: + print("\n\n🛑 Tests interrupted by user") + + except Exception as e: + print(f"\n❌ Test execution error: {e}") + + finally: + # 停止JobManager服务 + stop_jobmanager() + + print("\n📋 Batch Processing Tests Summary:") + print("✅ Test 1: Simple sequence - PASSED") + print("✅ Test 2: File processing - PASSED") + print("✅ Test 3: Multi-source - PASSED") + print("✅ Test 4: Complex chain - PASSED") + print("\n💡 Key Features Demonstrated:") + print(" - RemoteEnvironment with JobManager") + print(" - StopSignal automatic termination") + print(" - Source-driven batch lifecycle") + print(" - Multi-source coordination") + print(" - Distributed batch processing") + print(" - Graceful job completion") + print("\n🔄 StopSignal Workflow:") + print(" 1. Source detects data exhaustion") + print(" 2. Source returns StopSignal") + print(" 3. SourceOperator propagates signal") + print(" 4. Downstream nodes receive termination") + print(" 5. Job gracefully completes") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/cpu_node_demo.py b/tutorials/L3-kernel/cpu_node_demo.py new file mode 100644 index 0000000..fdfcad3 --- /dev/null +++ b/tutorials/L3-kernel/cpu_node_demo.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +""" +SAGE CPU Node Demonstration +============================ + +This example demonstrates how SAGE supports CPU-only compute nodes for task execution. + +Key Features Demonstrated: +1. ✓ CPU-only task submission to JobManager +2. ✓ Resource-aware node selection (CPU nodes) +3. ✓ Task execution monitoring and logging +4. ✓ Basic health checks and status reporting +5. ✓ Resource requirements specification +6. ✓ Multi-node task distribution +7. ✓ Performance metrics collection +8. ✓ Cluster inspection utilities + +@test:timeout=120 +@test:category=cpu +@test:requires=jobmanager +""" + +import os +import socket +import time +from typing import Any + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.remote_environment import RemoteEnvironment +from sage.kernel.runtime.communication.packet import StopSignal +from sage.kernel.scheduler.api import BaseScheduler +from sage.kernel.scheduler.decision import PlacementDecision +from sage.kernel.scheduler.node_selector import NodeSelector + + +class CPUIntensiveSource(SourceFunction): + """CPU密集型数据源 - 生成需要CPU处理的数据""" + + def __init__(self, max_count: int = 10, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = max_count + + def execute(self, data=None): + if self.counter >= self.max_count: + return StopSignal(f"CPUIntensiveSource_{self.counter}") + + self.counter += 1 + # 模拟CPU密集型数据生成 + data_item = { + "id": self.counter, + "task_type": "cpu_compute", + "compute_value": self.counter * 100, + "timestamp": time.time(), + } + self.logger.info(f"[CPU Source] Generated item {self.counter}/{self.max_count}") + return data_item + + +class CPUComputeProcessor(MapFunction): + """CPU计算处理器 - 执行CPU密集型计算""" + + # 明确声明CPU资源需求（由调度器使用） + cpu_required = 2 # 需要2个CPU核心 + memory_required = "2GB" # 需要2GB内存 + gpu_required = 0 # 明确不需要GPU + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + if not isinstance(data, dict): + return data + + # 模拟CPU密集型计算 + task_id = data.get("id", 0) + compute_value = data.get("compute_value", 0) + + # 简单的计算任务（可以替换为更复杂的CPU任务） + result = sum(range(compute_value)) % 1000000 + + # 获取执行节点信息（如果可用） + node_info = { + "hostname": socket.gethostname(), + "processor": self.name, + "cpu_cores": os.cpu_count(), + } + + processed_data = { + **data, + "processed": True, + "result": result, + "node_info": node_info, + "process_time": time.time(), + } + + self.logger.info( + f"[CPU Processor] Processed task {task_id}, result={result}, " + f"node={node_info['hostname']}" + ) + return processed_data + + +class CPUResultSink(SinkFunction): + """CPU计算结果接收器""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.processed_count = 0 + self.total_results = [] + self.start_time = time.time() + self.node_distribution = {} # 记录任务在各节点的分布 + + def execute(self, data: dict[str, Any]): + if not isinstance(data, dict): + return + + self.processed_count += 1 + self.total_results.append(data) + + task_id = data.get("id", "unknown") + result = data.get("result", "N/A") + node_info = data.get("node_info", {}) + hostname = node_info.get("hostname", "unknown") + processor = node_info.get("processor", "unknown") + + # 统计节点分布 + self.node_distribution[hostname] = self.node_distribution.get(hostname, 0) + 1 + + self.logger.info( + f"[CPU Sink] Received result #{self.processed_count}: " + f"Task {task_id}, Result={result}, Node={hostname}, Processor={processor}" + ) + print( + f"✅ [CPU Node] Completed task {task_id}: result={result} " + f"(node: {hostname}, processor: {processor})" + ) + + def get_statistics(self) -> dict[str, Any]: + """获取执行统计信息""" + elapsed = time.time() - self.start_time + return { + "total_processed": self.processed_count, + "elapsed_time": elapsed, + "throughput": self.processed_count / elapsed if elapsed > 0 else 0, + "node_distribution": self.node_distribution, + } + + +class CPUOnlyScheduler(BaseScheduler): + """ + CPU专用调度器 + + 特点: + - 只选择CPU节点（不需要GPU） + - 优先选择CPU资源充足的节点 + - 支持负载均衡 + """ + + def __init__(self): + super().__init__() + self.node_selector = NodeSelector() + + def make_decision(self, task_node): + """ + 为任务选择CPU节点 + + 策略: + 1. 不需要GPU资源 + 2. 选择CPU负载最低的节点 + 3. 确保有足够的CPU和内存 + """ + + # 提取CPU资源需求（默认1核） + cpu = ( + getattr(task_node.transformation, "cpu_required", 1) + if hasattr(task_node, "transformation") + else 1 + ) + + # 提取内存需求（默认1GB） + memory = ( + getattr(task_node.transformation, "memory_required", "1GB") + if hasattr(task_node, "transformation") + else "1GB" + ) + + # 选择CPU节点（不需要GPU） + target_node = self.node_selector.select_best_node( + cpu_required=cpu, + gpu_required=0, # 明确指定不需要GPU + strategy="balanced", # 负载均衡策略 + ) + + decision = PlacementDecision( + target_node=target_node, + resource_requirements={ + "cpu": cpu, + "memory": memory, + "gpu": 0, # CPU节点不需要GPU + }, + placement_strategy="cpu_only", + reason=f"CPU task: selected CPU node {target_node} (no GPU required)", + ) + + self.scheduled_count += 1 + self.decision_history.append(decision) + + return decision + + +def demo_basic_cpu_node(): + """ + 示例1: 基本的CPU节点任务执行 + + 演示: + - CPU-only任务提交 + - 任务在CPU节点上执行 + - 监控和日志记录 + """ + print("\n" + "=" * 70) + print("示例1: 基本CPU节点任务执行") + print("=" * 70) + print("\n📊 功能: 提交CPU计算任务到JobManager并在CPU节点执行") + print("🎯 验收标准:") + print(" ✓ 可以通过JobManager将任务分配给CPU SAGE节点") + print(" ✓ 节点能够正常执行并返回结果") + print(" ✓ 任务执行过程中具备基本的监控和日志记录能力\n") + + # 创建RemoteEnvironment（默认会使用CPU节点） + env = RemoteEnvironment(name="cpu_node_basic_demo") + + # 构建CPU任务流 + ( + env.from_source(CPUIntensiveSource, max_count=5, delay=0.5) + .map(CPUComputeProcessor, parallelism=2) # 2个并行CPU处理器 + .sink(CPUResultSink) + ) + + print("🚀 提交任务到JobManager...") + print("📍 任务将被分配到可用的CPU节点\n") + + # 提交并自动停止 + env.submit(autostop=True) + + print("\n✅ 示例1完成!") + print("=" * 70) + + +def demo_cpu_scheduler(): + """ + 示例2: 使用CPU专用调度器 + + 演示: + - 自定义CPU节点选择策略 + - 资源感知调度 + - 负载均衡 + """ + print("\n" + "=" * 70) + print("示例2: CPU专用调度器") + print("=" * 70) + print("\n📊 功能: 使用自定义调度器确保任务只分配到CPU节点") + print("🎯 特性:") + print(" ✓ 明确排除GPU节点") + print(" ✓ CPU资源感知调度") + print(" ✓ 负载均衡策略\n") + + # 创建使用CPU专用调度器的环境 + cpu_scheduler = CPUOnlyScheduler() + env = RemoteEnvironment( + name="cpu_scheduler_demo", + scheduler=cpu_scheduler, + ) + + # 构建CPU任务流 + ( + env.from_source(CPUIntensiveSource, max_count=8, delay=0.3) + .map(CPUComputeProcessor, parallelism=3) # 3个并行处理器 + .sink(CPUResultSink) + ) + + print("🚀 使用CPU专用调度器提交任务...") + print("📍 调度器将选择最优的CPU节点\n") + + # 提交并自动停止 + env.submit(autostop=True) + + # 查看调度统计 + metrics = cpu_scheduler.get_metrics() + print("\n📊 调度器统计:") + print(f" - 调度任务数: {metrics.get('scheduled_count', 0)}") + print(f" - 跳过任务数: {metrics.get('skipped_count', 0)}") + + print("\n✅ 示例2完成!") + print("=" * 70) + + +def demo_cpu_node_monitoring(): + """ + 示例3: CPU节点监控和日志 + + 演示: + - 任务执行监控 + - 日志记录 + - 状态查询 + """ + print("\n" + "=" * 70) + print("示例3: CPU节点监控和日志") + print("=" * 70) + print("\n📊 功能: 展示CPU节点的监控和日志能力") + print("🎯 特性:") + print(" ✓ 实时任务状态监控") + print(" ✓ 详细的日志记录") + print(" ✓ JobManager健康检查\n") + + env = RemoteEnvironment(name="cpu_monitoring_demo") + + # 构建任务流 + ( + env.from_source(CPUIntensiveSource, max_count=6, delay=0.4) + .map(CPUComputeProcessor, parallelism=2) + .sink(CPUResultSink) + ) + + print("🚀 提交任务并监控执行...") + + # 提交任务 + env.submit(autostop=True) + + print("\n📋 监控信息:") + print(" - 任务日志: 查看 .sage/logs/jobmanager/ 目录") + print(" - 所有任务执行均有日志记录") + print(" - JobManager 提供健康检查接口") + + print("\n✅ 示例3完成!") + print("=" * 70) + + +def demo_cluster_inspection(): + """ + 示例4: 集群节点检查 + + 演示: + - 查看可用的CPU节点 + - 节点资源信息 + - 集群统计 + """ + print("\n" + "=" * 70) + print("示例4: 集群节点检查") + print("=" * 70) + print("\n📊 功能: 检查集群中的CPU节点信息") + print("🎯 展示:") + print(" ✓ 可用CPU节点列表") + print(" ✓ 节点资源统计") + print(" ✓ 集群总体状态\n") + + try: + import ray + + if not ray.is_initialized(): + print("⚠️ Ray 未初始化，跳过集群检查") + return + + # 创建节点选择器 + node_selector = NodeSelector() + + # 获取集群统计信息 + stats = node_selector.get_cluster_stats() + + print("📊 集群资源统计:") + print(f" • 节点数量: {stats.get('node_count', 0)}") + print(f" • 总CPU核心: {stats.get('total_cpu', 0):.1f}") + print(f" • 可用CPU: {stats.get('available_cpu', 0):.1f}") + print(f" • CPU使用率: {stats.get('avg_cpu_usage', 0):.1%}") + print(f" • 总内存: {stats.get('total_memory', 0) / (1024**3):.2f} GB") + print(f" • 可用内存: {stats.get('available_memory', 0) / (1024**3):.2f} GB") + print(f" • 总任务数: {stats.get('total_tasks', 0)}") + + # 列出所有节点 + nodes = stats.get("nodes", []) + if nodes: + print(f"\n📋 节点详情 ({len(nodes)} 个节点):") + for i, node in enumerate(nodes, 1): + print(f"\n 节点 #{i}:") + print(f" 主机名: {node.get('hostname', 'unknown')}") + print(f" CPU使用率: {node.get('cpu_usage', 0):.1%}") + print(f" GPU使用率: {node.get('gpu_usage', 0):.1%}") + print(f" 内存使用率: {node.get('memory_usage', 0):.1%}") + print(f" 任务数: {node.get('task_count', 0)}") + + # 选择CPU节点 + print("\n🔍 选择最佳CPU节点:") + cpu_node = node_selector.select_best_node( + cpu_required=2, gpu_required=0, strategy="balanced" + ) + if cpu_node: + print(f" ✓ 选中节点: {cpu_node[:16]}...") + node_res = node_selector.get_node(cpu_node) + if node_res: + print(f" 主机名: {node_res.hostname}") + print(f" 可用CPU: {node_res.available_cpu:.1f}") + print(f" CPU使用率: {node_res.cpu_usage:.1%}") + else: + print(" ⚠️ 未找到合适的CPU节点") + + print("\n✅ 示例4完成!") + print("=" * 70) + + except ImportError: + print("⚠️ Ray 未安装，无法进行集群检查") + except Exception as e: + print(f"⚠️ 集群检查失败: {e}") + + +def demo_resource_requirements(): + """ + 示例5: 显式资源需求规范 + + 演示: + - 在Operator级别指定CPU/内存需求 + - 调度器根据资源需求选择节点 + - 资源感知任务分配 + """ + print("\n" + "=" * 70) + print("示例5: 显式资源需求规范") + print("=" * 70) + print("\n📊 功能: 为CPU任务指定精确的资源需求") + print("🎯 特性:") + print(" ✓ Operator级别资源声明") + print(" ✓ 调度器资源感知") + print(" ✓ 智能节点选择\n") + + # 创建环境 + env = RemoteEnvironment(name="cpu_resource_demo") + + # CPUComputeProcessor 已声明: cpu_required=2, memory_required="2GB", gpu_required=0 + print("💡 CPUComputeProcessor 资源需求:") + print(f" • CPU: {CPUComputeProcessor.cpu_required} 核") + print(f" • 内存: {CPUComputeProcessor.memory_required}") + print(f" • GPU: {CPUComputeProcessor.gpu_required} (不需要)") + print() + + # 构建任务流 + ( + env.from_source(CPUIntensiveSource, max_count=10, delay=0.2) + .map(CPUComputeProcessor, parallelism=3) # 每个实例需要2核CPU + .sink(CPUResultSink) + ) + + print("🚀 提交任务（调度器将选择满足资源需求的CPU节点）...") + env.submit(autostop=True) + + print("\n✅ 示例5完成!") + print("=" * 70) + + +def check_jobmanager_available(): + """检查 JobManager 是否可用""" + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + result = sock.connect_ex(("localhost", 19001)) + sock.close() + return result == 0 + except Exception: + return False + + +def print_usage_guide(): + """打印使用指南""" + print("\n" + "=" * 70) + print("📚 CPU节点使用指南") + print("=" * 70) + + print("\n1️⃣ 启动JobManager (必需):") + print(" $ sage jobmanager start") + print(" 或者手动启动:") + print(" $ python -m sage.kernel.runtime.job_manager --host 127.0.0.1 --port 19001") + + print("\n2️⃣ 启动Ray集群 (可选，JobManager会自动初始化):") + print(" $ ray start --head # 启动头节点") + print(" $ ray start --address= # 添加CPU工作节点") + + print("\n3️⃣ 配置CPU工作节点:") + print(" # 在工作节点机器上") + print(" $ ray start --address= --num-cpus=8 --num-gpus=0") + print(" # 指定只有CPU资源，不分配GPU") + + print("\n4️⃣ 检查集群状态:") + print(" $ sage jobmanager status") + print(" $ ray status") + + print("\n5️⃣ 运行CPU任务:") + print(" $ python cpu_node_demo.py") + + print("\n6️⃣ 查看日志:") + print(" $ ls -la .sage/logs/jobmanager/") + print(" $ tail -f .sage/logs/jobmanager/session_*/jobmanager.log") + + print("\n" + "=" * 70) + + +def main(): + """主函数""" + print( + """ +╔══════════════════════════════════════════════════════════════════════╗ +║ SAGE CPU Node 完整演示 ║ +║ ║ +║ 本示例演示SAGE框架对CPU版本计算节点的完整支持 ║ +║ ║ +║ 验收标准: ║ +║ ✓ 可以通过JobManager将任务分配给CPU SAGE节点 ║ +║ ✓ 节点能够正常执行并返回结果 ║ +║ ✓ 任务执行过程中具备基本的监控和日志记录能力 ║ +║ ✓ 支持资源需求规范和智能节点选择 ║ +║ ✓ 提供集群检查和统计功能 ║ +╚══════════════════════════════════════════════════════════════════════╝ + """ + ) + + # 检查JobManager是否可用 + if not check_jobmanager_available(): + print("\n⚠️ JobManager 未运行!") + print_usage_guide() + print("\n💡 请先启动 JobManager，然后重新运行本示例\n") + return + + print("\n✅ JobManager 已就绪\n") + + try: + # 运行所有示例 + demo_basic_cpu_node() + time.sleep(1) + + demo_cpu_scheduler() + time.sleep(1) + + demo_cpu_node_monitoring() + time.sleep(1) + + demo_cluster_inspection() + time.sleep(1) + + demo_resource_requirements() + + print("\n" + "=" * 70) + print("🎉 所有CPU节点演示完成!") + print("=" * 70) + + print("\n📋 验收标准确认:") + print(" ✅ JobManager成功分配任务给CPU节点") + print(" ✅ CPU节点正常执行任务并返回结果") + print(" ✅ 提供完整的监控和日志记录") + print(" ✅ 支持资源需求规范和节点选择") + print(" ✅ 提供集群检查和统计功能") + + print("\n💡 关键要点:") + print(" • CPU节点通过NodeSelector自动选择（gpu_required=0）") + print(" • RemoteEnvironment自动与JobManager协作") + print(" • 支持自定义调度策略（CPUOnlyScheduler）") + print(" • 内置监控和日志系统") + print(" • 可在无GPU环境中运行") + print(" • 支持Operator级别资源需求声明") + print(" • 提供集群资源检查工具") + + print("\n🔗 相关文件:") + print(" • JobManager: sage/kernel/runtime/job_manager.py") + print(" • NodeSelector: sage/kernel/scheduler/node_selector.py") + print(" • RemoteEnvironment: sage/kernel/api/remote_environment.py") + print(" • Scheduler: sage/kernel/scheduler/impl/resource_aware_scheduler.py") + print(" • 日志目录: .sage/logs/jobmanager/") + + print_usage_guide() + + except KeyboardInterrupt: + print("\n\n⚠️ 用户中断执行") + except Exception as e: + print(f"\n❌ 错误: {e}") + import traceback + + traceback.print_exc() + print("\n💡 提示:") + print(" 1. 确保JobManager已启动: sage jobmanager start") + print(" 2. 检查Ray是否运行: ray status") + print(" 3. 查看日志: .sage/logs/jobmanager/") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/functions/hello_comap_function_example.py b/tutorials/L3-kernel/functions/hello_comap_function_example.py new file mode 100644 index 0000000..477771e --- /dev/null +++ b/tutorials/L3-kernel/functions/hello_comap_function_example.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +SAGE CoMap Function 示例 +@test:timeout=120 +@test:category=streaming +""" + +import logging +import os +import random +import time + +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + +# 设置日志级别为ERROR减少输出 +os.environ.setdefault("SAGE_LOG_LEVEL", "ERROR") + +# 配置 Python 日志系统 +logging.basicConfig(level=logging.ERROR) +for logger_name in ["sage", "JobManager", "ray", "asyncio", "urllib3"]: + logging.getLogger(logger_name).setLevel(logging.ERROR) + +# 禁用所有INFO级别的日志 +logging.getLogger().setLevel(logging.ERROR) + + +# 温度传感器数据源 +class TemperatureSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + + def execute(self, data=None): + self.counter += 1 + # 模拟温度数据 (18-35°C) + temperature = round(random.uniform(18.0, 35.0), 1) + return { + "sensor_type": "temperature", + "value": temperature, + "unit": "°C", + "id": self.counter, + } + + +# 湿度传感器数据源 +class HumiditySource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + + def execute(self, data=None): + self.counter += 1 + # 模拟湿度数据 (30-90%) + humidity = round(random.uniform(30.0, 90.0), 1) + return { + "sensor_type": "humidity", + "value": humidity, + "unit": "%", + "id": self.counter, + } + + +# 压力传感器数据源 +class PressureSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + + def execute(self, data=None): + self.counter += 1 + # 模拟压力数据 (900-1100 hPa) + pressure = round(random.uniform(900.0, 1100.0), 1) + return { + "sensor_type": "pressure", + "value": pressure, + "unit": "hPa", + "id": self.counter, + } + + +# CoMap函数：分别处理不同类型的传感器数据 +class SensorDataProcessor(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.temp_alert_threshold = 30.0 + self.humidity_alert_threshold = 80.0 + self.pressure_alert_threshold = 1050.0 + + def map0(self, data): + """处理温度数据（来自输入流0）""" + temp_value = data["value"] + status = "🔥 HIGH" if temp_value > self.temp_alert_threshold else "✅ Normal" + return { + "stream": "temperature", + "original": data, + "processed_value": f"{temp_value}°C", + "status": status, + "alert": temp_value > self.temp_alert_threshold, + } + + def map1(self, data): + """处理湿度数据（来自输入流1）""" + humidity_value = data["value"] + status = "💧 HIGH" if humidity_value > self.humidity_alert_threshold else "✅ Normal" + return { + "stream": "humidity", + "original": data, + "processed_value": f"{humidity_value}%", + "status": status, + "alert": humidity_value > self.humidity_alert_threshold, + } + + def map2(self, data): + """处理压力数据（来自输入流2）""" + pressure_value = data["value"] + status = "⚡ HIGH" if pressure_value > self.pressure_alert_threshold else "✅ Normal" + return { + "stream": "pressure", + "original": data, + "processed_value": f"{pressure_value} hPa", + "status": status, + "alert": pressure_value > self.pressure_alert_threshold, + } + + +# 类型特定处理的CoMap函数 +class TypeSpecificProcessor(BaseCoMapFunction): + def map0(self, data): + """简单的温度数据格式化""" + return f"🌡️ Temperature: {data['value']}°C (ID: {data['id']})" + + def map1(self, data): + """简单的湿度数据格式化""" + return f"💧 Humidity: {data['value']}% (ID: {data['id']})" + + def map2(self, data): + """简单的压力数据格式化""" + return f"🔘 Pressure: {data['value']} hPa (ID: {data['id']})" + + +# 汇总输出函数 +class SensorSink(SinkFunction): + def execute(self, data): + if isinstance(data, dict) and "alert" in data: + prefix = "🚨 ALERT" if data["alert"] else "📊 DATA" + print( + f"[{self.name}] {prefix}: {data['stream']} = {data['processed_value']} ({data['status']})" + ) + else: + print(f"[{self.name}] {data}") + return data + + +def main(): + # 创建环境 + env = LocalEnvironment("comap_function_example") + + print("🚀 Starting CoMap Function Example") + print("🌡️ Demonstrating multi-sensor data processing with CoMap") + print("📊 Each sensor type is processed independently") + print("⏹️ Press Ctrl+C to stop\n") + + # 创建不同类型的传感器数据源 + temp_stream = env.from_source(TemperatureSource, delay=1.5) + humidity_stream = env.from_source(HumiditySource, delay=2.0) + pressure_stream = env.from_source(PressureSource, delay=2.5) + + print("🔗 Creating connected streams...") + + # 示例1：使用CoMap进行复杂的传感器数据处理 + print("\n📈 Example 1: Advanced Sensor Processing with CoMap") + connected_sensors = temp_stream.connect(humidity_stream).connect(pressure_stream) + + # 使用CoMap分别处理每种传感器数据 + connected_sensors.comap(SensorDataProcessor).sink(SensorSink, name="AdvancedProcessor") + + # 示例2：简单的类型特定格式化 + print("📝 Example 2: Simple Type-Specific Formatting") + connected_sensors.comap(TypeSpecificProcessor).print("🎯 Formatted Output") + + print("\n📈 All sensors connected and processing with CoMap...\n") + print("💡 CoMap Features Demonstrated:") + print(" - map0() processes temperature data independently") + print(" - map1() processes humidity data independently") + print(" - map2() processes pressure data independently") + print(" - Each stream maintains its own processing logic") + print(" - No data merging - streams are processed separately\n") + + try: + # 运行流处理 + env.submit() + + # 在测试模式下运行更短时间 + test_mode = os.environ.get("SAGE_EXAMPLES_MODE") == "test" + runtime = 8 if test_mode else 40 + + print(f"⏰ Running for {runtime} seconds...") + time.sleep(runtime) # 测试模式运行8秒，正常模式40秒 + + except KeyboardInterrupt: + print("\n\n🛑 Stopping CoMap Function Example...") + + finally: + print("\n📋 Example completed!") + print("💡 This example demonstrated:") + print(" - Multiple independent sensor data sources") + print(" - CoMap function with map0, map1, map2 methods") + print(" - Stream-specific processing logic") + print(" - Alert detection based on sensor type") + print(" - Independent processing without data merging") + print("\n🔄 Comparison with regular map():") + print(" - Regular map(): All inputs merged → single execute() method") + print(" - CoMap: Each input stream → dedicated mapN() method") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/functions/hello_comap_lambda_example.py b/tutorials/L3-kernel/functions/hello_comap_lambda_example.py new file mode 100644 index 0000000..680c3be --- /dev/null +++ b/tutorials/L3-kernel/functions/hello_comap_lambda_example.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +""" +CoMap Lambda/Callable Support Example +@test:tim # Execute example 1 + print("Processing sensor data...") + + test_mode = os.environ.get("SAGE_EXAMPLES_MODE") == "test" + if test_mode: + # In test mode, skip actual execution for faster testing + print("✅ Test mode: Skipping actual execution") + else: + env1.submit(autostop=True) + # Wait for processing to complete + import time + wait_time = 5 + time.sleep(wait_time) + + print("✅ Example 1 completed!"):category=streaming + +This example demonstrates the new lambda and callable support for CoMap operations, +showing different ways to define multi-stream processing without requiring class definitions. +""" + +import os +import sys +from typing import Any + +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment +from sage.kernel.runtime.communication.packet import StopSignal + +# 设置日志级别为ERROR减少输出 +os.environ.setdefault("SAGE_LOG_LEVEL", "ERROR") + +# Add the project root to Python path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + + +class ListSource(SourceFunction): + """Simple source that emits items from a predefined list with proper termination""" + + def __init__(self, data_list: list[Any], *args, **kwargs): + super().__init__(*args, **kwargs) + self.data_list = data_list + self.index = 0 + + def execute(self) -> Any: + if self.index >= len(self.data_list): + # Data exhausted, send stop signal + return StopSignal(f"ListSource_{self.index}") + + result = self.data_list[self.index] + self.index += 1 + return result + + +def main(): + """Demonstrate different lambda/callable usage patterns for CoMap operations""" + + print("🚀 CoMap Function Examples") + print("=" * 60) + + # Check if running in test mode - only run first example for faster testing + test_mode = os.environ.get("SAGE_EXAMPLES_MODE") == "test" + if test_mode: + print("🧪 Running in test mode - executing only first example") + + # Create environment + env1 = LocalEnvironment() + + # Example 1: Sensor Data Processing + print("\n📋 Example 1: Sensor Data Processing") + print("-" * 40) + + # Create a CoMap function to process sensor data + class SensorCoMapFunction(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def map0(self, temp): + """Process temperature data""" + status = "Hot" if temp > 23 else "Normal" + return f"🌡️ Temperature: {temp}°C ({status})" + + def map1(self, humid): + """Process humidity data""" + status = "High" if humid > 60 else "Normal" + return f"💧 Humidity: {humid}% ({status})" + + def map2(self, press): + """Process pressure data""" + status = "High" if press > 1015 else "Normal" + return f"🔘 Pressure: {press} hPa ({status})" + + # Create streams and connect them + temp_stream = env1.from_source(ListSource, [20.5, 22.1, 19.8, 25.3, 21.7]) + humidity_stream = env1.from_source(ListSource, [45, 52, 38, 67, 41]) + pressure_stream = env1.from_source(ListSource, [1013.2, 1015.8, 1012.1, 1018.5, 1014.3]) + + # Connect streams properly + connected_sensors = temp_stream.connect(humidity_stream).connect(pressure_stream) + + # Apply CoMap function + connected_sensors.comap(SensorCoMapFunction).print("Sensor Data") + + # Execute example 1 + print("Processing sensor data...") + + test_mode = os.environ.get("SAGE_EXAMPLES_MODE") == "test" + if test_mode: + # In test mode, skip actual execution for faster testing + print("✅ Test mode: Skipping actual execution") + else: + env1.submit(autostop=True) + # Wait for processing to complete + import time + + time.sleep(5) + + print("✅ Example 1 completed!") + + # In test mode, only run the first example for faster testing + if test_mode: + print("\n🧪 Test mode: Skipping remaining examples for faster execution") + print("\n✅ CoMap function example completed successfully!") + print("\n💡 Summary of CoMap usage patterns:") + print(" 1. Class-based CoMap functions (recommended)") + print(" 2. process_stream_N methods for each connected stream") + print(" 3. Built-in error handling and validation") + print(" 4. Type safety and documentation support") + + # Clean up environment + print("\n🧹 Cleaning up environment...") + env1.close() + print("✅ Environment closed successfully!") + return + + # Example 2: Weather Data Processing + print("\n📋 Example 2: Weather Data Processing") + print("-" * 40) + + # Reset environment for new example + env2 = LocalEnvironment() + + # Create weather data CoMap function + class WeatherCoMapFunction(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def map0(self, temp: float) -> str: + """Format temperature data""" + celsius = temp + fahrenheit = temp * 9 / 5 + 32 + return f"🌡️ {celsius}°C / {fahrenheit:.1f}°F" + + def map1(self, humidity: int) -> str: + """Format humidity data""" + level = "Low" if humidity < 40 else "High" if humidity > 70 else "Normal" + return f"💧 {humidity}% ({level})" + + # Create new sources + temp_source2 = env2.from_source(ListSource, [18.5, 26.2, 23.1, 29.8]) + humidity_source2 = env2.from_source(ListSource, [35, 75, 55, 82]) + + # Create and connect streams + temp_stream2 = temp_source2 + humidity_stream2 = humidity_source2 + + connected_weather = temp_stream2.connect(humidity_stream2) + + # Apply weather CoMap function + connected_weather.comap(WeatherCoMapFunction).print("Weather Data") + + # Execute example 2 + print("Processing weather data...") + env2.submit(autostop=True) + + # Wait for processing to complete + wait_time = 5 + time.sleep(wait_time) + print("✅ Example 2 completed!") + + # Example 3: Mixed Data Processing + print("\n📋 Example 3: Mixed Data Processing") + print("-" * 40) + + # Reset environment for new example + env3 = LocalEnvironment() + + # Create mixed data CoMap function + class MixedDataCoMapFunction(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def map0(self, data: float) -> str: + """Complex numeric processing with validation""" + if data < 0: + return f"⚠️ Negative value: {data}" + elif data > 100: + return f"🔥 High value: {data}" + else: + return f"✅ Normal: {data:.2f}" + + def map1(self, text: str) -> str: + """Text processing""" + return f"📝 Text: '{text}' (len={len(text)})" + + def map2(self, flag: bool) -> str: + """Boolean processing""" + return f"🏁 Flag: {flag} ({'ON' if flag else 'OFF'})" + + # Create diverse data sources + numeric_source = env3.from_source(ListSource, [15.5, -2.3, 105.7, 42.1, 0.0]) + text_source = env3.from_source(ListSource, ["hello", "world", "sage", "framework", "lambda"]) + boolean_source = env3.from_source(ListSource, [True, False, True, True, False]) + + # Create and connect streams + numeric_stream = numeric_source + text_stream = text_source + boolean_stream = boolean_source + + connected_mixed = numeric_stream.connect(text_stream).connect(boolean_stream) + + # Apply mixed data CoMap function + connected_mixed.comap(MixedDataCoMapFunction).print("Mixed Data") + + # Execute example 3 + print("Processing mixed data types...") + env3.submit(autostop=True) + + # Wait for processing to complete + time.sleep(wait_time) + print("✅ Example 3 completed!") + + # Example 4: Mathematical Operations + print("\n📋 Example 4: Mathematical Operations") + print("-" * 40) + + # Reset environment for new example + env4 = LocalEnvironment() + + # Create mathematical CoMap function + class MathCoMapFunction(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def map0(self, x: int) -> str: + """Square first stream""" + result = x**2 + return f"🔢 {x}² = {result}" + + def map1(self, x: int) -> str: + """Divide second stream by 10""" + result = x / 10 + return f"➗ {x}/10 = {result}" + + def map2(self, x: float) -> str: + """Multiply third stream by 100 and round""" + result = round(x * 100, 1) + return f"✖️ {x}×100 = {result}" + + # Create numeric data sources + input1_source = env4.from_source(ListSource, [1, 2, 3, 4, 5]) + input2_source = env4.from_source(ListSource, [10, 20, 30, 40, 50]) + input3_source = env4.from_source(ListSource, [0.1, 0.2, 0.3, 0.4, 0.5]) + + # Create and connect streams + input1 = input1_source + input2 = input2_source + input3 = input3_source + + connected_math = input1.connect(input2).connect(input3) + + # Apply mathematical transformations + connected_math.comap(MathCoMapFunction).print("Math Results") + + # Execute example 4 + print("Processing mathematical operations...") + env4.submit(autostop=True) + + # Wait for processing to complete + time.sleep(wait_time) + print("✅ Example 4 completed!") + + # Example 5: Error Handling and Validation + print("\n📋 Example 5: Error Handling and Validation") + print("-" * 40) + + # Reset environment for new example + env5 = LocalEnvironment() + + # Create validation CoMap function + class ValidationCoMapFunction(BaseCoMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def map0(self, x) -> str: + """Clamp negative numbers""" + result = max(0, x) if x is not None else 0 + status = "⬆️ clamped" if x is not None and x < 0 else "✅ valid" + return f"🔢 {x} → {result} ({status})" + + def map1(self, s) -> str: + """Handle empty/None strings""" + if s and isinstance(s, str) and s.strip(): + result = s.strip() + return f"📝 '{s}' → '{result}' (✅ valid)" + else: + return f"📝 '{s}' → 'EMPTY' (⚠️ fixed)" + + # Create data with potential issues + mixed_data1 = env5.from_source(ListSource, [5, -3, 0, 12, -1]) + mixed_data2 = env5.from_source(ListSource, ["valid", "", "test", None, "data"]) + + # Create and connect streams + data1 = mixed_data1 + data2 = mixed_data2 + + connected_validation = data1.connect(data2) + + # Apply validation and error handling + connected_validation.comap(ValidationCoMapFunction).print("Validated Data") + + # Execute example 5 + print("Processing with validation...") + env5.submit(autostop=True) + + # Wait for processing to complete + time.sleep(wait_time) + print("✅ Example 5 completed!") + + print("\n✅ All CoMap function examples completed successfully!") + print("\n💡 Summary of CoMap usage patterns:") + print(" 1. Class-based CoMap functions (recommended)") + print(" 2. process_stream_N methods for each connected stream") + print(" 3. Built-in error handling and validation") + print(" 4. Type safety and documentation support") + + # Clean up all environments + print("\n🧹 Cleaning up environments...") + env1.close() + env2.close() + env3.close() + env4.close() + env5.close() + print("✅ All environments closed successfully!") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/functions/hello_wordcount_batch_example.py b/tutorials/L3-kernel/functions/hello_wordcount_batch_example.py new file mode 100644 index 0000000..2786321 --- /dev/null +++ b/tutorials/L3-kernel/functions/hello_wordcount_batch_example.py @@ -0,0 +1,69 @@ +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.flatmap_function import FlatMapFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 批处理数据源：生成几行句子 +class SentenceBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sentences = [ + "hello world", + "hello sage", + "hello chatgpt", + "world of ai", + "sage world", + ] + self.index = 0 + + def execute(self): + if self.index >= len(self.sentences): + return None + sentence = self.sentences[self.index] + self.index += 1 + return sentence + + +# 拆分句子为单词 +class SplitWords(FlatMapFunction): + def execute(self, data): + return data.split() + + +# 转换为 (word, 1) +class WordToPair(MapFunction): + def execute(self, data): + return (data, 1) + + +class PrintResult(SinkFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counts = {} + + def execute(self, data): + word, cnt = data + self.counts[word] = self.counts.get(word, 0) + cnt + + def close(self): + print("WordCount 结果：") + for word, count in self.counts.items(): + print(f"{word}: {count}") + + +def main(): + env = LocalEnvironment("WordCount") + + # 批处理：句子 -> 拆分单词 -> 转换为(word,1) -> 聚合 -> 输出 + env.from_batch(SentenceBatch).flatmap(SplitWords).map(WordToPair).sink(PrintResult) + + env.submit(autostop=True) + print("WordCount 批处理示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/functions/hello_wordcount_lambda_example.py b/tutorials/L3-kernel/functions/hello_wordcount_lambda_example.py new file mode 100644 index 0000000..64f3112 --- /dev/null +++ b/tutorials/L3-kernel/functions/hello_wordcount_lambda_example.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +SAGE WordCount Lambda 示例 +@test:timeout=120 +@test:category=streaming +""" + +import os +import time +from collections import Counter + +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + +# 设置日志级别为ERROR减少输出 +os.environ.setdefault("SAGE_LOG_LEVEL", "ERROR") + + +# 简单的句子源，重复输出同一句话 +class SentenceSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sentences = [ + "hello world sage framework", + "this is a streaming data processing example", + "lambda functions make the code much cleaner", + "word count is a classic big data example", + "sage provides powerful stream processing capabilities", + ] + self.counter = 0 + + def execute(self, data=None): + # 随机选择一个句子，或者循环输出 + sentence = self.sentences[self.counter % len(self.sentences)] + self.counter += 1 + return sentence + + +def main(): + # 创建环境 + env = LocalEnvironment("wordcount_example") + + # 全局词汇计数器 + word_counts = Counter() + total_processed = 0 + + def update_word_count(words_with_count): + """更新全局词汇计数""" + nonlocal total_processed + word, count = words_with_count + word_counts[word] += count + total_processed += count + + # 每处理10个词就打印一次统计结果 + if total_processed % 10 == 0: + print(f"\n=== Word Count Statistics (Total: {total_processed}) ===") + for word, count in word_counts.most_common(10): + print(f"{word:20}: {count:3d}") + print("=" * 50) + + # 构建流处理管道 + ( + env.from_source(SentenceSource, delay=1.0) # 每秒产生一个句子 + # 数据清洗和预处理 + .map(lambda sentence: sentence.lower()) # 转小写 + .map(lambda sentence: sentence.strip()) # 去除首尾空白 + .filter(lambda sentence: len(sentence) > 0) # 过滤空字符串 + # 分词处理 + .flatmap(lambda sentence: sentence.split()) # 按空格分词 + .filter(lambda word: len(word) > 2) # 过滤长度小于3的词 + .map(lambda word: word.replace(",", "").replace(".", "")) # 去除标点 + # 词汇统计 + .map(lambda word: (word, 1)) # 转换为 (word, count) 格式 + .print() # 更新计数器 + ) + + print("🚀 Starting WordCount Example with Lambda Functions") + print("📝 Processing sentences and counting words...") + print("⏹️ Press Ctrl+C to stop") + + try: + # 运行流处理 + env.submit() + + # 在测试模式下运行更短时间 + test_mode = os.environ.get("SAGE_EXAMPLES_MODE") == "test" + runtime = 10 if test_mode else 60 + + print(f"⏰ Running for {runtime} seconds...") + time.sleep(runtime) # 测试模式运行10秒，正常模式60秒 + except KeyboardInterrupt: + print("\n\n🛑 Stopping WordCount Example...") + print("\n📊 Final Word Count Results:") + print("=" * 60) + for word, count in word_counts.most_common(): + print(f"{word:20}: {count:3d}") + print("=" * 60) + print(f"Total words processed: {total_processed}") + finally: + env.close() + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/keyed_state_example.py b/tutorials/L3-kernel/keyed_state_example.py new file mode 100644 index 0000000..36cde8c --- /dev/null +++ b/tutorials/L3-kernel/keyed_state_example.py @@ -0,0 +1,403 @@ +""" +Keyed State Example - User Session Tracking + +This example demonstrates how to use SAGE's keyed state support to track +user sessions with automatic state persistence. + +The example implements: +1. Real-time user session tracking +2. Per-user feature aggregation +3. Time window-based aggregations +4. Automatic state persistence and recovery + +Usage: + python examples/tutorials/l3-kernel/keyed_state_example.py +""" + +import time + +from sage.common.core.functions import MapFunction, SinkFunction, SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + +# ============================================================================== +# Data Source: User Activity Events +# ============================================================================== + + +class UserActivitySource(SourceFunction): + """ + Generate simulated user activity events. + + Events include user actions like login, page_view, click, purchase, etc. + """ + + def __init__(self, num_users=5, events_per_user=10, **kwargs): + super().__init__(**kwargs) + self.num_users = num_users + self.events_per_user = events_per_user + self.counter = 0 + self.total_events = num_users * events_per_user + + # Predefined user activity patterns + self.users = [f"user_{i}" for i in range(num_users)] + self.actions = [ + "login", + "page_view", + "click", + "add_to_cart", + "purchase", + "logout", + ] + self.pages = ["/home", "/products", "/cart", "/checkout", "/account"] + + def execute(self, data=None): + if self.counter >= self.total_events: + return None # Stop + + # Generate event + user_idx = self.counter % self.num_users + action_idx = (self.counter // self.num_users) % len(self.actions) + + event = { + "timestamp": time.time(), + "user_id": self.users[user_idx], + "action": self.actions[action_idx], + "page": self.pages[action_idx % len(self.pages)], + "value": (action_idx + 1) * 10, # Simulated value + "session_id": f"session_{self.counter // (self.num_users * 3)}", + } + + self.counter += 1 + self.logger.info(f"Generated event: {event['user_id']} - {event['action']}") + return event + + +# ============================================================================== +# Key Extractor: Extract User ID as Key +# ============================================================================== + + +class UserIdExtractor(MapFunction): + """ + Extract user_id from event as the partition key. + + This enables per-user state management downstream. + """ + + def execute(self, event: dict) -> str: + user_id = event["user_id"] + self.logger.debug(f"Extracted key: {user_id}") + return user_id + + +# ============================================================================== +# Keyed State Function: User Session Manager +# ============================================================================== + + +class UserSessionManager(MapFunction): + """ + Manage user sessions with keyed state. + + This function demonstrates: + 1. Accessing current key via ctx.get_key() + 2. Maintaining per-user state (automatically persisted) + 3. Aggregating metrics per user + 4. Time-based session management + + State Structure: + self.user_sessions = { + user_id: { + 'first_seen': timestamp, + 'last_seen': timestamp, + 'session_count': int, + 'current_session': {...}, + 'total_value': float, + 'action_counts': {...} + } + } + """ + + def __init__(self, session_timeout=300, **kwargs): + super().__init__(**kwargs) + # Keyed state - automatically persisted by SAGE + self.user_sessions = {} + self.session_timeout = session_timeout + + # Global metrics (not keyed) + self.total_events_processed = 0 + self.unique_users_seen = set() + + def execute(self, event: dict): + # Get current packet's key (user_id in this case) + user_id = self.ctx.get_key() + + # Update global metrics + self.total_events_processed += 1 + self.unique_users_seen.add(user_id) + + # Initialize user session if first time seeing this user + if user_id not in self.user_sessions: + self._initialize_user_session(user_id, event) + else: + self._update_user_session(user_id, event) + + # Get current session state + session = self.user_sessions[user_id] + + # Prepare enriched event with session context + enriched_event = { + "original_event": event, + "user_id": user_id, + "session_metrics": { + "session_count": session["session_count"], + "current_session_actions": len(session["current_session"]["actions"]), + "total_value": session["total_value"], + "session_duration": time.time() - session["current_session"]["start_time"], + "lifetime_actions": sum(session["action_counts"].values()), + }, + "global_metrics": { + "total_events": self.total_events_processed, + "unique_users": len(self.unique_users_seen), + }, + } + + self.logger.info( + f"User {user_id}: Session #{session['session_count']}, " + f"Total Value: ${session['total_value']:.2f}, " + f"Actions: {sum(session['action_counts'].values())}" + ) + + return enriched_event + + def _initialize_user_session(self, user_id: str, event: dict): + """Initialize state for a new user""" + current_time = time.time() + + self.user_sessions[user_id] = { + "first_seen": current_time, + "last_seen": current_time, + "session_count": 1, + "current_session": { + "session_id": event["session_id"], + "start_time": current_time, + "actions": [event["action"]], + "pages_visited": [event["page"]], + }, + "total_value": event["value"], + "action_counts": {event["action"]: 1}, + } + + self.logger.info(f"Initialized session for new user: {user_id}") + + def _update_user_session(self, user_id: str, event: dict): + """Update existing user session""" + session = self.user_sessions[user_id] + current_time = time.time() + + # Check if we need to start a new session (timeout or explicit session change) + time_since_last = current_time - session["last_seen"] + session_changed = event["session_id"] != session["current_session"]["session_id"] + + if time_since_last > self.session_timeout or session_changed: + # Start new session + session["session_count"] += 1 + session["current_session"] = { + "session_id": event["session_id"], + "start_time": current_time, + "actions": [event["action"]], + "pages_visited": [event["page"]], + } + self.logger.info(f"Started new session #{session['session_count']} for user {user_id}") + else: + # Update current session + session["current_session"]["actions"].append(event["action"]) + if event["page"] not in session["current_session"]["pages_visited"]: + session["current_session"]["pages_visited"].append(event["page"]) + + # Update session state + session["last_seen"] = current_time + session["total_value"] += event["value"] + + # Update action counts + action = event["action"] + session["action_counts"][action] = session["action_counts"].get(action, 0) + 1 + + +# ============================================================================== +# Window Aggregation Function +# ============================================================================== + + +class TimeWindowAggregator(MapFunction): + """ + Perform time-based window aggregations with keyed state. + + Demonstrates: + 1. Sliding time windows per user + 2. Automatic cleanup of old windows + 3. Aggregations within windows + """ + + def __init__(self, window_size=60, **kwargs): # 60 second windows + super().__init__(**kwargs) + self.window_size = window_size + # Keyed state: {user_id: {window_id: [events]}} + self.window_data = {} + + def execute(self, enriched_event: dict): + user_id = self.ctx.get_key() + current_time = time.time() + window_id = int(current_time // self.window_size) + + # Initialize user's window data + if user_id not in self.window_data: + self.window_data[user_id] = {} + + # Add event to current window + if window_id not in self.window_data[user_id]: + self.window_data[user_id][window_id] = [] + + self.window_data[user_id][window_id].append(enriched_event) + + # Cleanup old windows (keep last 3 windows) + old_windows = [wid for wid in self.window_data[user_id] if wid < window_id - 2] + for wid in old_windows: + del self.window_data[user_id][wid] + + # Calculate window aggregations + current_window = self.window_data[user_id][window_id] + aggregations = { + "window_id": window_id, + "window_start": window_id * self.window_size, + "window_end": (window_id + 1) * self.window_size, + "event_count": len(current_window), + "total_value": sum(e["original_event"]["value"] for e in current_window), + "unique_actions": len({e["original_event"]["action"] for e in current_window}), + } + + return { + "user_id": user_id, + "window": aggregations, + "latest_event": enriched_event, + } + + +# ============================================================================== +# Sink: Display Results +# ============================================================================== + + +class ResultDisplaySink(SinkFunction): + """Display processed results with session and window information""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.result_count = 0 + + def execute(self, result: dict): + self.result_count += 1 + + print(f"\n{'=' * 70}") + print(f"Result #{self.result_count}") + print(f"{'=' * 70}") + + # User info + print(f"👤 User: {result['user_id']}") + + # Window info + if "window" in result: + window = result["window"] + print(f"\n📊 Window Aggregation (Window ID: {window['window_id']}):") + print(f" Events in Window: {window['event_count']}") + print(f" Total Value: ${window['total_value']:.2f}") + print(f" Unique Actions: {window['unique_actions']}") + + # Session info + if "latest_event" in result and "session_metrics" in result["latest_event"]: + metrics = result["latest_event"]["session_metrics"] + print("\n📈 Session Metrics:") + print(f" Session #: {metrics['session_count']}") + print(f" Current Session Actions: {metrics['current_session_actions']}") + print(f" Lifetime Value: ${metrics['total_value']:.2f}") + print(f" Total Actions: {metrics['lifetime_actions']}") + print(f" Session Duration: {metrics['session_duration']:.1f}s") + + # Global metrics + if "latest_event" in result and "global_metrics" in result["latest_event"]: + global_m = result["latest_event"]["global_metrics"] + print("\n🌍 Global Metrics:") + print(f" Total Events Processed: {global_m['total_events']}") + print(f" Unique Users: {global_m['unique_users']}") + + return result + + +# ============================================================================== +# Main Example +# ============================================================================== + + +def main(): + """ + Run the keyed state example. + + Pipeline: + UserActivitySource + -> KeyBy(UserIdExtractor) + -> Map(UserSessionManager) # Maintains per-user sessions + -> Map(TimeWindowAggregator) # Maintains per-user time windows + -> Sink(ResultDisplaySink) + """ + print("\n" + "=" * 70) + print("SAGE Keyed State Example - User Session Tracking") + print("=" * 70) + + # Create environment + env = LocalEnvironment("keyed_state_example") + + # Build pipeline + ( + env.from_source( + UserActivitySource, + num_users=3, # 3 users + events_per_user=8, # 8 events each + delay=0.5, # 0.5s between events + ) + .keyby(UserIdExtractor, strategy="hash") # Partition by user_id + .map(UserSessionManager, session_timeout=10) # Track sessions + .map(TimeWindowAggregator, window_size=30) # 30-second windows + .sink(ResultDisplaySink) + ) + + print("\n🚀 Starting pipeline...") + print(" - 3 users, 8 events each (24 total events)") + print(" - Events generated every 0.5 seconds") + print(" - Sessions tracked per user with 10s timeout") + print(" - Time windows of 30 seconds per user") + print() + + try: + # Submit and run + env.submit() + + # Let it run for a while + time.sleep(15) + + print("\n" + "=" * 70) + print("Pipeline completed successfully!") + print("=" * 70) + + except KeyboardInterrupt: + print("\n⚠️ Pipeline interrupted by user") + except Exception as e: + print(f"\n❌ Error: {e}") + import traceback + + traceback.print_exc() + finally: + env.close() + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/operators/hello_comap_world.py b/tutorials/L3-kernel/operators/hello_comap_world.py new file mode 100644 index 0000000..76808c0 --- /dev/null +++ b/tutorials/L3-kernel/operators/hello_comap_world.py @@ -0,0 +1,65 @@ +# 此例意在说明如何将两个流通过comap合为一个流 + +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 定义两个简单数据源： +class SourceOne(BatchFunction): + def __init__(self): + super().__init__() + self.counter = 0 + + def execute(self): + self.counter += 1 + if self.counter > 5: + return None + return {"msg": f"No.{self.counter}: Hello"} + + +class SourceTwo(BatchFunction): + def __init__(self): + super().__init__() + self.counter = 0 + + def execute(self): + self.counter += 1 + if self.counter > 5: + return None + return {"msg": f"World! #{self.counter}"} + + +class HelloCoMapProcessor(BaseCoMapFunction): + def map0(self, data): + return f"[Stream0] 👋 {data['msg']}" + + def map1(self, data): + return f"[Stream1] 🌍 {data['msg']}" + + +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +def main(): + env = LocalEnvironment("Hello_CoMap_World") + + # 两个数据源 + source1 = env.from_batch(SourceOne) + source2 = env.from_batch(SourceTwo) + + # 将两个流 connect 在一起，并用 comap 分开处理 + source1.connect(source2).comap(HelloCoMapProcessor).sink(PrintSink) + + env.submit(autostop=True) + + print("Hello Comap World 示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/operators/hello_filter_world.py b/tutorials/L3-kernel/operators/hello_filter_world.py new file mode 100644 index 0000000..c3f740f --- /dev/null +++ b/tutorials/L3-kernel/operators/hello_filter_world.py @@ -0,0 +1,54 @@ +# 此例意在说明 Fileter 算子的使用 +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.filter_function import FilterFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +class HelloBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 10 + + def execute(self): + if self.counter >= self.max_count: + return None + self.counter += 1 + return f"Hello, World! #{self.counter}" + + +class UpperCaseMap(MapFunction): + def execute(self, data): + return data.upper() + + +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +# 过滤器示例，过滤所有偶数结尾的数据 +class Oddpicker(FilterFunction): + def execute(self, data): + if int(data[-1]) % 2 != 0: + return data + else: + return None + + +def main(): + env = LocalEnvironment("Hello_Filter_World") + + env.from_batch(HelloBatch).map(UpperCaseMap).filter(Oddpicker).sink(PrintSink) + + env.submit(autostop=True) + print("Hello Filter World 示例结束") + + +if __name__ == "__main__": + # 关闭日志输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/operators/hello_flatmap_world.py b/tutorials/L3-kernel/operators/hello_flatmap_world.py new file mode 100644 index 0000000..85e2f9f --- /dev/null +++ b/tutorials/L3-kernel/operators/hello_flatmap_world.py @@ -0,0 +1,52 @@ +# 此例意在说明FlatMap的使用 +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.flatmap_function import FlatMapFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +class HelloBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 10 + + def execute(self): + if self.counter >= self.max_count: + return None + self.counter += 1 + return f"Hello, World! #{self.counter}" + + +class UpperCaseMap(MapFunction): + def execute(self, data): + return data.upper() + + +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +# 利用FlatMapFunction实现单词拆分 +class SplitWords(FlatMapFunction): + def execute(self, data): + words = data.split() + return words + + +def main(): + env = LocalEnvironment("Hello_Flatmap_World") + + env.from_batch(HelloBatch).map(UpperCaseMap).flatmap(SplitWords).sink(PrintSink) + + env.submit(autostop=True) + print("Hello Flatmap World 示例结束") + + +if __name__ == "__main__": + # 关闭日志输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/operators/hello_join_world.py b/tutorials/L3-kernel/operators/hello_join_world.py new file mode 100644 index 0000000..c8ffb56 --- /dev/null +++ b/tutorials/L3-kernel/operators/hello_join_world.py @@ -0,0 +1,104 @@ +# 此例用到了keyby和join操作符，展示如何将两个数据流按key进行关联。 +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.join_function import BaseJoinFunction +from sage.common.core.functions.keyby_function import KeyByFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +class SourceOne(BatchFunction): + def __init__(self): + super().__init__() + self.counter = 0 + + def execute(self): + self.counter += 1 + if self.counter > 5: + return None + return {"id": self.counter, "msg": f"Hello-{self.counter}", "type": "hello"} + + +class SourceTwo(BatchFunction): + def __init__(self): + super().__init__() + self.counter = 0 + + def execute(self): + self.counter += 1 + if self.counter > 5: + return None + return {"id": self.counter, "msg": f"World-{self.counter}", "type": "world"} + + +class IdKeyBy(KeyByFunction): + def execute(self, data): + return data.get("id") + + +class PrintSink(SinkFunction): + def execute(self, data): + print(f"🔗 Joined Streaming: {data}") + + +class HelloWorldJoin(BaseJoinFunction): + """ + Join 算子示例： + execute(payload, key, tag) 参数说明： + - payload: 流里传过来的原始数据 (dict) + - key: 由 keyby 算子提取出来的分区键 (比如这里的 id) + - tag: 数据来源标识 (0=左流 / 第一个流, 1=右流 / 第二个流) + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.hello_cache = {} # {key: [payloads]} + self.world_cache = {} # {key: [payloads]} + + def execute(self, payload, key, tag): + results = [] + data_type = payload.get("type", "") + + if tag == 0: # 第一个流 (Hello) + if data_type == "hello": + # 缓存 Hello 数据 + self.hello_cache.setdefault(key, []).append(payload) + + # 检查是否有匹配的 World 数据 + if key in self.world_cache: + for world_data in self.world_cache[key]: + results.append(self._merge(payload, world_data, key)) + + elif tag == 1: # 第二个流 (World) + if data_type == "world": + # 缓存 World 数据 + self.world_cache.setdefault(key, []).append(payload) + + # 检查是否有匹配的 Hello 数据 + if key in self.hello_cache: + for hello_data in self.hello_cache[key]: + results.append(self._merge(hello_data, payload, key)) + + return results + + def _merge(self, hello_data, world_data, key): + return {"id": key, "msg": f"{hello_data['msg']} + {world_data['msg']}"} + + +def main(): + env = LocalEnvironment("hello_join_world") + + source1 = env.from_batch(SourceOne) + source2 = env.from_batch(SourceTwo) + + source1.keyby(IdKeyBy).connect(source2.keyby(IdKeyBy)).join(HelloWorldJoin).sink(PrintSink) + + # 使用 autostop=True 让框架自动检测处理完成 + env.submit(autostop=True) + + print("Hello Join World 示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/operators/hello_three_input_comap.py b/tutorials/L3-kernel/operators/hello_three_input_comap.py new file mode 100644 index 0000000..afbe5cc --- /dev/null +++ b/tutorials/L3-kernel/operators/hello_three_input_comap.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Hello Three Input CoMap World + +这个例子演示了如何使用CoMap操作处理三个输入流，每个流的数据 +会被路由到对应的mapN方法进行独立处理。 + +CoMap（Co-processing Map）是一种多流处理操作，允许对连接的多个 +数据流进行协同处理，每个输入流通过专用的mapN方法独立处理。 +""" + +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.comap_function import BaseCoMapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +class SimpleDataSource(BatchFunction): + """简单的批量数据源""" + + def __init__(self, data): + super().__init__() + self.data = data + self.index = 0 + + def execute(self): + if self.index >= len(self.data): + return None + result = self.data[self.index] + self.index += 1 + return result + + +class ThreeStreamCoMapFunction(BaseCoMapFunction): + """ + 三输入流CoMap函数 + + 演示如何处理三个不同的输入流： + - map0: 处理第一个流的数据 + - map1: 处理第二个流的数据 + - map2: 处理第三个流的数据 + """ + + def map0(self, data): + """处理第一个输入流的数据""" + return f"🔴 Stream-0: {data}" + + def map1(self, data): + """处理第二个输入流的数据""" + return f"🟡 Stream-1: {data}" + + def map2(self, data): + """处理第三个输入流的数据""" + return f"🔵 Stream-2: {data}" + + +class ConsoleSink(SinkFunction): + """控制台输出Sink""" + + def execute(self, data): + print(data) + + +def main(): + """主函数：演示三输入流CoMap操作""" + + # 创建本地环境 + env = LocalEnvironment("ThreeInputCoMapExample") + + print("🚀 Starting Three Input CoMap Example...") + print("=" * 50) + + # 创建三个数据源 + stream1 = env.from_batch(SimpleDataSource, ["Apple", "Banana"]) + stream2 = env.from_batch(SimpleDataSource, ["Cat", "Dog"]) + stream3 = env.from_batch(SimpleDataSource, ["Red", "Blue"]) + + print("📊 Data sources created:") + print(" Stream 1 (Fruits): [Apple, Banana]") + print(" Stream 2 (Animals): [Cat, Dog]") + print(" Stream 3 (Colors): [Red, Blue]") + print() + + # 连接三个流并应用CoMap + print("🔗 Connecting streams and applying CoMap...") + (stream1.connect(stream2).connect(stream3).comap(ThreeStreamCoMapFunction).sink(ConsoleSink)) + + print("⚙️ Processing data...") + print() + + # 执行流处理 + env.submit(autostop=True) + + print() + print("✅ Three Input CoMap Example completed!") + print("=" * 50) + print("📝 Each input stream was processed by its corresponding mapN method:") + print(" - Stream 1 data → map0() → 🔴 Stream-0: ...") + print(" - Stream 2 data → map1() → 🟡 Stream-1: ...") + print(" - Stream 3 data → map2() → 🔵 Stream-2: ...") + + +if __name__ == "__main__": + # 禁用全局调试日志以获得更清晰的输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/stream/hello_connected_stream_example.py b/tutorials/L3-kernel/stream/hello_connected_stream_example.py new file mode 100644 index 0000000..c781751 --- /dev/null +++ b/tutorials/L3-kernel/stream/hello_connected_stream_example.py @@ -0,0 +1,102 @@ +import time + +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + + +# 简单的数字源 +class NumberSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + + def execute(self, data=None): + self.counter += 1 + return self.counter + + +# 简单的统计汇总函数 +class StatsSink(SinkFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def execute(self, data): + print(f"[{self.name}] Received: {data}") + return data + + +def main(): + # 创建环境 + env = LocalEnvironment("simple_connected_example") + + # 设置日志级别为WARNING以减少调试输出 + env.set_console_log_level("WARNING") + + print("🚀 Starting Simple Connected Streams Example") + print("📊 Demonstrating multiple stream processing and connection") + print("⏹️ Press Ctrl+C to stop\n") + + # 创建主数据源 + main_stream = env.from_source(NumberSource, delay=1.0) + + # 分支1：偶数流 + even_stream = ( + main_stream.filter(lambda x: x % 2 == 0).map(lambda x: ("EVEN", x)) + # .print("🔵 Even Stream") + ) + + # 分支2：奇数流 + odd_stream = ( + main_stream.filter(lambda x: x % 2 == 1).map(lambda x: ("ODD", x)) + # .print("🔴 Odd Stream") + ) + + # 分支3：倍数流（3的倍数） + multiple_stream = ( + main_stream.filter(lambda x: x % 3 == 0).map(lambda x: ("MULTIPLE_3", x)) + # .print("🟡 Multiple-3 Stream") + ) + + # 分支4：大数流（大于5） + large_stream = ( + main_stream.filter(lambda x: x > 5).map(lambda x: ("LARGE", x)) + # .print("🟢 Large Stream") + ) + + # 使用 ConnectedStreams 将所有分支连接起来 + print("\n🔗 Connecting all streams...") + connected_streams = ( + even_stream.connect(odd_stream).connect(multiple_stream).connect(large_stream) + ) + + # 对连接的流进行统一处理 + ( + connected_streams.map(lambda data: f"Processed: {data[0]} -> {data[1]}") + .print("🎯 Final Result") + .sink(StatsSink, name="FinalSink") + ) + + print("📈 All streams connected and processing...\n") + + try: + # 运行流处理 + env.submit() + + time.sleep(5) # 运行5秒 + + except KeyboardInterrupt: + print("\n\n🛑 Stopping Simple Connected Streams Example...") + + finally: + print("\n📋 Example completed!") + print("💡 This example demonstrated:") + print(" - Multiple stream branches from single source") + print(" - Independent filtering and processing") + print(" - ConnectedStreams merging multiple flows") + print(" - Unified final processing of merged streams") + env.close() + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-kernel/stream/hello_onebyone_world.py b/tutorials/L3-kernel/stream/hello_onebyone_world.py new file mode 100644 index 0000000..a05ba58 --- /dev/null +++ b/tutorials/L3-kernel/stream/hello_onebyone_world.py @@ -0,0 +1,47 @@ +import time + +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +class SyncBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 5 + + def execute(self): + if self.counter >= self.max_count: + return None + self.counter += 1 + data = f"hello, No. {str(self.counter)} one by one world~" + print(f" ⚡ {data}") + return data + + +class UpperMap(MapFunction): + def execute(self, data): + print(" 🔔 uppering word!!!") + time.sleep(1) + return data.upper() + + +class SyncSink(SinkFunction): + def execute(self, data): + print(f" ✅ {data}") + time.sleep(1) + + +def main(): + env = LocalEnvironment("Test_Sync") + env.from_batch(SyncBatch).map(UpperMap).sink(SyncSink) + env.submit(autostop=True) + print("Hello one by one World 批处理示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/stream/hello_streaming_world.py b/tutorials/L3-kernel/stream/hello_streaming_world.py new file mode 100644 index 0000000..e000a56 --- /dev/null +++ b/tutorials/L3-kernel/stream/hello_streaming_world.py @@ -0,0 +1,53 @@ +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 流式数据源：从BatchFunction变成SourceFunction +class HelloStreaming(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + + def execute(self, data=None): + self.counter += 1 + return f"Hello, Streaming World! #{self.counter}" + + +class UpperCaseMap(MapFunction): + def execute(self, data): + return data.upper() + + +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +def main(): + env = LocalEnvironment("hello_streaming_world") + + # 流式源，从 from_batch 变成 from_source + env.from_source(HelloStreaming).map(UpperCaseMap).sink(PrintSink) + + try: + print("Waiting for streaming processing to complete...") + env.submit() + + # 暂停主程序，因为在LocalEnvironment下，流式处理是异步的 + from time import sleep + + sleep(1) + + except KeyboardInterrupt: + print("停止运行") + finally: + print("Hello Streaming World 流式处理示例结束") + + +if __name__ == "__main__": + # 关闭日志输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-kernel/stream/hello_wordcount_source_example.py b/tutorials/L3-kernel/stream/hello_wordcount_source_example.py new file mode 100644 index 0000000..21d41f8 --- /dev/null +++ b/tutorials/L3-kernel/stream/hello_wordcount_source_example.py @@ -0,0 +1,77 @@ +import time + +from sage.common.core.functions.flatmap_function import FlatMapFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 流数据源：每次输出一行句子 +class SentenceSource(SourceFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sentences = [ + "hello world", + "hello sage", + "hello chatgpt", + "world of ai", + "sage world", + ] + self.index = 0 + + def execute(self, data=None): + # 无限流：每次输出一句话，模拟流数据源 + if self.index >= len(self.sentences): + self.index = 0 # 重置索引，实现循环输出 + sentence = self.sentences[self.index] + self.index += 1 + return sentence + + +# 拆分句子为单词 +class SplitWords(FlatMapFunction): + def execute(self, data): + return data.split() + + +# 转换为 (word, 1) +class WordToPair(MapFunction): + def execute(self, data): + return (data, 1) + + +# SinkFunction 输出结果：每次输出单词计数 +class PrintResult(SinkFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counts = {} + + def execute(self, data): + word, cnt = data + self.counts[word] = self.counts.get(word, 0) + cnt + + # 每次接收到新数据时，输出当前统计结果 + print("当前单词计数：") + for word, count in self.counts.items(): + print(f"{word}: {count}") + print("------") + + +def main(): + env = LocalEnvironment("WordCount") + + # 流式处理：句子 -> 拆分单词 -> 转换为(word,1) -> 输出每次的单词统计 + env.from_source(SentenceSource).flatmap(SplitWords).map(WordToPair).sink(PrintResult) + + env.submit() # 设置为 False 以保持流式执行 + + # 模拟流式数据源持续运行一段时间（这里设定为 10 秒） + time.sleep(10) + print("WordCount 流式示例结束") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L3-libs/README.md b/tutorials/L3-libs/README.md new file mode 100644 index 0000000..49a6a9c --- /dev/null +++ b/tutorials/L3-libs/README.md @@ -0,0 +1,87 @@ +# L3: Libs - 算法库层示例 + +> 对应 SAGE 包：`sage-libs` + +## 📖 层级说明 + +**Libs** 层提供算法库和工具（与 Kernel 同层）： + +- RAG - 检索增强生成 +- Agents - 智能体框架 +- Embeddings - 向量嵌入 +- LLM - 大语言模型集成 +- Unlearning - 机器遗忘 + +## 📚 目录结构 + +``` +L3-libs/ +├── rag/ # RAG 应用 +├── agents/ # 智能体应用 +├── embeddings/ # 嵌入应用 +├── llm/ # LLM 应用 +└── unlearning/ # 机器遗忘 +``` + +## 🎯 学习路径 + +### 1️⃣ RAG 应用 (`rag/`) + +从简单到完整的 RAG 系统： + +- `simple_rag.py` - 简单 RAG 示例 +- `qa_local_llm.py` - 本地 LLM 问答 +- `qa_no_retrieval.py` - 无检索问答 +- `usage_1_direct_library.py` - 直接使用库 +- `usage_2_sage_function.py` - SAGE 函数集成 +- `usage_3_memory_service.py` - 内存服务集成 +- `usage_4_complete_rag.py` - 完整 RAG 系统 + +### 2️⃣ Agents 应用 (`agents/`) + +构建智能体系统： + +- `basic_agent.py` - 基础智能体 +- `workflow_demo.py` - 工作流演示 +- `arxiv_search_tool.py` - arXiv 搜索工具 +- `demo_arxiv_search.py` - 搜索演示 + +### 3️⃣ Embeddings 应用 (`embeddings/`) + +向量嵌入和相似度搜索： + +- `embedding_demo.py` - 嵌入演示 +- `embedding_service_demo.py` - 嵌入服务 +- `pipeline_builder_embedding_demo.py` - 管道构建器 +- `cross_modal_search.py` - 跨模态搜索 + +### 4️⃣ LLM 应用 (`llm/`) + +大语言模型集成： + +- `pipeline_builder_llm_demo.py` - LLM 管道 +- `templates_to_llm_demo.py` - 模板演示 +- `demo_new_templates.py` - 新模板演示 +- `test_real_llm.py` - 真实 LLM 测试 + +### 5️⃣ Unlearning 应用 (`unlearning/`) + +机器遗忘技术： + +- `basic_unlearning_demo.py` - 基础演示 + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. 如何构建 RAG 系统 +1. 如何设计智能体 +1. 向量嵌入的应用 +1. LLM 集成的最佳实践 +1. 机器遗忘的基本原理 + +## ⏭️ 下一步 + +学完算法库层后，继续学习： + +- **L4-middleware/** - 中间件和领域算子 diff --git a/tutorials/L3-libs/agent_sft_demo.py b/tutorials/L3-libs/agent_sft_demo.py new file mode 100644 index 0000000..46247e0 --- /dev/null +++ b/tutorials/L3-libs/agent_sft_demo.py @@ -0,0 +1,179 @@ +""" +Agent SFT + Eval Usage Demo + +Demonstrates the complete workflow of using agent_sft data source +and agent_eval usage profiles. +""" + +from sage.data.sources.agent_sft import AgentSFTDataLoader + + +def demo_agent_sft_loader(): + """Demonstrate AgentSFTDataLoader functionality.""" + print("=" * 70) + print("AGENT SFT DATA SOURCE DEMO") + print("=" * 70) + print() + + # Initialize loader + print("1. Initializing AgentSFTDataLoader...") + loader = AgentSFTDataLoader() + print(" ✓ Loader initialized successfully") + print() + + # Display statistics + print("2. Dataset Statistics:") + stats = loader.get_stats() + print(f" Total dialogs: {stats.total_dialogs}") + print(f" - Train: {stats.train_count}") + print(f" - Dev: {stats.dev_count}") + print(f" - Test: {stats.test_count}") + print(f" Average turns per dialog: {stats.avg_turns}") + print(f" Average tools per dialog: {stats.avg_tools_per_dialog}") + print(f" Unique tools used: {stats.unique_tools}") + print() + + # Show top tools + print("3. Top 5 Most Used Tools:") + sorted_tools = sorted(stats.tool_coverage.items(), key=lambda x: x[1], reverse=True) + for i, (tool_id, count) in enumerate(sorted_tools[:5], 1): + print(f" {i}. {tool_id}: {count} dialogs") + print() + + # Demonstrate iteration + print("4. Sample Dialogs from Training Set:") + for i, dialog in enumerate(loader.iter_dialogs("train")): + if i >= 2: # Show only 2 dialogs + break + print(f"\n Dialog {dialog.dialog_id}:") + print(f" Goal: {dialog.goal}") + print(f" Tools: {', '.join(dialog.target_tools)}") + print(f" Turns: {len(dialog.turns)}") + print(" First 3 turns:") + for j, turn in enumerate(dialog.turns[:3], 1): + content_preview = turn.content[:50] + "..." if len(turn.content) > 50 else turn.content + print(f" {j}. [{turn.role}] {content_preview}") + print() + + # Demonstrate batch sampling + print("5. Batch Sampling:") + batch = loader.sample_batch(batch_size=5, split="train", shuffle=False) + print(f" Sampled {len(batch)} dialogs for training") + for dialog in batch: + print(f" - {dialog.dialog_id}: {dialog.goal[:40]}...") + print() + + # Demonstrate filtering + print("6. Filtering by Difficulty:") + hard_dialogs = loader.filter_by_difficulty("hard", split="test") + print(f" Found {len(hard_dialogs)} hard dialogs in test set") + if hard_dialogs: + print(f" Example: {hard_dialogs[0].dialog_id} - {hard_dialogs[0].goal}") + print() + + # Demonstrate tool filtering + print("7. Filtering by Tool:") + if stats.tool_coverage: + sample_tool = list(stats.tool_coverage.keys())[0] + tool_dialogs = loader.filter_by_tool(sample_tool, split="train") + print(f" Tool '{sample_tool}' is used in {len(tool_dialogs)} training dialogs") + print() + + # Demonstrate dialog lookup + print("8. Dialog Lookup by ID:") + dialog = loader.get_dialog("sft_000001") + if dialog: + print(f" Found dialog: {dialog.dialog_id}") + print(f" Goal: {dialog.goal}") + print(f" Split: {dialog.split}") + else: + print(" Dialog not found (may have been filtered out due to validation)") + print() + + print("=" * 70) + print("✓ DEMO COMPLETED SUCCESSFULLY") + print("=" * 70) + + +def demo_usage_profiles(): + """Demonstrate agent_eval usage profiles (conceptual).""" + print("\n" + "=" * 70) + print("AGENT EVAL USAGE PROFILES (Conceptual Demo)") + print("=" * 70) + print() + + print("Profile Configurations:") + print() + + print("1. quick_eval:") + print(" - Purpose: Fast validation during development") + print(" - Sources: agent_benchmark") + print(" - Tasks: tool_selection only") + print(" - Split: dev (100 samples max)") + print(" - Use: CI testing, rapid iteration") + print() + + print("2. full_eval:") + print(" - Purpose: Comprehensive model evaluation") + print(" - Sources: agent_benchmark + agent_tools") + print(" - Tasks: tool_selection, task_planning, timing_judgment") + print(" - Split: test") + print(" - Use: Final benchmarks, paper results") + print() + + print("3. sft_training:") + print(" - Purpose: Agent model training") + print(" - Sources: agent_sft + agent_tools") + print(" - Split: train") + print(" - Parameters: batch_size=32, max_turns=12, shuffle=true") + print(" - Use: Supervised fine-tuning workflows") + print() + + print("Integration Example:") + print(""" + from sage.data import DataManager + + # Load usage + manager = DataManager.get_instance() + agent_eval = manager.get_by_usage("agent_eval") + + # Quick evaluation + quick = agent_eval.load_profile("quick_eval") + for sample in quick["benchmark"].iter_split("tool_selection", "dev"): + # Run fast evaluation + ... + + # Full evaluation + full = agent_eval.load_profile("full_eval") + benchmark = full["benchmark"] + tools = full["tools"] + # Run comprehensive tests + ... + + # SFT training + sft = agent_eval.load_profile("sft_training") + training_data = sft["sft"] + # Train model + ... + """) + + print("=" * 70) + + +def main(): + """Run all demos.""" + # Demo 1: Agent SFT DataLoader + demo_agent_sft_loader() + + # Demo 2: Usage Profiles (conceptual, since other sources aren't implemented yet) + demo_usage_profiles() + + print("\n✅ All demos completed successfully!") + print("\nNote: Full integration requires:") + print(" - Subtask 1: agent_tools data source") + print(" - Subtask 2: agent_benchmark data source") + print(" - DataManager registration (may be automatic)") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/agents/README.md b/tutorials/L3-libs/agents/README.md new file mode 100644 index 0000000..7b3e5ca --- /dev/null +++ b/tutorials/L3-libs/agents/README.md @@ -0,0 +1,34 @@ +# Agent Tutorials + +Simple examples demonstrating how to build and use agents with SAGE. + +## Examples + +### 1. Basic Agent (`basic_agent.py`) + +Learn the fundamentals of creating an agent. + +```bash +python examples/tutorials/agents/basic_agent.py +``` + +### 2. Agent Workflow (`workflow_demo.py`) + +Demonstrates agent workflow patterns. + +```bash +python examples/tutorials/agents/workflow_demo.py +``` + +### 3. ArXiv Search Tool (`arxiv_search_tool.py`) + +Example of a custom tool for agents - searches academic papers. + +```bash +python examples/tutorials/agents/demo_arxiv_search.py +``` + +## Next Steps + +- Check out the full agent library in `packages/sage-libs/src/sage/libs/agentic/agents/` +- See production agent applications in `packages/sage-apps/` diff --git a/tutorials/L3-libs/agents/__init__.py b/tutorials/L3-libs/agents/__init__.py new file mode 100644 index 0000000..fe6397b --- /dev/null +++ b/tutorials/L3-libs/agents/__init__.py @@ -0,0 +1 @@ +"""Agent examples and tools.""" diff --git a/tutorials/L3-libs/agents/arxiv_search_tool.py b/tutorials/L3-libs/agents/arxiv_search_tool.py new file mode 100644 index 0000000..cf66c7a --- /dev/null +++ b/tutorials/L3-libs/agents/arxiv_search_tool.py @@ -0,0 +1,239 @@ +# examples/agents/tools/arxiv_mcp_tool.py +from __future__ import annotations + +import logging +import re +import time +from typing import Any + +import requests +from bs4 import BeautifulSoup +from bs4.element import Tag + + +class ArxivSearchTool: + """ + MCP 工具：arxiv_search + - 接口三要素： + name = "arxiv_search" + description = "Search arXiv papers; return a list of {title, authors, link, abstract}." + input_schema = {...} + - 入口： + call({"query": "...", "size": 25, "max_results": 2}) -> {"output": [...], "meta": {...}} + """ + + name = "arxiv_search" + description = "Search arXiv papers; return a list of {title, authors, link, abstract}." + input_schema = { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query for arXiv."}, + "size": { + "type": "integer", + "enum": [25, 50, 100, 200], + "default": 25, + "description": "Results per page on arXiv.", + }, + "max_results": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 10, + "description": "Maximum number of papers to return (<=100).", + }, + "with_abstract": { + "type": "boolean", + "default": True, + "description": "Whether to include abstract in results.", + }, + }, + "required": ["query"], + "additionalProperties": False, + } + + def __init__(self): + self.base_url = "https://arxiv.org/search/" + self.valid_sizes = [25, 50, 100, 200] + self.session = requests.Session() + # 设置更像真实浏览器的请求头 + self.session.headers.update( + { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Cache-Control": "max-age=0", + } + ) + + # === MCP 入口 === + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + query: str = (arguments.get("query") or "").strip() + if not query: + raise ValueError("`query` is required and must be a non-empty string.") + + # 清理查询字符串，移除可能导致问题的字符 + query = self._clean_query(query) + + size: int = int(arguments.get("size", 25) or 25) + if size not in self.valid_sizes: + size = min(self.valid_sizes, key=lambda x: abs(x - size)) + + max_results: int = int(arguments.get("max_results", 10) or 10) + max_results = max(1, min(max_results, 100)) + + with_abs: bool = bool(arguments.get("with_abstract", True)) + + try: + items = self._search_arxiv( + query=query, size=size, max_results=max_results, with_abstract=with_abs + ) + return { + "output": items, + "meta": {"query": query, "size": size, "max_results": max_results}, + } + except Exception as e: + logging.error(f"[arxiv_search] online search failed: {e}") + # 离线兜底：返回 mock，保证示例可跑 + k = max_results + demo = [ + { + "title": f"Survey of LLM Agents ({i + 1})", + "authors": "Alice, Bob", + "link": f"https://arxiv.org/abs/2509.{1234 + i}", + "abstract": "(mock) An overview of LLM-based agents, planning, and tool use.", + } + for i in range(k) + ] + return {"output": demo, "meta": {"query": query, "offline_mock": True}} + + def _clean_query(self, query: str) -> str: + """清理查询字符串，避免可能导致服务器错误的字符""" + import re + + # 替换可能导致问题的词汇组合 + problematic_patterns = { + r"\bvs\b": "versus", + r"\bcompare?\b": "analysis", + r"\bcomparison\b": "analysis", + r"\bdifferent\b": "analysis", + r"\bdifference\b": "analysis", + } + + for pattern, replacement in problematic_patterns.items(): + query = re.sub(pattern, replacement, query, flags=re.IGNORECASE) + + # 移除多余的空格和特殊字符 + query = re.sub(r"\s+", " ", query) # 多个空格变成单个空格 + query = re.sub(r"[^\w\s\-\+\.]", " ", query) # 只保留字母数字、空格、连字符、加号、点号 + query = query.strip() + + # 限制查询长度 + if len(query) > 100: + query = query[:100] + + return query + + # === 具体抓取 === + def _search_arxiv( + self, query: str, size: int, max_results: int, with_abstract: bool + ) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + start = 0 + retry_count = 0 + max_retries = 3 + base_delay = 1.5 # 基础延迟时间 + + while len(results) < max_results: + # 在每次请求前添加延迟，避免过于频繁的请求 + if start > 0 or retry_count > 0: + time.sleep(base_delay) + + params = { + "searchtype": "all", + "query": query, + "abstracts": "show", + "order": "", + "size": str(size), + "start": str(start), + } + + try: + # 如果是重试，增加额外延迟 + if retry_count > 0: + time.sleep(2**retry_count) # 指数退避：2s, 4s, 8s + + resp = self.session.get(self.base_url, params=params, timeout=20) + resp.raise_for_status() + soup = BeautifulSoup(resp.content, "html.parser") + retry_count = 0 # 重置重试计数 + + except requests.exceptions.HTTPError as e: + if e.response.status_code in [500, 503, 429] and retry_count < max_retries: + logging.warning( + f"[arxiv_search] HTTP {e.response.status_code} error, retrying ({retry_count + 1}/{max_retries}) after delay..." + ) + retry_count += 1 + continue + else: + raise # 重试次数用完或其他错误 + + except Exception as e: + if retry_count < max_retries: + logging.warning( + f"[arxiv_search] Request failed, retrying ({retry_count + 1}/{max_retries}): {e}" + ) + retry_count += 1 + continue + else: + raise + papers = soup.find_all("li", class_="arxiv-result") # type: ignore + if not papers: + break + + for paper in papers: + if len(results) >= max_results: + break + + title_elem = paper.find("p", class_="title") # type: ignore + title = title_elem.text.strip() if title_elem else "No title" + + authors_elem = paper.find("p", class_="authors") # type: ignore + authors = authors_elem.text.strip() if authors_elem else "No authors" + authors = re.sub(r"^Authors:\s*", "", authors) + authors = re.sub(r"\s+", " ", authors).strip() + + abstract = "" + if with_abstract: + abstract_elem = paper.find("span", class_="abstract-full") # type: ignore + abstract = ( + (abstract_elem.text.strip() if abstract_elem else "") + .replace("△ Less", "") + .strip() + ) + + link_elem = paper.find("p", class_="list-title") # type: ignore + link_tag = link_elem.find("a") if isinstance(link_elem, Tag) else None # type: ignore + link = ( + link_tag["href"] + if isinstance(link_tag, Tag) and link_tag.has_attr("href") + else "" + ) + + results.append( + { + "title": title, + "authors": authors, + "abstract": abstract, + "link": link or "https://arxiv.org", + } + ) + + start += size + + return results[:max_results] diff --git a/tutorials/L3-libs/agents/basic_agent.py b/tutorials/L3-libs/agents/basic_agent.py new file mode 100644 index 0000000..b1b70ef --- /dev/null +++ b/tutorials/L3-libs/agents/basic_agent.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import importlib +import importlib.util +import json +import os +import sys +from typing import Any, Iterable + +from sage.cli.utils.env import get_api_key, load_environment_file, should_use_real_api +from sage.common.utils.config.loader import load_config +from sage.libs.agentic.agents.action.mcp_registry import MCPRegistry +from sage.libs.agentic.agents.planning.simple_llm_planner import SimpleLLMPlanner +from sage.libs.agentic.agents.profile.profile import BaseProfile +from sage.middleware.operators.agent.runtime import AgentRuntime +from sage.middleware.operators.rag import OpenAIGenerator + +# 加载环境配置 +try: + load_environment_file() +except RuntimeError as exc: + print(f"⚠️ 无法加载 .env: {exc}") + + +# ====== 读取 source ====== +def iter_queries(source_cfg: dict[str, Any]) -> Iterable[str]: + stype = source_cfg.get("type", "local") + if stype == "local": + path = source_cfg["data_path"] + field = source_cfg.get("field_query", "query") + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + obj = json.loads(line) + q = obj.get(field, "") + if isinstance(q, str) and q.strip(): + yield q + elif stype == "hf": + from datasets import load_dataset + + name = source_cfg["hf_dataset_name"] + config = source_cfg.get("hf_dataset_config") + split = source_cfg.get("hf_split", "dev") + field = source_cfg.get("field_query", "query") + ds = load_dataset(name, config, split=split) + for row in ds: + # HuggingFace dataset row is dict-like + if isinstance(row, dict): + q = row.get(field, "") + if isinstance(q, str) and q.strip(): + yield q + else: + raise ValueError(f"Unsupported source.type: {stype}") + + +def main(): + # ====== 读取配置 ====== + cfg_path = os.path.join(os.path.dirname(__file__), "config", "config_agent_min.yaml") + if not os.path.exists(cfg_path): + print(f"❌ Configuration file not found: {cfg_path}") + sys.exit(1) + config: dict[str, Any] = load_config(cfg_path) + + # ====== Profile ====== + profile = BaseProfile.from_dict(config["profile"]) + + # 检查是否在测试模式 + use_real_api = should_use_real_api() + test_mode = ( + os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true" + ) and not use_real_api # 如果明确要求使用真实API，则不进入测试模式 + + # 在真实API模式下，使用简化的查询数据以避免超时 + if use_real_api: + # 使用测试数据 + config["source"]["data_path"] = "examples/tutorials/agents/data/agent_queries_test.jsonl" + + # ====== Generator====== + gen_cfg = config["generator"]["remote"] # 可改为 "local"/"remote" + + # 验证 API key 配置（在测试和非测试模式下都需要检查） + try: + api_key = get_api_key("openai", required=True) + gen_cfg["api_key"] = api_key + if use_real_api: + print("🌐 Real API mode: API key configuration validated") + else: + print("✅ API key configuration validated") + except ValueError as e: + if test_mode: + print(f"⚠️ Test mode: {e}") + print("💡 Tip: Copy .env.template to .env and fill in your API keys") + print("✅ Test mode: API key validation completed (missing key is OK in test)") + else: + print(f"❌ {e}") + print("💡 Tip: Copy .env.template to .env and fill in your API keys") + sys.exit(1) + + if test_mode: + # 在测试模式下，验证配置加载和模块导入，但不实际初始化组件 + print( + "🧪 Test mode: Configuration loaded successfully (add --use-real-api to use real API)" + ) + print("✅ Test mode: Profile created successfully") + + # 验证配置文件结构 + required_sections = ["generator", "planner", "tools", "runtime"] + for section in required_sections: + if section in config: + print(f"✅ Test mode: {section} config found") + else: + print(f"❌ Test mode: {section} config missing") + + # 验证工具模块可以导入（但不实际初始化） + try: + for item in config.get("tools", []): + module_name = item["module"] + # 处理相对路径导入 + if module_name.startswith("examples.tutorials.agents."): + # 使用绝对路径导入 + tool_name = module_name.split(".")[-1] + tool_path = os.path.join(current_dir, f"{tool_name}.py") + spec = importlib.util.spec_from_file_location(tool_name, tool_path) + if spec is not None and spec.loader is not None: + mod = importlib.util.module_from_spec(spec) + sys.modules[tool_name] = mod + spec.loader.exec_module(mod) + else: + raise ImportError(f"Failed to load spec for {tool_name}") + else: + mod = importlib.import_module(module_name) + cls = getattr(mod, item["class"]) + print(f"✅ Test mode: Tool {item['class']} import successful") + except Exception as e: + print(f"⚠️ Test mode: Tool import failed (this is OK in test): {e}") + + print("✅ Test mode: Agent pipeline structure validated") + return + + if use_real_api: + print("🌐 Real API mode: Will make actual API calls with qwen-turbo") + + generator = OpenAIGenerator(gen_cfg) # ====== Planner ====== + planner_cfg = config["planner"] + planner = SimpleLLMPlanner( + generator=generator, + max_steps=planner_cfg.get("max_steps", 6), + enable_repair=planner_cfg.get("enable_repair", True), + topk_tools=planner_cfg.get("topk_tools", 6), + ) + + # ====== MCP 工具注册：按配置动态 import 并注册 ====== + registry = MCPRegistry() + for item in config.get("tools", []): + module_name = item["module"] + # 处理相对路径导入 + if module_name.startswith("examples.tutorials.agents."): + # 使用绝对路径导入 + tool_name = module_name.split(".")[-1] + tool_path = os.path.join(current_dir, f"{tool_name}.py") + spec = importlib.util.spec_from_file_location(tool_name, tool_path) + if spec is not None and spec.loader is not None: + mod = importlib.util.module_from_spec(spec) + sys.modules[tool_name] = mod + spec.loader.exec_module(mod) + else: + raise ImportError(f"Failed to load spec for {tool_name}") + else: + mod = importlib.import_module(module_name) + cls = getattr(mod, item["class"]) + kwargs = item.get("init_kwargs", {}) + registry.register(cls(**kwargs) if kwargs else cls()) + + # ====== Runtime ====== + runtime_cfg = config["runtime"] + agent = AgentRuntime( + profile=profile, + planner=planner, + tools=registry, + summarizer=(generator if runtime_cfg.get("summarizer") == "reuse_generator" else None), + # memory=None, # 如需接入 MemoryServiceAdapter，再按配置打开 + max_steps=runtime_cfg.get("max_steps", 6), + ) + + # ====== 跑一遍 queries====== + for q in iter_queries(config["source"]): + print("\n==========================") + print(f"🧑‍💻 User: {q}") + ans = agent.execute({"query": q}) + print(f"🤖 Agent:\n{ans}") + + +if __name__ == "__main__": + # 和 RAG 示例一致的“测试模式”友好输出 + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + try: + main() + print("\n✅ Test passed: Agent pipeline structure validated") + except Exception as e: + print(f"❌ Test failed: {e}") + sys.exit(1) + else: + main() diff --git a/tutorials/L3-libs/agents/config/README.md b/tutorials/L3-libs/agents/config/README.md new file mode 100644 index 0000000..aa5ed0e --- /dev/null +++ b/tutorials/L3-libs/agents/config/README.md @@ -0,0 +1,24 @@ +# Agent Tutorials - Configuration + +This directory contains configuration files for agent tutorials. + +## Files + +- **`config_agent_min.yaml`** - Minimal agent configuration + +## Usage + +Referenced by `../basic_agent.py`: + +```python +cfg_path = os.path.join(os.path.dirname(__file__), "config", "config_agent_min.yaml") +``` + +## Customization + +Modify the config file to change: + +- LLM model +- Agent tools +- Data sources +- Behavior parameters diff --git a/tutorials/L3-libs/agents/config/config_agent_min.yaml b/tutorials/L3-libs/agents/config/config_agent_min.yaml new file mode 100644 index 0000000..9b3466f --- /dev/null +++ b/tutorials/L3-libs/agents/config/config_agent_min.yaml @@ -0,0 +1,93 @@ +# examples/config/agent_planner_mcp.yaml + +pipeline: + name: "sage-agent-base-pipeline" + description: "Minimal agent pipeline(profile+planner+mcp+runtime)" + version: "1.0.0" + +# 数据源：每行 JSON 必须至少包含一个字段：{"query": "...用户问题..."} +source: + type: "local" + data_path: "examples/tutorials/agents/data/agent_queries.jsonl" + field_query: "query" # 例如： + # {"query": "在 arXiv 搜 2 篇 LLM agents survey，最后中文总结"} + +profile: + name: "ResearchOrchestrator" + role: "planner" + language: "zh" + goals: + - "以最少步骤完成用户意图" + - "优先使用提供的 MCP 工具" + constraints: + - "工具参数必须符合 JSONSchema" + - "计划步数不超过 6" + persona: + style: "concise" + +planner: + llm: + method: "openai" + model_name: "gpt-4o-mini" + base_url: "http://localhost:8000/v1" + api_key: "" + temperature: 0.2 + max_steps: 6 + enable_repair: true + topk_tools: 6 + +mcp: + # 本地示例工具（在脚本里实现并注册） + local_tools: + - name: "calculator" + enable: true + - name: "arxiv_search" + enable: true + # 可选：远程 MCP Server（若有） + remotes: [] + # remotes: + # - adapter_id: "t1" + # base_url: "http://localhost:9000" + # prefix: "up_" + +generator: + local: + method: "hf" + model_name: "meta-llama/Llama-2-13b-chat-hf" + seed: 42 + + vllm: + api_key: "" + method: "openai" + model_name: "meta-llama/Llama-2-7b-chat-hf" + base_url: "http://sage3:8000/v1" + seed: 42 + + remote: + api_key: "" + method: "openai" + model_name: "Qwen/Qwen2.5-7B-Instruct" + base_url: "http://127.0.0.1:8888/v1" + seed: 42 + +memory: + enable: false + session_id: "demo-session" + similarity_threshold: 0.2 + include_graph_context: false + create_knowledge_graph: false + +tools: +- module: "examples.tutorials.agents.arxiv_search_tool" + class: "ArxivSearchTool" + init_kwargs: {} + +runtime: + max_steps: 6 + summarizer: "reuse_generator" # 复用同一个 generator 做总结（AgentRuntime里直接传入同一个实例 + +sink: + platform: "local" + format: "json" + show_metadata: true + save_to_file: "results/agent_planner_mcp_output.jsonl" diff --git a/tutorials/L3-libs/agents/data/README.md b/tutorials/L3-libs/agents/data/README.md new file mode 100644 index 0000000..b60ff42 --- /dev/null +++ b/tutorials/L3-libs/agents/data/README.md @@ -0,0 +1,27 @@ +# Agent Tutorials - Data Files + +This directory contains sample data for agent tutorials. + +## Files + +- **`agent_queries.jsonl`** - Standard agent queries for testing +- **`agent_queries_test.jsonl`** - Test queries for agent examples + +## Format + +Each line in the JSONL file contains a query object: + +```json +{"query": "Your question here", "metadata": {...}} +``` + +## Usage + +These files are referenced in: + +- `../basic_agent.py` +- `../config/config_agent_min.yaml` + +## Custom Queries + +Add your own queries to test different agent behaviors. diff --git a/tutorials/L3-libs/agents/data/agent_queries.jsonl b/tutorials/L3-libs/agents/data/agent_queries.jsonl new file mode 100644 index 0000000..4de6e94 --- /dev/null +++ b/tutorials/L3-libs/agents/data/agent_queries.jsonl @@ -0,0 +1,4 @@ +{"query": "在 arXiv 搜 2 篇 LLM agents survey，最后中文总结"} +{"query": "帮我总结一下，GPT-4 和 Claude 2 在使用体验上有什么不同？"} +{"query": "帮我总结一下，GPT-4 和 Claude 2 在使用体验上有什么不同？再帮我推荐一个适合做 agent 的大模型"} +{"query": "帮我总结一下，GPT-4 和 Claude 2 在使用体验上有什么不同？再帮我推荐一个适合做 agent 的大模型。最后给出参考文献"} diff --git a/tutorials/L3-libs/agents/data/agent_queries_test.jsonl b/tutorials/L3-libs/agents/data/agent_queries_test.jsonl new file mode 100644 index 0000000..66d486c --- /dev/null +++ b/tutorials/L3-libs/agents/data/agent_queries_test.jsonl @@ -0,0 +1 @@ +{"query": "在 arXiv 搜 1 篇 LLM agents survey"} diff --git a/tutorials/L3-libs/agents/demo_arxiv_search.py b/tutorials/L3-libs/agents/demo_arxiv_search.py new file mode 100644 index 0000000..1963b68 --- /dev/null +++ b/tutorials/L3-libs/agents/demo_arxiv_search.py @@ -0,0 +1,179 @@ +""" +Demo: ArxivSearchTool Usage Examples + +This demo file shows practical examples of how to use the ArxivSearchTool +in different scenarios. These are educational demonstrations for developers +and users, not formal unit tests. + +For the actual tool implementation, see: arxiv_search_tool.py +For formal unit tests, see: packages/sage-libs/tests/lib/agents/test_arxiv_tool.py + +Examples included: +- Basic usage with different parameters +- Error handling and offline fallback +- Integration with MCP Registry +- Parameter variations and configurations + +@test:allow-demo +""" + +from unittest.mock import patch + +import pytest # noqa: F401 +from arxiv_search_tool import ArxivSearchTool + + +def example_basic_usage(): + """Example: Basic usage of ArxivSearchTool.""" + print("=== Basic ArxivSearchTool Usage Example ===") + + tool = ArxivSearchTool() + + # Mock the network call for demonstration + with patch.object(tool, "_search_arxiv") as mock_search: + mock_search.return_value = [ + { + "title": "Attention Is All You Need", + "authors": "Ashish Vaswani, Noam Shazeer, Niki Parmar", + "link": "https://arxiv.org/abs/1706.03762", + "abstract": "The dominant sequence transduction models...", + }, + { + "title": "BERT: Pre-training of Deep Bidirectional Transformers", + "authors": "Jacob Devlin, Ming-Wei Chang, Kenton Lee", + "link": "https://arxiv.org/abs/1810.04805", + "abstract": "We introduce a new language representation model...", + }, + ] + + # Example usage + result = tool.call( + {"query": "transformer attention", "max_results": 2, "with_abstract": True} + ) + + print(f"Query: {result['meta']['query']}") + print(f"Found {len(result['output'])} papers:") + for i, paper in enumerate(result["output"], 1): + print(f"\n{i}. {paper['title']}") + print(f" Authors: {paper['authors']}") + print(f" Link: {paper['link']}") + print(f" Abstract: {paper['abstract'][:100]}...") + + +def example_parameter_variations(): + """Example: Different parameter configurations.""" + print("\n=== Parameter Variations Example ===") + + tool = ArxivSearchTool() + + with patch.object(tool, "_search_arxiv") as mock_search: + mock_search.return_value = [ + { + "title": "Sample Paper", + "authors": "Sample Author", + "link": "https://arxiv.org/abs/1234.5678", + "abstract": "Sample abstract", + } + ] + + # Example 1: Minimal parameters + result1 = tool.call({"query": "machine learning"}) + print(f"Minimal call - max_results: {result1['meta']['max_results']}") + + # Example 2: Custom parameters + result2 = tool.call( + { + "query": "deep learning", + "max_results": 5, + "size": 50, + "with_abstract": False, + } + ) + print( + f"Custom call - max_results: {result2['meta']['max_results']}, size: {result2['meta']['size']}" + ) + + +def example_error_handling(): + """Example: Error handling and offline fallback.""" + print("\n=== Error Handling Example ===") + + tool = ArxivSearchTool() + + # Simulate network error + with patch.object(tool, "_search_arxiv") as mock_search: + mock_search.side_effect = Exception("Network error") + + result = tool.call({"query": "neural networks", "max_results": 3}) + + print("Network error occurred, using offline fallback:") + print(f"Offline mock: {result['meta'].get('offline_mock', False)}") + print(f"Results: {len(result['output'])} papers") + + +def example_mcp_integration(): + """Example: Integration with MCP Registry.""" + print("\n=== MCP Registry Integration Example ===") + + try: + from sage.libs.agentic.agents.action.mcp_registry import MCPRegistry + + # Create registry and register tool + registry = MCPRegistry() + tool = ArxivSearchTool() + registry.register(tool) + + # Mock the search for demonstration + with patch.object(tool, "_search_arxiv") as mock_search: + mock_search.return_value = [ + { + "title": "GPT-3 Paper", + "authors": "OpenAI Team", + "link": "https://arxiv.org/abs/example", + "abstract": "Language models are few-shot learners", + } + ] + + # Call through registry + result = registry.call( + "arxiv_search", {"query": "GPT language models", "max_results": 1} + ) + + print("Called through MCP Registry:") + print(f"Tool: {tool.name}") + print(f"Result: {result['output'][0]['title']}") + + except ImportError: + print("MCP Registry not available - skipping integration example") + + +def test_example_runs(): + """Test that all examples run without errors.""" + try: + example_basic_usage() + example_parameter_variations() + example_error_handling() + example_mcp_integration() + print("\n✅ All examples completed successfully!") + return True + except Exception as e: + print(f"\n❌ Example failed: {e}") + return False + + +if __name__ == "__main__": + print("ArxivSearchTool Examples") + print("=" * 40) + + success = test_example_runs() + + if success: + print("\nThese examples show how to:") + print("1. Use ArxivSearchTool with different parameters") + print("2. Handle network errors with offline fallback") + print("3. Integrate with MCP Registry") + print("4. Process and display results") + + print("\nFor more details, see the tool implementation in arxiv_search_tool.py") + else: + print("\nSome examples failed. Check the implementation and dependencies.") diff --git a/tutorials/L3-libs/agents/tool_use_pipeline.py b/tutorials/L3-libs/agents/tool_use_pipeline.py new file mode 100644 index 0000000..cc1a0b3 --- /dev/null +++ b/tutorials/L3-libs/agents/tool_use_pipeline.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +""" +Tool Use Agent Pipeline Demo +Agentic Workflow: User Query -> Select Tool(s) -> Use Tool(s) -> Generate Response + +This example demonstrates how to build an Agent Pipeline with tool calling capabilities: +1. Receive user queries +2. LLM reasoning to select appropriate tools +3. Execute selected tools (Web Search, Vector Search, Calculator, etc.) +4. Generate final response based on tool execution results + +Pipeline Architecture: + UserQuerySource -> ToolSelector -> ToolExecutor -> ResponseGenerator -> ResponseSink + +# test_tags: category=agent, timeout=180, requires_llm=true +""" + +from __future__ import annotations + +import json +import os +import re +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.kernel.api.local_environment import LocalEnvironment + +# ============================================================================= +# Data Models +# ============================================================================= + + +@dataclass +class ToolCallRequest: + """Tool call request""" + + tool_name: str + arguments: dict[str, Any] + + +@dataclass +class ToolCallResult: + """Tool call result""" + + tool_name: str + success: bool + result: Any + error: str | None = None + + +@dataclass +class AgentState: + """Agent state flowing through the pipeline""" + + query: str + selected_tools: list[ToolCallRequest] = field(default_factory=list) + tool_results: list[ToolCallResult] = field(default_factory=list) + response: str = "" + metadata: dict[str, Any] = field(default_factory=dict) + + +# ============================================================================= +# Tool Definitions - MCP Style +# ============================================================================= + + +class BaseTool(ABC): + """Base tool class - MCP style""" + + name: str = "" + description: str = "" + input_schema: dict[str, Any] = {} + + @abstractmethod + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + """Execute the tool""" + pass + + +class WebSearchTool(BaseTool): + """Web search tool - simulates search engine""" + + name = "web_search" + description = "Search the web for information. Returns relevant search results." + input_schema = { + "type": "object", + "properties": { + "query": {"type": "string", "description": "The search query"}, + "max_results": {"type": "integer", "default": 5}, + }, + "required": ["query"], + } + + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + query = arguments.get("query", "") + max_results = arguments.get("max_results", 5) + mock_results = [ + { + "title": f"Result {i + 1} for: {query}", + "url": f"https://example.com/result{i + 1}", + "snippet": f"This is a relevant snippet about {query}...", + } + for i in range(min(max_results, 5)) + ] + return {"success": True, "results": mock_results, "query": query} + + +class VectorSearchTool(BaseTool): + """Vector search tool - RAG retrieval""" + + name = "vector_search" + description = "Search internal knowledge base using vector similarity." + input_schema = { + "type": "object", + "properties": { + "query": {"type": "string", "description": "The search query"}, + "top_k": {"type": "integer", "default": 3}, + }, + "required": ["query"], + } + + def __init__(self): + self.knowledge_base = [ + { + "id": "doc1", + "content": "SAGE is a Python framework for building AI/LLM data processing pipelines.", + }, + { + "id": "doc2", + "content": "The architecture consists of 6 layers: L1-Common to L6-Interface.", + }, + { + "id": "doc3", + "content": "To install SAGE, run ./quickstart.sh --dev --yes for development setup.", + }, + ] + + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + query = arguments.get("query", "").lower() + top_k = arguments.get("top_k", 3) + scored_docs = [] + for doc in self.knowledge_base: + content_lower = doc["content"].lower() + score = sum(1 for word in query.split() if word in content_lower) + if score > 0: + scored_docs.append({"doc": doc, "score": score}) + scored_docs.sort(key=lambda x: x["score"], reverse=True) + return { + "success": True, + "documents": [ + {"id": d["doc"]["id"], "content": d["doc"]["content"], "score": d["score"]} + for d in scored_docs[:top_k] + ], + "query": query, + } + + +class CalculatorTool(BaseTool): + """Calculator tool""" + + name = "calculator" + description = "Perform mathematical calculations." + input_schema = { + "type": "object", + "properties": {"expression": {"type": "string", "description": "Math expression"}}, + "required": ["expression"], + } + + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + expression = arguments.get("expression", "") + try: + safe_expr = re.sub(r"[^0-9+\-*/(). ]", "", expression) + if not safe_expr: + return {"success": False, "error": "Invalid expression"} + result = eval(safe_expr, {"__builtins__": {}}, {}) + return {"success": True, "expression": expression, "result": result} + except Exception as e: + return {"success": False, "expression": expression, "error": str(e)} + + +class EmailSearchTool(BaseTool): + """Email search tool""" + + name = "email_search" + description = "Search emails by sender, subject, or content." + input_schema = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + } + + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + query = arguments.get("query", "") + mock_emails = [ + { + "id": "email1", + "from": "team@company.com", + "subject": f"RE: {query}", + "snippet": f"Info about {query}...", + }, + { + "id": "email2", + "from": "support@example.com", + "subject": f"Update on {query}", + "snippet": f"Review of {query}...", + }, + ] + return {"success": True, "emails": mock_emails, "query": query} + + +class SlackSearchTool(BaseTool): + """Slack message search tool""" + + name = "slack_search" + description = "Search Slack messages and channels." + input_schema = { + "type": "object", + "properties": {"query": {"type": "string"}, "channel": {"type": "string"}}, + "required": ["query"], + } + + def call(self, arguments: dict[str, Any]) -> dict[str, Any]: + query = arguments.get("query", "") + channel = arguments.get("channel", "general") + mock_messages = [ + {"channel": channel, "user": "alice", "message": f"Discussing {query} in the meeting."}, + {"channel": channel, "user": "bob", "message": f"Good point about {query}."}, + ] + return {"success": True, "messages": mock_messages, "query": query} + + +# ============================================================================= +# Tool Registry +# ============================================================================= + + +class ToolRegistry: + """Tool registry - manages all available tools""" + + def __init__(self): + self._tools: dict[str, BaseTool] = {} + + def register(self, tool: BaseTool) -> None: + self._tools[tool.name] = tool + + def get(self, name: str) -> BaseTool | None: + return self._tools.get(name) + + def list_tools(self) -> list[str]: + return list(self._tools.keys()) + + def describe_tools(self) -> list[dict[str, Any]]: + return [ + {"name": tool.name, "description": tool.description, "input_schema": tool.input_schema} + for tool in self._tools.values() + ] + + def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]: + tool = self._tools.get(name) + if not tool: + return {"success": False, "error": f"Tool '{name}' not found"} + return tool.call(arguments) + + +def create_default_registry() -> ToolRegistry: + """Create default tool registry""" + registry = ToolRegistry() + registry.register(WebSearchTool()) + registry.register(VectorSearchTool()) + registry.register(CalculatorTool()) + registry.register(EmailSearchTool()) + registry.register(SlackSearchTool()) + return registry + + +# ============================================================================= +# Pipeline Operators +# ============================================================================= + + +class UserQuerySource(SourceFunction): + """User query source - receives user queries and creates AgentState""" + + def __init__(self, queries: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.queries = queries or [ + "What is SAGE framework and how to install it?", + "Calculate 15 * 23 + 47", + "Search for recent emails about project update", + ] + self.current_index = 0 + + def execute(self, data=None) -> AgentState | None: + if self.current_index >= len(self.queries): + from sage.kernel.runtime.communication.packet import StopSignal + + return StopSignal("All queries processed") + + query = self.queries[self.current_index] + self.current_index += 1 + print(f"\n{'=' * 60}") + print(f"[UserQuerySource] Query {self.current_index}: {query}") + print("=" * 60) + return AgentState( + query=query, metadata={"query_id": self.current_index, "timestamp": time.time()} + ) + + +class ToolSelector(MapFunction): + """Tool selector - uses LLM or fallback to select appropriate tools""" + + def __init__(self, tool_registry: ToolRegistry | None = None, **kwargs): + super().__init__(**kwargs) + self.tool_registry = tool_registry or create_default_registry() + self._llm_client = None + + def _get_llm_client(self): + if self._llm_client is None: + try: + from sage.common.components.sage_llm import UnifiedInferenceClient + + self._llm_client = UnifiedInferenceClient.create() + except Exception as e: + print(f"[ToolSelector] Warning: Could not create LLM client: {e}") + self._llm_client = None + return self._llm_client + + def _build_tool_selection_prompt(self, query: str) -> str: + tools_desc = self.tool_registry.describe_tools() + tools_json = json.dumps(tools_desc, indent=2, ensure_ascii=False) + return f"""You are an AI assistant that helps select the right tools. + +Available Tools: +{tools_json} + +User Query: {query} + +Select appropriate tool(s). Return JSON array: +[{{"tool_name": "name", "arguments": {{"arg": "value"}}}}] + +If no tools needed, return: [] + +Your response (JSON only):""" + + def _parse_tool_selection(self, response: str) -> list[ToolCallRequest]: + try: + json_match = re.search(r"\[[\s\S]*\]", response) + if json_match: + tools_data = json.loads(json_match.group()) + return [ + ToolCallRequest( + tool_name=t.get("tool_name", t.get("name", "")), + arguments=t.get("arguments", {}), + ) + for t in tools_data + if t.get("tool_name") or t.get("name") + ] + except Exception as e: + print(f"[ToolSelector] Failed to parse response: {e}") + return [] + + def _fallback_tool_selection(self, query: str) -> list[ToolCallRequest]: + query_lower = query.lower() + selected_tools = [] + + if any(word in query_lower for word in ["calculate", "math", "+", "-", "*", "/"]): + expr_match = re.search(r"[\d\s+\-*/().]+", query) + expr = expr_match.group().strip() if expr_match else query + selected_tools.append( + ToolCallRequest(tool_name="calculator", arguments={"expression": expr}) + ) + elif any(word in query_lower for word in ["email", "mail"]): + selected_tools.append( + ToolCallRequest(tool_name="email_search", arguments={"query": query}) + ) + elif any(word in query_lower for word in ["slack", "message"]): + selected_tools.append( + ToolCallRequest(tool_name="slack_search", arguments={"query": query}) + ) + elif any(word in query_lower for word in ["what is", "how to", "explain", "sage"]): + selected_tools.append( + ToolCallRequest(tool_name="vector_search", arguments={"query": query}) + ) + else: + selected_tools.append( + ToolCallRequest(tool_name="web_search", arguments={"query": query}) + ) + + return selected_tools + + def execute(self, data: AgentState) -> AgentState: + if not isinstance(data, AgentState): + return data + + print(f"\n[ToolSelector] Analyzing query: {data.query}") + + llm_client = self._get_llm_client() + if llm_client: + try: + prompt = self._build_tool_selection_prompt(data.query) + response = llm_client.chat(prompt) + selected_tools = self._parse_tool_selection(response) + if selected_tools: + data.selected_tools = selected_tools + print(f"[ToolSelector] LLM selected: {[t.tool_name for t in selected_tools]}") + return data + except Exception as e: + print(f"[ToolSelector] LLM call failed: {e}") + + print("[ToolSelector] Using fallback keyword matching...") + data.selected_tools = self._fallback_tool_selection(data.query) + print(f"[ToolSelector] Fallback selected: {[t.tool_name for t in data.selected_tools]}") + return data + + +class ToolExecutor(MapFunction): + """Tool executor - executes selected tools and collects results""" + + def __init__(self, tool_registry: ToolRegistry | None = None, **kwargs): + super().__init__(**kwargs) + self.tool_registry = tool_registry or create_default_registry() + + def execute(self, data: AgentState) -> AgentState: + if not isinstance(data, AgentState): + return data + + print(f"\n[ToolExecutor] Executing {len(data.selected_tools)} tool(s)...") + + for tool_request in data.selected_tools: + print(f" -> Calling: {tool_request.tool_name}") + print(f" Arguments: {tool_request.arguments}") + try: + result = self.tool_registry.call_tool( + tool_request.tool_name, tool_request.arguments + ) + tool_result = ToolCallResult( + tool_name=tool_request.tool_name, + success=result.get("success", True), + result=result, + ) + print(" Result: Success") + except Exception as e: + tool_result = ToolCallResult( + tool_name=tool_request.tool_name, success=False, result=None, error=str(e) + ) + print(f" Result: Error - {e}") + data.tool_results.append(tool_result) + + return data + + +class ResponseGenerator(MapFunction): + """Response generator - generates final response based on tool results""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._llm_client = None + + def _get_llm_client(self): + if self._llm_client is None: + try: + from sage.common.components.sage_llm import UnifiedInferenceClient + + self._llm_client = UnifiedInferenceClient.create() + except Exception: + self._llm_client = None + return self._llm_client + + def _build_response_prompt(self, state: AgentState) -> str: + results_text = "" + for tr in state.tool_results: + if tr.success: + results_text += f"\n### {tr.tool_name} Result:\n" + results_text += json.dumps(tr.result, indent=2, ensure_ascii=False) + else: + results_text += f"\n### {tr.tool_name} Error:\n{tr.error}" + return f"""Based on the tool results, generate a helpful response. + +User Query: {state.query} + +Tool Results: +{results_text} + +Provide a clear, concise response:""" + + def _generate_fallback_response(self, state: AgentState) -> str: + response_parts = [f"Query: {state.query}\n"] + for tr in state.tool_results: + response_parts.append(f"\n[{tr.tool_name}]:") + if tr.success and isinstance(tr.result, dict): + if "results" in tr.result: + for i, r in enumerate(tr.result["results"][:3], 1): + response_parts.append(f" {i}. {r.get('title', str(r))}") + elif "documents" in tr.result: + for doc in tr.result["documents"][:3]: + content = doc.get("content", "")[:100] + response_parts.append(f" - {content}...") + elif "result" in tr.result: + response_parts.append(f" Result: {tr.result['result']}") + elif "emails" in tr.result: + for email in tr.result["emails"][:2]: + response_parts.append(f" - {email.get('subject', '')}") + elif "messages" in tr.result: + for msg in tr.result["messages"][:2]: + response_parts.append( + f" - @{msg.get('user', '')}: {msg.get('message', '')}" + ) + else: + response_parts.append(f" {json.dumps(tr.result, ensure_ascii=False)[:200]}") + elif not tr.success: + response_parts.append(f" Error: {tr.error}") + return "\n".join(response_parts) + + def execute(self, data: AgentState) -> AgentState: + if not isinstance(data, AgentState): + return data + + print("\n[ResponseGenerator] Generating response...") + + llm_client = self._get_llm_client() + if llm_client and data.tool_results: + try: + prompt = self._build_response_prompt(data) + response = llm_client.chat(prompt) + data.response = response + print("[ResponseGenerator] LLM response generated.") + return data + except Exception as e: + print(f"[ResponseGenerator] LLM call failed: {e}") + + print("[ResponseGenerator] Using fallback response generation...") + data.response = self._generate_fallback_response(data) + return data + + +class ResponseSink(SinkFunction): + """Response output - outputs final response to console""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.test_mode = os.getenv("SAGE_TEST_MODE") == "true" + + def execute(self, data: AgentState) -> None: + if not isinstance(data, AgentState): + print(f"[ResponseSink] Received non-AgentState data: {type(data)}") + return + + print("\n" + "=" * 60) + print(f"[Final Response] Query: {data.query}") + print("-" * 60) + if data.selected_tools: + print(f"Tools Used: {', '.join(t.tool_name for t in data.selected_tools)}") + print("-" * 60) + + response = data.response + if self.test_mode and len(response) > 500: + response = response[:500] + "... (truncated)" + print(response) + print("=" * 60) + + +# ============================================================================= +# Pipeline Build and Run +# ============================================================================= + + +def run_tool_use_pipeline(queries: list[str] | None = None): + """Run Tool Use Agent Pipeline""" + print(""" +======================================================== + Tool Use Agent Pipeline Demo +======================================================== + Pipeline: UserQuery -> ToolSelector -> ToolExecutor + -> ResponseGenerator -> ResponseSink + + Available Tools: + - web_search: Search the web + - vector_search: Search knowledge base (RAG) + - calculator: Mathematical calculations + - email_search: Search emails + - slack_search: Search Slack messages +======================================================== + """) + + tool_registry = create_default_registry() + print(f"Registered tools: {tool_registry.list_tools()}\n") + + env = LocalEnvironment() + ( + env.from_source(UserQuerySource, queries=queries) + .map(ToolSelector, tool_registry=tool_registry) + .map(ToolExecutor, tool_registry=tool_registry) + .map(ResponseGenerator) + .sink(ResponseSink) + ) + + start_time = time.time() + env.submit(autostop=True) + total_time = time.time() - start_time + + print(f"\nPipeline completed in {total_time:.2f} seconds") + env.close() + + +def run_interactive_mode(): + """Interactive mode - user can input queries in real-time""" + print(""" +======================================================== + Tool Use Agent - Interactive Mode + Type your query and press Enter. + Type 'quit' or 'exit' to stop. +======================================================== + """) + + tool_registry = create_default_registry() + print(f"Available tools: {tool_registry.list_tools()}\n") + + while True: + try: + query = input("\n> Your query: ").strip() + if not query: + continue + if query.lower() in ("quit", "exit"): + print("Goodbye.") + break + + state = AgentState(query=query) + selector = ToolSelector(tool_registry=tool_registry) + state = selector.execute(state) + executor = ToolExecutor(tool_registry=tool_registry) + state = executor.execute(state) + generator = ResponseGenerator() + state = generator.execute(state) + sink = ResponseSink() + sink.execute(state) + except KeyboardInterrupt: + print("\nInterrupted. Goodbye.") + break + except Exception as e: + print(f"Error: {e}") + + +def main(): + """Main entry point""" + import sys + + test_mode = os.getenv("SAGE_TEST_MODE") == "true" + + if len(sys.argv) > 1: + if sys.argv[1] == "--interactive": + run_interactive_mode() + return + elif sys.argv[1] == "--query": + queries = sys.argv[2:] if len(sys.argv) > 2 else None + run_tool_use_pipeline(queries) + return + + example_queries = [ + "What is SAGE framework and how to install it?", + "Calculate 15 * 23 + 47", + "Search for recent emails about project update", + ] + + if test_mode: + example_queries = example_queries[:1] + + run_tool_use_pipeline(example_queries) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/agents/workflow_demo.py b/tutorials/L3-libs/agents/workflow_demo.py new file mode 100644 index 0000000..0a32ef9 --- /dev/null +++ b/tutorials/L3-libs/agents/workflow_demo.py @@ -0,0 +1,260 @@ +""" +Agent Workflow Demo + +This demonstrates how the agent pipeline works with real examples. +This is an educational demonstration, not a formal test. + +Usage: + python agent_workflow_demo.py + +This demo shows: +- How to read queries from different sources +- Complete agent workflow with mocked components +- Integration examples for learning purposes + +@test:allow-demo +""" + +import json +import os +import tempfile +from unittest.mock import Mock + + +def create_test_queries_file(): + """Create a temporary test queries file.""" + test_queries = [ + {"query": "在 arXiv 搜索关于 transformer 的论文"}, + {"query": "帮我找一些深度学习的最新研究"}, + {"query": "总结一下注意力机制的发展历程"}, + ] + + temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) + for query in test_queries: + temp_file.write(json.dumps(query, ensure_ascii=False) + "\n") + temp_file.close() + + return temp_file.name + + +def example_iter_queries(): + """Example: How to read queries from different sources.""" + print("=== Query Reading Examples ===") + + # Import the agent module using importlib to handle path with dashes + try: + import importlib.util + import sys + from pathlib import Path + + # Get the path to basic_agent.py + basic_agent_path = Path(__file__).parent / "basic_agent.py" + spec = importlib.util.spec_from_file_location("basic_agent", basic_agent_path) + if spec is not None and spec.loader is not None: + basic_agent = importlib.util.module_from_spec(spec) + sys.modules["basic_agent"] = basic_agent + spec.loader.exec_module(basic_agent) + iter_queries = basic_agent.iter_queries + else: + raise ImportError("Failed to load basic_agent module") + + # Example 1: Local JSONL file + temp_file = create_test_queries_file() + try: + source_config = { + "type": "local", + "data_path": temp_file, + "field_query": "query", + } + + print("Reading from local JSONL file:") + queries = list(iter_queries(source_config)) + for i, query in enumerate(queries, 1): + print(f" {i}. {query}") + + finally: + os.unlink(temp_file) + + print(f"\nTotal queries loaded: {len(queries)}") + + except ImportError as e: + print(f"Could not import agent module: {e}") + + +def example_mock_agent_workflow(): + """Example: Complete agent workflow with mocks.""" + print("\n=== Mock Agent Workflow Example ===") + + try: + # Mock all the components + print("Setting up mock components...") + + # Mock generator that creates plans + mock_generator = Mock() + + def mock_execute(data): + user_query = data[0] + if "arxiv" in user_query.lower() or "论文" in user_query: + plan = [ + { + "type": "tool", + "name": "arxiv_search", + "arguments": {"query": "transformer", "max_results": 2}, + }, + {"type": "reply", "text": "已为您找到相关论文"}, + ] + else: + plan = [{"type": "reply", "text": "我理解您的问题，正在思考..."}] + return (user_query, json.dumps(plan, ensure_ascii=False)) + + mock_generator.execute = mock_execute + + # Mock ArxivSearchTool + mock_tool = Mock() + mock_tool.name = "arxiv_search" + mock_tool.description = "Search arXiv papers" + + def mock_call(arguments): + return { + "output": [ + { + "title": "Attention Is All You Need", + "authors": "Vaswani et al.", + "link": "https://arxiv.org/abs/1706.03762", + "abstract": "The dominant sequence transduction models...", + } + ], + "meta": arguments, + } + + mock_tool.call = mock_call + + # Simulate the agent workflow + test_query = "在 arXiv 搜索关于 transformer 的论文" + print(f"\nProcessing query: {test_query}") + + # Step 1: Generate plan + print("1. Generating plan...") + _, plan_json = mock_generator.execute([test_query, "system prompt"]) + plan = json.loads(plan_json) + print(f" Plan: {len(plan)} steps") + for i, step in enumerate(plan, 1): + print(f" Step {i}: {step['type']}") + + # Step 2: Execute tools + print("2. Executing tools...") + observations = [] + for step in plan: + if step["type"] == "tool": + result = mock_tool.call(step["arguments"]) + observations.append({"tool": step["name"], "result": result, "success": True}) + print(f" Tool {step['name']}: Found {len(result['output'])} results") + + # Step 3: Generate response + print("3. Generating response...") + if any(step["type"] == "reply" for step in plan): + reply = next(step["text"] for step in plan if step["type"] == "reply") + else: + reply = f"基于工具执行结果，为您找到了 {len(observations)} 个相关资源" + + print(f" Response: {reply}") + + print("\n✅ Workflow completed successfully!") + return True + + except Exception as e: + print(f"❌ Workflow failed: {e}") + return False + + +def example_configuration_usage(): + """Example: How configuration is used in the agent.""" + print("\n=== Configuration Usage Example ===") + + # Example configuration structure + example_config = { + "profile": { + "name": "ResearchAssistant", + "role": "助手", + "language": "zh", + "goals": ["帮助用户搜索和分析学术论文"], + "constraints": ["使用提供的工具", "提供准确信息"], + }, + "tools": [ + { + "module": "examples.agents.tools.arxiv_search_tool", + "class": "ArxivSearchTool", + "init_kwargs": {}, + } + ], + "generator": {"remote": {"method": "openai", "model_name": "gpt-3.5-turbo"}}, + "planner": {"max_steps": 5, "enable_repair": True}, + "runtime": {"max_steps": 5, "summarizer": "reuse_generator"}, + } + + print("Example configuration structure:") + for section, content in example_config.items(): + print(f" {section}: {type(content).__name__}") + if isinstance(content, dict): + for key in content.keys(): + print(f" - {key}") + elif isinstance(content, list) and content: + print(f" - {len(content)} items") + + print("\nThis configuration defines:") + print("- Agent profile and personality") + print("- Available tools and their setup") + print("- Generator settings for LLM") + print("- Planning and runtime parameters") + + +def test_all_examples(): + """Run all examples and report results.""" + print("Agent Workflow Examples") + print("=" * 50) + + examples = [ + ("Query Reading", example_iter_queries), + ("Mock Workflow", example_mock_agent_workflow), + ("Configuration", example_configuration_usage), + ] + + results = [] + for name, example_func in examples: + try: + print(f"\n--- {name} ---") + result = example_func() + if result is None: # For examples that don't return boolean + result = True + results.append((name, result)) + except Exception as e: + print(f"❌ {name} failed: {e}") + results.append((name, False)) + + print("\n" + "=" * 50) + print("Example Results:") + for name, success in results: + status = "✅" if success else "❌" + print(f" {status} {name}") + + total_success = sum(1 for _, success in results if success) + print(f"\nCompleted: {total_success}/{len(results)} examples") + + return total_success == len(results) + + +if __name__ == "__main__": + success = test_all_examples() + + if success: + print("\n🎉 All examples completed successfully!") + print("\nThese examples demonstrate:") + print("- How to read queries from different sources") + print("- Complete agent workflow with mocking") + print("- Configuration structure and usage") + print("- Integration between components") + else: + print("\n⚠️ Some examples had issues. Check the implementation.") + + print("\nFor the real agent implementation, see: agent.py") + print("For configuration examples, see: config/config_agent_min.yaml") diff --git a/tutorials/L3-libs/amms_example.py b/tutorials/L3-libs/amms_example.py new file mode 100644 index 0000000..475b1ae --- /dev/null +++ b/tutorials/L3-libs/amms_example.py @@ -0,0 +1,116 @@ +"""Example: Using the AMMS unified interface. + +This example demonstrates how to use the refactored AMMS interface +to perform approximate matrix multiplication. +""" + +from sage.libs.amms import registered + + +def example_basic_usage(): + """Basic usage of AMMS interface.""" + print("=== Basic AMMS Usage ===\n") + + # Check available algorithms + available = registered() + print(f"Available algorithms: {available}\n") + + # Note: Actual algorithm implementations need to be registered first + # This is a template showing how to use the interface + + # Example: Create a CountSketch AMM instance (once implemented) + # amm = create("countsketch", sketch_size=1000) + + # Setup with configuration + # config = { + # "sketch_size": 1000, + # "hash_functions": 5, + # "use_gpu": False + # } + # amm.setup(config) + + # Create sample matrices + # matrix_a = np.random.randn(100, 50) + # matrix_b = np.random.randn(50, 80) + + # Perform approximate multiplication + # result = amm.multiply(matrix_a, matrix_b) + + # Compare with exact result + # exact = matrix_a @ matrix_b + # error = np.linalg.norm(result - exact) / np.linalg.norm(exact) + # print(f"Relative error: {error:.4f}") + + print("Note: Algorithm implementations need to be registered first.") + print("See implementations/ for C++ algorithm code.") + + +def example_streaming_amm(): + """Example of streaming AMM (for algorithms that support it).""" + print("\n=== Streaming AMM Example ===\n") + + # For streaming algorithms, you can update matrices incrementally + # streaming_amm = create("streaming_countsketch", sketch_size=1000) + + # Initial setup + # streaming_amm.setup({"sketch_size": 1000}) + + # Update rows incrementally + # for i in range(100): + # row_data = np.random.randn(50) + # streaming_amm.update_row("A", i, row_data) + + # Get current result + # result = streaming_amm.get_current_result() + + print("Note: Streaming AMM requires StreamingAmmIndex implementation.") + + +def example_batch_processing(): + """Example of batch matrix multiplication.""" + print("\n=== Batch Processing Example ===\n") + + # Batch processing multiple matrix pairs + # amm = create("fastjlt", sketch_size=500) + + # matrices_a = [np.random.randn(100, 50) for _ in range(10)] + # matrices_b = [np.random.randn(50, 80) for _ in range(10)] + + # results = amm.batch_multiply(matrices_a, matrices_b) + + print("Note: Batch processing uses default implementation.") + print("Override batch_multiply() for optimized batch processing.") + + +def example_algorithm_metadata(): + """Example of accessing algorithm metadata.""" + print("\n=== Algorithm Metadata Example ===\n") + + # from sage.libs.amms import get_meta + + # meta = get_meta("countsketch") + # if meta: + # print(f"Algorithm: {meta.name}") + # print(f"Type: {meta.algorithm_type}") + # print(f"Supports streaming: {meta.supports_streaming}") + # print(f"Supports GPU: {meta.supports_gpu}") + # print(f"Requires training: {meta.requires_training}") + + print("Note: Metadata is registered when algorithms are registered.") + + +if __name__ == "__main__": + print("AMMS Interface Examples") + print("=" * 50) + + example_basic_usage() + example_streaming_amm() + example_batch_processing() + example_algorithm_metadata() + + print("\n" + "=" * 50) + print("To use these examples:") + print("1. Implement algorithms in implementations/") + print("2. Create wrappers in wrappers/") + print("3. Register algorithms using register()") + print("4. Then run these examples") diff --git a/tutorials/L3-libs/embeddings/README.md b/tutorials/L3-libs/embeddings/README.md new file mode 100644 index 0000000..ca35919 --- /dev/null +++ b/tutorials/L3-libs/embeddings/README.md @@ -0,0 +1,26 @@ +# Multimodal Tutorials + +Examples demonstrating multimodal AI capabilities (text, image, video). + +## Examples + +### 1. Quickstart (`quickstart.py`) + +Simple text-image multimodal example. + +```bash +python examples/tutorials/multimodal/quickstart.py +``` + +### 2. Cross-Modal Search (`cross_modal_search.py`) + +Search images using text queries and vice versa. + +```bash +python examples/tutorials/multimodal/cross_modal_search.py +``` + +## Next Steps + +- See production multimodal applications in `packages/sage-apps/` +- Check video intelligence app: `examples/apps/run_video_intelligence.py` diff --git a/tutorials/L3-libs/embeddings/cross_modal_search.py b/tutorials/L3-libs/embeddings/cross_modal_search.py new file mode 100644 index 0000000..9c073d3 --- /dev/null +++ b/tutorials/L3-libs/embeddings/cross_modal_search.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Cross-modal retrieval demo highlighting fusion vs. single-modality ranking. + +Requirements: + pip install isage-middleware>=0.2.0 +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np + +from sage.middleware.components.sage_db.python.multimodal_sage_db import ( + FusionParams, + FusionStrategy, + ModalityType, + MultimodalSearchParams, + create_text_image_db, +) + + +def seeded_embedding(seed: int, dim: int) -> np.ndarray: + rng = np.random.default_rng(seed) + vec = rng.normal(size=dim).astype("float32") + return vec / (np.linalg.norm(vec) + 1e-12) + + +def sample_collection() -> dict[str, tuple[np.ndarray, np.ndarray]]: + return { + "sunrise-cliffs": (seeded_embedding(21, 768), seeded_embedding(22, 512)), + "rooftop-dusk": (seeded_embedding(23, 768), seeded_embedding(24, 512)), + "forest-trail": (seeded_embedding(25, 768), seeded_embedding(26, 512)), + "city-rain": (seeded_embedding(27, 768), seeded_embedding(28, 512)), + "gallery-exhibit": (seeded_embedding(29, 768), seeded_embedding(30, 512)), + } + + +def load_demo_db(): + db = create_text_image_db(dimension=512) + for name, (text_emb, image_emb) in sample_collection().items(): + db.add_from_embeddings( + { + ModalityType.TEXT: text_emb, + ModalityType.IMAGE: image_emb, + }, + {"label": name, "collection": "demo"}, + ) + return db + + +def describe_results(title: str, results) -> None: + print(f"\n{title}") + for idx, result in enumerate(results, start=1): + print(f" {idx}. id={result.id:>2} score={result.score:.4f} metadata={result.metadata}") + + +def main() -> None: + print("=== Cross-modal search walkthrough ===") + + db = load_demo_db() + backend = "C++ accelerated" if getattr(db, "_db", None) is not None else "Python mock" + print("Backend:", backend) + + query_text = seeded_embedding(31, 768) + query_image = seeded_embedding(32, 512) + + fused_params = MultimodalSearchParams(k=3) + fused_params.query_fusion_params.strategy = FusionStrategy.WEIGHTED_AVERAGE + + fusion_results = db.search_multimodal( + {ModalityType.TEXT: query_text, ModalityType.IMAGE: query_image}, fused_params + ) + describe_results("🔗 Fusion search (text + image)", fusion_results) + + cross_params = MultimodalSearchParams(k=3) + cross_params.use_cross_modal_search = True + cross_params.target_modalities = [ModalityType.IMAGE] + cross_results = db.cross_modal_search( + ModalityType.TEXT, + query_text, + target_modalities=[ModalityType.IMAGE], + params=cross_params, + ) + describe_results("🧭 Text → image cross search", cross_results) + + print("\n🎛️ Re-weighting modalities (text emphasis)") + tuned = FusionParams(strategy=FusionStrategy.WEIGHTED_AVERAGE) + tuned.modality_weights[ModalityType.TEXT] = 0.7 + tuned.modality_weights[ModalityType.IMAGE] = 0.3 + db.update_fusion_params(tuned) + + adjusted_results = db.search_multimodal( + {ModalityType.TEXT: query_text, ModalityType.IMAGE: query_image}, fused_params + ) + describe_results("🎯 Fusion search after weight tweak", adjusted_results) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/embeddings/embedding_demo.py b/tutorials/L3-libs/embeddings/embedding_demo.py new file mode 100644 index 0000000..7293400 --- /dev/null +++ b/tutorials/L3-libs/embeddings/embedding_demo.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +SAGE Embedding 方法演示 + +展示所有 11 个 embedding 方法的使用。 + +@test:allow-demo +""" + +from sage.common.components.sage_embedding import ( + check_model_availability, + get_embedding_model, + list_embedding_models, +) + + +def print_separator(title=""): + """打印分隔线""" + if title: + print(f"\n{'=' * 60}") + print(f" {title}") + print(f"{'=' * 60}") + else: + print(f"{'=' * 60}") + + +def list_all_methods(): + """列出所有可用的 embedding 方法""" + print_separator("所有 Embedding 方法") + + models = list_embedding_models() + + for method, info in models.items(): + print(f"\n📦 {method} - {info['display_name']}") + print(f" 描述: {info['description']}") + + features = [] + if info["requires_api_key"]: + features.append("🔑 需要 API Key") + else: + features.append("🔓 无需 API Key") + + if info["requires_download"]: + features.append("📥 需要下载模型") + else: + features.append("☁️ 云端服务") + + if info["default_dimension"]: + features.append(f"📊 默认维度: {info['default_dimension']}") + + print(f" 特性: {', '.join(features)}") + + if info["examples"]: + print(f" 示例: {', '.join(info['examples'][:3])}") + + +def check_all_status(): + """检查所有方法的可用性""" + print_separator("方法可用性检查") + + methods = [ + "hash", + "mockembedder", + "hf", + "openai", + "jina", + "zhipu", + "cohere", + "bedrock", + "ollama", + "siliconcloud", + "nvidia_openai", + ] + + for method in methods: + result = check_model_availability(method) + status_icon = { + "available": "✅", + "cached": "✅", + "needs_api_key": "⚠️", + "needs_download": "⚠️", + "unavailable": "❌", + }.get(result["status"], "❓") + + print(f"{status_icon} {method:20} - {result['message']}") + + +def demo_no_api_key_methods(): + """演示无需 API Key 的方法""" + print_separator("演示：无需 API Key 的方法") + + # Hash Embedding + print("\n1. Hash Embedding (轻量级)") + try: + emb = get_embedding_model("hash", dim=384) + vec = emb.embed("Hello World") + print(f" {emb}") + print(f" 向量维度: {len(vec)}") + print(f" 向量示例: {vec[:5]}...") + except Exception as e: + print(f" ❌ 错误: {e}") + + # Mock Embedding + print("\n2. Mock Embedding (测试用)") + try: + emb = get_embedding_model("mockembedder", dim=128) + vec = emb.embed("Test") + print(f" {emb}") + print(f" 向量维度: {len(vec)}") + print(f" 向量示例: {vec[:5]}...") + except Exception as e: + print(f" ❌ 错误: {e}") + + +def demo_batch_embedding(): + """演示批量 embedding""" + print_separator("演示：批量 Embedding") + + print("\n批量处理 3 个文本:") + texts = ["文本1", "文本2", "文本3"] + + try: + emb = get_embedding_model("hash", dim=256) + vecs = emb.embed_batch(texts) + print(f" ✅ 成功生成 {len(vecs)} 个向量") + print(f" 每个向量维度: {len(vecs[0])}") + for i, vec in enumerate(vecs): + print(f" 向量 {i + 1}: {vec[:3]}...") + except Exception as e: + print(f" ❌ 错误: {e}") + + +def demo_api_key_methods(): + """演示需要 API Key 的方法（仅展示如何调用）""" + print_separator("演示：需要 API Key 的方法（代码示例）") + + examples = { + "openai": """ +# OpenAI Embedding +emb = get_embedding_model( + "openai", + model="text-embedding-3-small", + api_key="sk-xxx" +) +vec = emb.embed("hello world") +""", + "jina": """ +# Jina Embedding (Late Chunking) +emb = get_embedding_model( + "jina", + dimensions=256, + late_chunking=True, + api_key="jina-xxx" +) +vec = emb.embed("你好世界") +""", + "zhipu": """ +# 智谱 Embedding (批量) +emb = get_embedding_model( + "zhipu", + model="embedding-3", + api_key="zhipu-xxx" +) +vecs = emb.embed_batch(["文本1", "文本2", "文本3"]) +""", + "cohere": """ +# Cohere Embedding (多种 input_type) +emb = get_embedding_model( + "cohere", + model="embed-multilingual-v3.0", + input_type="classification", + api_key="cohere-xxx" +) +vec = emb.embed("positive review") +""", + "bedrock": """ +# AWS Bedrock Embedding +emb = get_embedding_model( + "bedrock", + model="amazon.titan-embed-text-v2:0", + aws_access_key_id="xxx", + aws_secret_access_key="xxx" +) +vec = emb.embed("hello world") +""", + "ollama": """ +# Ollama Embedding (本地) +emb = get_embedding_model( + "ollama", + model="nomic-embed-text", + base_url="http://localhost:11434" +) +vec = emb.embed("hello world") +""", + "siliconcloud": """ +# SiliconCloud Embedding +emb = get_embedding_model( + "siliconcloud", + model="netease-youdao/bce-embedding-base_v1", + api_key="silicon-xxx" +) +vec = emb.embed("你好") +""", + "nvidia_openai": """ +# NVIDIA NIM Embedding +emb = get_embedding_model( + "nvidia_openai", + model="nvidia/llama-3.2-nv-embedqa-1b-v1", + input_type="passage", + api_key="nvapi-xxx" +) +vec = emb.embed("document text") +""", + } + + for method, code in examples.items(): + print(f"\n{method}:") + print(code) + + +def main(): + """主函数""" + print( + """ +╔══════════════════════════════════════════════════════════╗ +║ ║ +║ SAGE Embedding 方法演示 ║ +║ Phase 2 Complete - 11 个统一接口 ║ +║ ║ +╚══════════════════════════════════════════════════════════╝ + """ + ) + + # 1. 列出所有方法 + list_all_methods() + + # 2. 检查可用性 + check_all_status() + + # 3. 演示无 API Key 方法 + demo_no_api_key_methods() + + # 4. 演示批量 embedding + demo_batch_embedding() + + # 5. 展示 API Key 方法示例 + demo_api_key_methods() + + print_separator() + print("\n✅ 演示完成！") + print("\n💡 提示:") + print(" - 无需 API Key 的方法可以直接使用") + print(" - 需要 API Key 的方法需要先设置环境变量或传递参数") + print(" - 使用 list_embedding_models() 查看所有方法") + print(" - 使用 check_model_availability() 检查状态") + print_separator() + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/embeddings/embedding_service_demo.py b/tutorials/L3-libs/embeddings/embedding_service_demo.py new file mode 100644 index 0000000..8f288bd --- /dev/null +++ b/tutorials/L3-libs/embeddings/embedding_service_demo.py @@ -0,0 +1,448 @@ +""" +Embedding Service Demo - 展示如何使用统一的 EmbeddingService + +这个示例展示了: +1. 如何配置 EmbeddingService (本地模型, API, vLLM) +2. 如何在 Pipeline 中使用 embedding service +3. 如何实现高性能批处理 +4. 如何使用缓存优化性能 + +Requirements: + pip install isage-middleware>=0.2.0 +""" + +import os +import sys +from pathlib import Path + + +def demo_basic_embedding_service(): + """示例 1: 基本的 Embedding Service 使用""" + print("\n" + "=" * 60) + print("示例 1: 基本 Embedding Service") + print("=" * 60) + + from sage.common.components.sage_embedding import EmbeddingService + + # 检查是否在测试模式 + is_test_mode = os.getenv("SAGE_TEST_MODE") == "true" or os.getenv("CI") == "true" + + # 配置: 在测试模式使用 mock，否则使用 HuggingFace 模型 + if is_test_mode: + config = { + "method": "mockembedder", + "dimension": 384, # 模拟 bge-small-zh-v1.5 的维度 + "batch_size": 32, + "normalize": True, + "cache_enabled": True, + "cache_size": 1000, + } + else: + config = { + "method": "hf", + "model": "BAAI/bge-small-zh-v1.5", + "batch_size": 32, + "normalize": True, + "cache_enabled": True, + "cache_size": 1000, + } + + service = EmbeddingService(config) + service.setup() + + # 获取服务信息 + info = service.process({"task": "info"}) + print("\n服务信息:") + print(f" 方法: {info['method']}") + print(f" 模型: {info['model']}") + print(f" 维度: {info['dimension']}") + print(f" 缓存: {info['cache_enabled']}") + + # 单个文本 embedding + result = service.process( + {"task": "embed", "inputs": "你好世界", "options": {"return_stats": True}} + ) + + print("\n单个文本 embedding:") + print(f" 维度: {result['dimension']}") + print(f" 向量前5个值: {result['vectors'][0][:5]}") + print(f" 统计: {result['stats']}") + + # 批量文本 embedding + texts = [ + "人工智能正在改变世界", + "机器学习是AI的核心", + "深度学习推动了AI发展", + "自然语言处理很重要", + ] + + result = service.process({"task": "embed", "inputs": texts, "options": {"return_stats": True}}) + + print("\n批量文本 embedding:") + print(f" 文本数量: {result['count']}") + print(f" 计算数量: {result['stats']['computed']}") + print(f" 缓存数量: {result['stats']['cached']}") + + # 再次查询相同文本 (测试缓存) + result2 = service.process( + { + "task": "embed", + "inputs": texts[:2], # 重复前两个文本 + "options": {"return_stats": True}, + } + ) + + print("\n缓存测试:") + print(f" 缓存命中: {result2['stats']['cached']}/{result2['count']}") + print(f" 命中率: {result2['stats']['cache_hit_rate']:.2%}") + + service.cleanup() + + +def demo_vllm_embedding_service(): + """Demo 2: 使用 vLLM 作为 Embedding 后端 (需要 GPU)""" + print("\n" + "=" * 60) + print("Demo 2: vLLM Embedding Service (高性能)") + print("=" * 60) + + # 注意: 这个示例需要实际的 vLLM service 运行 + print("\n配置示例:") + config_example = """ +services: + vllm: + class: sage.llm.VLLMService + config: + model_id: "BAAI/bge-base-en-v1.5" + embedding_model_id: "BAAI/bge-base-en-v1.5" + auto_download: true + engine: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + + embedding: + class: sage.common.components.sage_embedding.EmbeddingService + config: + method: "vllm" + vllm_service_name: "vllm" + batch_size: 256 # vLLM 可以处理大批量 + normalize: true + cache_enabled: true + +# 在 pipeline/operator 中使用: +result = self.call_service("embedding", payload={ + "task": "embed", + "inputs": large_document_list, # 可以是成千上万个文档 + "options": { + "batch_size": 256, + "return_stats": True + } +}) + """ + print(config_example) + + +def demo_multi_embedding_pipeline(): + """Demo 3: 多 Embedding Service 的 Pipeline""" + print("\n" + "=" * 60) + print("Demo 3: 多 Embedding 策略 Pipeline") + print("=" * 60) + + pipeline_config = """ +# 使用场景: RAG 系统 +# - 查询使用快速本地模型 (低延迟) +# - 文档索引使用高质量云端模型 (高精度) +# - 批量处理使用 vLLM (高吞吐) + +services: + # 1. 快速本地 embedding (用于实时查询) + embedding_fast: + class: sage.common.components.sage_embedding.EmbeddingService + config: + method: "hf" + model: "BAAI/bge-small-zh-v1.5" # 小模型, 快速 + batch_size: 32 + cache_enabled: true + + # 2. 高质量云端 embedding (用于离线索引) + embedding_quality: + class: sage.common.components.sage_embedding.EmbeddingService + config: + method: "openai" + model: "text-embedding-3-large" + api_key: "${OPENAI_API_KEY}" + batch_size: 100 + + # 3. vLLM 高吞吐 embedding (用于大规模批处理) + vllm: + class: sage.llm.VLLMService + config: + model_id: "BAAI/bge-large-en-v1.5" + + embedding_batch: + class: sage.common.components.sage_embedding.EmbeddingService + config: + method: "vllm" + vllm_service_name: "vllm" + batch_size: 512 + +operators: + # 查询 embedding - 使用快速本地模型 + - name: query_embed + type: custom + code: | + result = self.call_service("embedding_fast", payload={ + "task": "embed", + "inputs": payload["query"] + }) + payload["query_vector"] = result["vectors"][0] + return payload + + # 文档 embedding - 根据情况选择 + - name: document_embed + type: custom + code: | + docs = payload["documents"] + + # 小批量: 使用本地模型 + if len(docs) < 100: + service = "embedding_fast" + # 大批量: 使用 vLLM + elif len(docs) > 1000: + service = "embedding_batch" + # 重要文档: 使用高质量云端 + elif payload.get("high_quality"): + service = "embedding_quality" + else: + service = "embedding_fast" + + result = self.call_service(service, payload={ + "task": "embed", + "inputs": [d["text"] for d in docs] + }) + + for doc, vec in zip(docs, result["vectors"]): + doc["embedding"] = vec + + return payload + """ + + print(pipeline_config) + + +def demo_embedding_operator(): + """Demo 4: 创建自定义的 Embedding Operator""" + print("\n" + "=" * 60) + print("Demo 4: 自定义 Embedding Operator") + print("=" * 60) + + operator_code = ''' +from sage.libs.operators import BaseOperator +from typing import Any, Dict, List + +class SmartEmbeddingOperator(BaseOperator): + """智能 Embedding Operator - 根据负载自动选择策略""" + + def __init__( + self, + embedding_service: str = "embedding", + batch_size: int = 32, + cache_threshold: int = 10, # 小于此数量启用缓存 + vllm_threshold: int = 1000, # 大于此数量使用 vLLM + ): + self.embedding_service = embedding_service + self.batch_size = batch_size + self.cache_threshold = cache_threshold + self.vllm_threshold = vllm_threshold + + def process(self, payload: Dict[str, Any]) -> Dict[str, Any]: + texts = payload.get("texts", []) + + if not texts: + payload["embeddings"] = [] + return payload + + # 智能选择策略 + options = { + "batch_size": self.batch_size, + "return_stats": True, + } + + # 小批量: 启用缓存 + if len(texts) <= self.cache_threshold: + # 假设有缓存配置的 service + service = self.embedding_service + "_cached" + # 大批量: 使用 vLLM + elif len(texts) >= self.vllm_threshold: + service = self.embedding_service + "_vllm" + options["batch_size"] = min(512, len(texts)) + else: + service = self.embedding_service + + # 调用 embedding service + result = self.call_service(service, payload={ + "task": "embed", + "inputs": texts, + "options": options + }) + + # 附加结果 + payload["embeddings"] = result["vectors"] + payload["embedding_dimension"] = result["dimension"] + payload["embedding_stats"] = result.get("stats", {}) + + self.logger.info( + f"Embedded {len(texts)} texts using {service}, " + f"cache_hit_rate={result['stats'].get('cache_hit_rate', 0):.2%}" + ) + + return payload + + +# 使用示例 +class RAGPipeline: + def build(self): + return { + "operators": [ + { + "name": "load_query", + "type": "QueryLoaderOperator" + }, + { + "name": "embed_query", + "type": "SmartEmbeddingOperator", + "config": { + "embedding_service": "embedding", + "batch_size": 32, + "cache_threshold": 10, + } + }, + { + "name": "retrieve", + "type": "VectorSearchOperator", + "config": { + "top_k": 5 + } + }, + { + "name": "generate", + "type": "LLMGenerateOperator" + } + ] + } +''' + print(operator_code) + + +def demo_performance_comparison(): + """Demo 5: 性能对比 - 不同 embedding 方法""" + print("\n" + "=" * 60) + print("Demo 5: 性能对比") + print("=" * 60) + + comparison = """ +测试场景: 1000 个文档, 每个文档平均 100 tokens + +方法 吞吐量 延迟 成本 推荐使用场景 +----------------------------------------------------------------- +hash 10000/s <1ms 免费 快速原型, 测试 +mockembedder 5000/s <1ms 免费 单元测试 + +hf (small) 100/s 10ms 免费 实时查询, 预算有限 +hf (base) 50/s 20ms 免费 平衡性能和质量 +hf (large) 20/s 50ms 免费 高质量离线处理 + +openai (small) 1000/s 10ms $$$ 大规模云端部署 +openai (large) 500/s 20ms $$$$ 最高质量要求 + +jina 800/s 15ms $$ 中等规模, 多语言 +zhipu 600/s 20ms $$ 中文优化 + +vLLM (GPU) 2000/s 5ms 硬件 大规模生产环境 +vLLM (多GPU) 5000/s 3ms 硬件 超大规模部署 + +推荐配置: + + 1. 开发/测试: + method: "hash" 或 "mockembedder" + + 2. 小规模生产 (< 1M 文档): + method: "hf", model: "BAAI/bge-small-zh-v1.5" + + 3. 中等规模 (1M - 10M 文档): + 查询: method: "hf", cache_enabled: true + 索引: method: "openai" 或 "jina" + + 4. 大规模生产 (> 10M 文档): + method: "vllm", vllm_service_name: "vllm" + 配置多 GPU 以提高吞吐量 + + 5. 成本敏感: + method: "hf" (完全免费, 需要 GPU 硬件) + + 6. 质量优先: + method: "openai", model: "text-embedding-3-large" +""" + print(comparison) + + +def main(): + """运行所有示例""" + print("\n" + "=" * 60) + print("Embedding Service 示例集") + print("=" * 60) + + demos = [ + ("基本使用", demo_basic_embedding_service), + ("vLLM 后端", demo_vllm_embedding_service), + ("多 Embedding 策略", demo_multi_embedding_pipeline), + ("自定义 Operator", demo_embedding_operator), + ("性能对比", demo_performance_comparison), + ] + + # 检查是否在测试模式 + is_test_mode = os.getenv("SAGE_TEST_MODE") == "true" or os.getenv("CI") == "true" + + if is_test_mode: + # 测试模式：运行所有示例 + print("\n🧪 测试模式：自动运行所有示例\n") + for name, demo_func in demos: + try: + demo_func() + except Exception as e: + print(f"\n❌ {name} 失败: {e}") + else: + # 交互模式：让用户选择 + print("\n可用示例:") + for i, (name, _) in enumerate(demos, 1): + print(f" {i}. {name}") + + print("\n选择要运行的示例 (1-5, 或 'all' 运行全部, 'q' 退出):") + choice = input("> ").strip().lower() + + if choice == "q": + return + elif choice == "all": + for name, demo_func in demos: + try: + demo_func() + except Exception as e: + print(f"\n❌ {name} 失败: {e}") + elif choice.isdigit() and 1 <= int(choice) <= len(demos): + name, demo_func = demos[int(choice) - 1] + try: + demo_func() + except Exception as e: + print(f"\n❌ {name} 失败: {e}") + import traceback + + traceback.print_exc() + else: + print("无效选择") + + print("\n" + "=" * 60) + print("示例结束") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/embeddings/pipeline_builder_embedding_demo.py b/tutorials/L3-libs/embeddings/pipeline_builder_embedding_demo.py new file mode 100644 index 0000000..c7a01c4 --- /dev/null +++ b/tutorials/L3-libs/embeddings/pipeline_builder_embedding_demo.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +SAGE Pipeline Builder - Embedding Integration 示例 + +演示如何使用不同的 embedding 方法来增强 Pipeline Builder 的知识检索能力。 + +@test:allow-demo +@test:timeout=120 +""" + +import os + +from sage.cli.commands.apps.pipeline_knowledge import ( + PipelineKnowledgeBase, + get_default_knowledge_base, +) + +# 检查是否在测试模式 +_IS_TEST_MODE = os.getenv("SAGE_TEST_MODE") == "true" or os.getenv("CI") == "true" + +# 在测试模式下，减少chunks以加快初始化 +_MAX_CHUNKS = 50 if _IS_TEST_MODE else 100 + +# 全局缓存知识库实例，避免重复初始化 +_KB_CACHE = {} + + +def _get_or_create_kb(method: str, model: str | None = None, max_chunks: int | None = None): + """获取或创建知识库实例（带缓存）""" + cache_key = f"{method}:{model}:{max_chunks or _MAX_CHUNKS}" + if cache_key not in _KB_CACHE: + if method == "default": + _KB_CACHE[cache_key] = get_default_knowledge_base( + max_chunks=max_chunks or _MAX_CHUNKS, allow_download=False + ) + else: + _KB_CACHE[cache_key] = PipelineKnowledgeBase( + max_chunks=max_chunks or _MAX_CHUNKS, + allow_download=False, + embedding_method=method, + embedding_model=model, + ) + return _KB_CACHE[cache_key] + + +def example_1_basic_usage(): + """示例 1: 基本使用 - 默认 hash 方法""" + print("=" * 80) + print("示例 1: 使用默认的 hash embedding 方法") + print("=" * 80) + + # 使用缓存的知识库 + kb = _get_or_create_kb("default") + + # 执行检索 + query = "如何构建 RAG pipeline" + results = kb.search(query, top_k=3) + + print(f"\n查询: {query}") + print("检索方法: hash") + print(f"结果数量: {len(results)}\n") + + for idx, chunk in enumerate(results, 1): + print(f"[{idx}] 得分: {chunk.score:.4f} | 类型: {chunk.kind}") + print(f" {chunk.text[:100]}...") + print() + + +def example_2_custom_method(): + """示例 2: 使用自定义 embedding 方法""" + print("=" * 80) + print("示例 2: 使用 mockembedder 方法") + print("=" * 80) + + # 使用缓存的知识库 + kb = _get_or_create_kb("mockembedder") + + query = "向量检索算法" + results = kb.search(query, top_k=3) + + print(f"\n查询: {query}") + print("检索方法: mockembedder") + print(f"结果数量: {len(results)}\n") + + for idx, chunk in enumerate(results, 1): + print(f"[{idx}] 得分: {chunk.score:.4f} | 类型: {chunk.kind}") + print(f" {chunk.text[:100]}...") + print() + + +def example_3_compare_methods(): + """示例 3: 对比不同 embedding 方法的检索效果""" + print("=" * 80) + print("示例 3: 对比 hash vs mockembedder") + print("=" * 80) + + query = "语义搜索" + methods = ["hash", "mockembedder"] + + for method in methods: + print(f"\n--- 方法: {method} ---") + + # 使用缓存的知识库 + kb = _get_or_create_kb(method) + + import time + + start = time.time() + results = kb.search(query, top_k=3) + elapsed = time.time() - start + + print(f"耗时: {elapsed * 1000:.2f}ms") + print(f"Top-3 得分: {[f'{r.score:.4f}' for r in results]}") + + if results and results[0].vector: + print(f"向量维度: {len(results[0].vector)}") + + +def example_4_with_specific_model(): + """示例 4: 使用特定模型（需要 API key）""" + print("=" * 80) + print("示例 4: 使用 HuggingFace 模型 (需要模型已下载)") + print("=" * 80) + + # 注意: 这需要模型已经下载到本地 + # 如果没有，会自动下载（需要网络） + try: + # 在测试模式下使用更小的数据集 + kb = _get_or_create_kb("hf", "BAAI/bge-small-zh-v1.5", max_chunks=_MAX_CHUNKS // 2) + + query = "RAG 系统架构" + results = kb.search(query, top_k=3) + + print(f"\n查询: {query}") + print("检索方法: HuggingFace") + print("模型: BAAI/bge-small-zh-v1.5") + print(f"结果数量: {len(results)}\n") + + for idx, chunk in enumerate(results, 1): + print(f"[{idx}] 得分: {chunk.score:.4f}") + print(f" {chunk.text[:80]}...") + print() + + except Exception as e: + print(f"❌ HuggingFace 方法失败: {e}") + print("💡 提示: 这通常是因为:") + print(" 1. 模型未下载") + print(" 2. 缺少依赖 (sentence-transformers)") + print(" 3. 需要指定正确的模型名称") + + +def example_5_environment_variables(): + """示例 5: 使用环境变量配置""" + print("=" * 80) + print("示例 5: 通过环境变量配置默认方法") + print("=" * 80) + + import os + + # 设置环境变量 + os.environ["SAGE_PIPELINE_EMBEDDING_METHOD"] = "mockembedder" + + # 使用缓存的知识库 + kb = _get_or_create_kb("mockembedder") + + query = "embedding 优化" + results = kb.search(query, top_k=2) + + print("\n环境变量: SAGE_PIPELINE_EMBEDDING_METHOD=mockembedder") + print(f"查询: {query}") + print(f"结果数量: {len(results)}\n") + + for idx, chunk in enumerate(results, 1): + print(f"[{idx}] {chunk.text[:100]}...") + print() + + +def example_6_fallback_mechanism(): + """示例 6: 自动后备机制""" + print("=" * 80) + print("示例 6: 演示自动后备机制") + print("=" * 80) + + # 尝试使用一个需要配置的方法（不提供配置） + # 应该自动回退到 hash + kb = PipelineKnowledgeBase( + max_chunks=_MAX_CHUNKS // 2, + allow_download=False, + embedding_method="hf", # 不提供 model，会失败 + # embedding_model 缺失! + ) + + print("\n✓ 知识库创建成功（即使 hf 方法失败也会自动回退到 hash）") + print("💡 这就是自动后备机制的作用") + + query = "测试后备" + results = kb.search(query, top_k=2) + print(f"\n查询仍然可以正常工作: {len(results)} 个结果") + + +if __name__ == "__main__": + print("\n🎯 SAGE Pipeline Builder - Embedding Integration 示例\n") + + examples = [ + ("基本使用", example_1_basic_usage), + ("自定义方法", example_2_custom_method), + ("方法对比", example_3_compare_methods), + ("特定模型", example_4_with_specific_model), + ("环境变量", example_5_environment_variables), + ("后备机制", example_6_fallback_mechanism), + ] + + for title, example_func in examples: + try: + example_func() + except Exception as e: + print(f"❌ 示例失败: {e}") + + # 在测试模式下不等待用户输入 + if not _IS_TEST_MODE: + input("\n按 Enter 继续下一个示例...") + print("\n") + + print("=" * 80) + print("✅ 所有示例演示完成!") + print("=" * 80) + print("\n💡 CLI 使用提示:") + print(" sage pipeline analyze-embedding '你的查询' -m hash -m mockembedder") + print( + " sage pipeline build --embedding-method openai --embedding-model text-embedding-3-small" + ) + print() diff --git a/tutorials/L3-libs/embeddings/quickstart.py b/tutorials/L3-libs/embeddings/quickstart.py new file mode 100644 index 0000000..af57790 --- /dev/null +++ b/tutorials/L3-libs/embeddings/quickstart.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Quick tour of the MultimodalSageDB helper with text+image content. + +Requirements: + pip install isage-middleware>=0.2.0 +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +from sage.middleware.components.sage_db.python.multimodal_sage_db import ( + ModalityType, + MultimodalSearchParams, + create_text_image_db, +) + + +def make_embedding(seed: int, dimension: int) -> np.ndarray: + rng = np.random.default_rng(seed) + vec = rng.normal(size=dimension).astype("float32") + norm = np.linalg.norm(vec) + 1e-12 + return vec / norm + + +def build_dataset() -> dict[str, dict[str, Any]]: + return { + "aurora": { + "text": make_embedding(1, 768), + "image": make_embedding(2, 512), + "meta": {"genre": "photography", "location": "iceland"}, + }, + "latte-art": { + "text": make_embedding(3, 768), + "image": make_embedding(4, 512), + "meta": {"genre": "food", "mood": "cozy"}, + }, + "city-skyline": { + "text": make_embedding(5, 768), + "image": make_embedding(6, 512), + "meta": {"genre": "architecture", "time": "night"}, + }, + "trail-run": { + "text": make_embedding(7, 768), + "image": make_embedding(8, 512), + "meta": {"genre": "outdoor", "mood": "energetic"}, + }, + "catnap": { + "text": make_embedding(9, 768), + "image": make_embedding(10, 512), + "meta": {"genre": "pets", "mood": "calm"}, + }, + } + + +def populate(db, items: dict[str, dict[str, Any]]) -> None: + for name, payload in items.items(): + embeddings = { + ModalityType.TEXT: payload["text"], + ModalityType.IMAGE: payload["image"], + } + data_id = db.add_from_embeddings(embeddings, {"label": name, **payload["meta"]}) + print(f"➕ added '{name}' -> id={data_id}") + + +def main() -> None: + print("=== Multimodal text+image quickstart ===") + + db = create_text_image_db(dimension=512) + native = getattr(db, "_db", None) is not None + print( + "Backend:", + "C++ accelerated" if native else "Python mock (build to enable native)", + ) + + dataset = build_dataset() + populate(db, dataset) + + params = MultimodalSearchParams(k=3) + params.query_fusion_params.target_dimension = 512 + + query = { + ModalityType.TEXT: make_embedding(11, 768), + ModalityType.IMAGE: make_embedding(12, 512), + } + + print("\n🔎 fused retrieval (text + image cues)") + results = db.search_multimodal(query, params) + for idx, result in enumerate(results, start=1): + print(f" {idx}. id={result.id:>2} score={result.score:.4f} metadata={result.metadata}") + + stats = db.get_modality_statistics() + print("\n📊 modality stats:") + for modality, info in stats.items(): + print(f" {modality.name:<6} -> count={info['count']} avg_dim={info['avg_dimension']:.1f}") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/llm/demo_new_templates.py b/tutorials/L3-libs/llm/demo_new_templates.py new file mode 100644 index 0000000..c9a2644 --- /dev/null +++ b/tutorials/L3-libs/llm/demo_new_templates.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""演示新增模板的使用方法 + +@test:allow-demo +""" + +from rich.console import Console +from rich.markdown import Markdown +from rich.panel import Panel + +from sage.cli.templates.catalog import get_template, list_templates + +console = Console() + + +def demo_template_usage(): + """演示模板使用""" + + console.print( + Panel.fit( + "🎨 新增模板使用演示\n展示如何使用 6 个新增的应用模板", + title="演示开始", + border_style="bold blue", + ) + ) + + # 新增的模板 ID + new_template_ids = [ + "rag-dense-milvus", + "rag-rerank", + "rag-bm25-sparse", + "agent-workflow", + "rag-memory-enhanced", + "multimodal-cross-search", + ] + + for template_id in new_template_ids: + template = get_template(template_id) + + console.print(f"\n{'=' * 80}", style="bold cyan") + console.print(f"模板: {template.title}", style="bold yellow") + console.print(f"ID: {template.id}", style="dim") + console.print(f"{'=' * 80}", style="bold cyan") + + # 基本信息 + console.print("\n📝 描述:", style="bold green") + console.print(f" {template.description}") + + console.print("\n🏷️ 标签:", style="bold blue") + console.print(f" {', '.join(template.tags[:8])}") + + console.print("\n📂 示例路径:", style="bold magenta") + console.print(f" {template.example_path}") + + # 显示 Pipeline 结构 + console.print("\n🔧 Pipeline 结构:", style="bold cyan") + plan = template.pipeline_plan() + source_class = plan.get("source", {}).get("class", "N/A") + console.print(f" Source: {source_class}", style="green") + + stages = plan.get("stages", []) + for i, stage in enumerate(stages, 1): + console.print( + f" Stage {i}: {stage.get('id', 'N/A')} → {stage.get('class', 'N/A')}", + style="yellow", + ) + if stage.get("summary"): + console.print(f" {stage.get('summary')}", style="dim") + + sink_class = plan.get("sink", {}).get("class", "N/A") + console.print(f" Sink: {sink_class}", style="red") + + # 显示使用指南 + console.print("\n💡 使用指南:", style="bold green") + console.print(f" {template.guidance.strip()}") + + # 显示注意事项 + if template.notes: + console.print("\n⚠️ 注意事项:", style="bold yellow") + for note in template.notes: + console.print(f" • {note}") + + console.print("\n" + "─" * 80) + + # 使用示例 + console.print(f"\n\n{'=' * 80}", style="bold blue") + console.print("📚 使用示例", style="bold blue") + console.print(f"{'=' * 80}", style="bold blue") + + usage_examples = """ +## 方式一: 在代码中使用 + +```python +from sage.cli.templates.catalog import get_template + +# 获取模板 +template = get_template("rag-dense-milvus") + +# 获取 Pipeline 配置 +config = template.pipeline_plan() + +# 查看配置 +print(config) +``` + +## 方式二: 通过 sage chat 命令使用 + +```bash +# 启动交互式 chat 界面 +sage chat + +# 输入需求，系统会自动匹配合适的模板 +"我想使用 Milvus 向量数据库构建一个语义检索系统" +``` + +## 方式三: 匹配最佳模板 + +```python +from sage.cli.templates.catalog import match_templates + +requirements = { + "name": "智能问答系统", + "goal": "使用向量检索和重排序构建高精度问答", + "data_sources": ["文档库"], + "constraints": "需要高精度" +} + +# 获取最匹配的模板 +matches = match_templates(requirements, top_k=3) +for match in matches: + print(f"{match.template.title}: {match.score:.3f}") +``` + +## 推荐的使用场景 + +### 1. Milvus 向量检索 (`rag-dense-milvus`) +- 适合: 大规模文档库、生产环境部署 +- 需要: Milvus 服务、嵌入模型 +- 优势: 高性能、可扩展 + +### 2. 重排序检索 (`rag-rerank`) +- 适合: 高精度要求场景（法律、医疗、金融） +- 需要: 向量库 + BGE Reranker +- 优势: 精确度高 + +### 3. BM25 检索 (`rag-bm25-sparse`) +- 适合: 关键词匹配、资源受限环境 +- 需要: 文本语料库 +- 优势: 无需 GPU、计算成本低 + +### 4. 智能体工作流 (`agent-workflow`) +- 适合: 复杂任务自动化、多步骤推理 +- 需要: LLM API、MCP 工具 +- 优势: 自主规划、工具调用 + +### 5. 记忆对话 (`rag-memory-enhanced`) +- 适合: 多轮对话、客服机器人 +- 需要: 记忆服务、对话历史存储 +- 优势: 上下文连贯 + +### 6. 跨模态搜索 (`multimodal-cross-search`) +- 适合: 图文混合检索、电商、媒体 +- 需要: 多模态向量库、图像编码器 +- 优势: 支持文本、图像、融合检索 +""" + + console.print(Markdown(usage_examples)) + + # 总结 + console.print(f"\n\n{'=' * 80}", style="bold blue") + console.print("✅ 总结", style="bold blue") + console.print(f"{'=' * 80}", style="bold blue") + + console.print(f"\n已展示 {len(new_template_ids)} 个新增模板", style="bold green") + console.print(f"总计 {len(list_templates())} 个可用模板", style="bold cyan") + console.print( + "\n💡 提示: 使用 'sage chat' 命令可以自动匹配最合适的模板！", + style="bold yellow", + ) + + +if __name__ == "__main__": + demo_template_usage() diff --git a/tutorials/L3-libs/llm/pipeline_builder_llm_demo.py b/tutorials/L3-libs/llm/pipeline_builder_llm_demo.py new file mode 100644 index 0000000..87f277b --- /dev/null +++ b/tutorials/L3-libs/llm/pipeline_builder_llm_demo.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +演示 SAGE Pipeline Builder 中的大模型交互流程 + +这个脚本展示了用户请求如何通过 RAG 和 LLM 转换为完整的 Pipeline 配置 + +@test:allow-demo +""" + +import json + +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax + +from sage.cli.commands.apps.pipeline_domain import load_domain_contexts +from sage.cli.commands.apps.pipeline_knowledge import get_default_knowledge_base + +console = Console() + + +def demonstrate_llm_pipeline(): + """演示完整的 LLM Pipeline 构建流程""" + + console.print("\n" + "=" * 80) + console.print("[bold cyan]SAGE Pipeline Builder - LLM 交互流程演示[/bold cyan]") + console.print("=" * 80 + "\n") + + # Step 1: 用户需求 + console.print("[bold]步骤 1: 用户需求[/bold]") + user_request = "请帮我构建一个基于文档检索的智能问答系统" + requirements = { + "name": "智能问答助手", + "goal": "构建基于文档检索的问答系统，支持向量检索和大模型生成", + "data_sources": ["文档知识库", "向量数据库"], + "latency_budget": "实时响应优先", + "constraints": "支持流式输出", + } + console.print(f"用户输入: [yellow]{user_request}[/yellow]") + console.print("\n收集到的需求:") + console.print( + Panel( + json.dumps(requirements, ensure_ascii=False, indent=2), + title="Requirements", + border_style="green", + ) + ) + + # Step 2: 加载 Domain Contexts + console.print("\n[bold]步骤 2: 加载 Domain Contexts (示例配置)[/bold]") + try: + domain_contexts = tuple(load_domain_contexts(limit=2)) + console.print(f"✓ 加载了 {len(domain_contexts)} 个示例配置片段") + if domain_contexts: + console.print("\n示例片段（前 200 字符）:") + console.print(f"[dim]{domain_contexts[0][:200]}...[/dim]") + except Exception as exc: + console.print(f"[yellow]加载失败: {exc}[/yellow]") + domain_contexts = () + + # Step 3: 初始化知识库 + console.print("\n[bold]步骤 3: 初始化知识库 (RAG)[/bold]") + try: + kb = get_default_knowledge_base(max_chunks=500, allow_download=False) + console.print("✓ 知识库初始化成功") + console.print(" - 文档来源: docs-public/, examples/, packages/sage-libs/") + console.print(" - 检索方法: 向量相似度匹配") + except Exception as exc: + console.print(f"[yellow]知识库初始化失败: {exc}[/yellow]") + console.print("[dim]提示: 在实际使用中会自动下载或使用本地文档[/dim]") + kb = None + + # Step 4: RAG 检索 + console.print("\n[bold]步骤 4: RAG 检索相关文档和代码[/bold]") + if kb: + from sage.cli.commands.apps.pipeline_knowledge import build_query_payload + + query = build_query_payload(requirements) + console.print(f"\n检索查询: [cyan]{query[:150]}...[/cyan]") + + try: + results = kb.search(query, top_k=3) + console.print(f"\n✓ 检索到 {len(results)} 个相关片段:") + for idx, chunk in enumerate(results, 1): + console.print( + f"\n[{idx}] 来源: [green]{chunk.source}[/green] (相关度: {chunk.score:.3f})" + ) + console.print(f"[dim]{chunk.text[:200]}...[/dim]") + except Exception as exc: + console.print(f"[yellow]检索失败: {exc}[/yellow]") + else: + console.print("[dim]知识库未初始化，跳过检索[/dim]") + + # Step 5: 模板匹配 + console.print("\n[bold]步骤 5: 匹配应用模板[/bold]") + try: + from sage.cli import templates + + matches = templates.match_templates(requirements, top_k=3) + console.print(f"✓ 找到 {len(matches)} 个相关模板:") + for match in matches[:3]: + console.print(f" - {match.template.title} ({match.template.id})") + console.print(f" 标签: {', '.join(match.template.tags)}") + console.print(f" 匹配度: {match.score:.2f}") + except Exception as exc: + console.print(f"[yellow]模板匹配失败: {exc}[/yellow]") + + # Step 6: 蓝图匹配 + console.print("\n[bold]步骤 6: 匹配配置蓝图[/bold]") + try: + from sage.cli.templates import pipeline_blueprints + + blueprint_matches = tuple(pipeline_blueprints.match_blueprints(requirements)) + console.print(f"✓ 找到 {len(blueprint_matches)} 个相关蓝图:") + for blueprint, score in blueprint_matches[:3]: + console.print(f" - {blueprint.id}: {blueprint.title}") + console.print(f" 匹配度: {score:.2f}") + except Exception as exc: + console.print(f"[yellow]蓝图匹配失败: {exc}[/yellow]") + + # Step 7: 构建提示词 + console.print("\n[bold]步骤 7: 构建 LLM 提示词[/bold]") + console.print( + """ +提示词结构: +┌──────────────────────────────────────┐ +│ System Prompt │ +│ - SAGE Pipeline 规范说明 │ +│ - JSON 结构定义 │ +│ - 生成规则 │ +└──────────────────────────────────────┘ + ↓ +┌──────────────────────────────────────┐ +│ User Prompt │ +│ 1. 用户需求 (JSON) │ +│ 2. 应用模板 (top 3) │ +│ 3. 配置蓝图 (top 3) │ +│ 4. 知识库检索结果 (top 5) │ +│ 5. Domain 上下文 (示例配置) │ +│ 6. 上一版配置 (如有) │ +│ 7. 用户反馈 (如有) │ +└──────────────────────────────────────┘ + """ + ) + + # Step 8: 模拟 LLM 调用 + console.print("\n[bold]步骤 8: 调用大模型生成配置[/bold]") + console.print( + """ +[cyan]>>> 调用 LLM API...[/cyan] +模型: qwen-max (或用户指定模型) +参数: max_tokens=1200, temperature=0.2 + """ + ) + + # 示例生成的配置 + example_config = { + "pipeline": { + "name": "智能问答助手", + "description": "基于文档检索的问答系统，支持向量检索和大模型生成", + "version": "1.0.0", + "type": "local", + }, + "source": {"class": "sage.libs.rag.source.TerminalInputSource", "params": {}}, + "stages": [ + { + "id": "retriever", + "kind": "map", + "class": "sage.libs.rag.retriever.FAISSRetriever", + "params": {"index_path": "data/vector_index", "top_k": 5}, + "summary": "向量检索相关文档", + }, + { + "id": "promptor", + "kind": "map", + "class": "sage.libs.rag.promptor.QAPromptor", + "params": {}, + "summary": "构建问答提示词", + }, + { + "id": "generator", + "kind": "map", + "class": "sage.libs.rag.generator.OpenAIGenerator", + "params": {"model": "qwen-max", "temperature": 0.7, "stream": True}, + "summary": "大模型生成回答", + }, + ], + "sink": {"class": "sage.libs.rag.sink.ConsoleSink", "params": {}}, + "services": [], + "monitors": [], + "notes": ["使用 FAISS 进行向量检索", "支持流式输出", "可配置检索相关文档数量"], + } + + console.print("\n[bold green]✓ LLM 返回配置:[/bold green]") + syntax = Syntax( + json.dumps(example_config, ensure_ascii=False, indent=2), + "json", + theme="monokai", + line_numbers=True, + ) + console.print(syntax) + + # Step 9: 验证配置 + console.print("\n[bold]步骤 9: 验证生成的配置[/bold]") + from sage.cli.commands.apps.chat import _validate_pipeline_config + + is_valid, errors = _validate_pipeline_config(example_config) + if is_valid: + console.print("[green]✓ 配置验证通过[/green]") + else: + console.print(f"[red]✗ 配置验证失败: {errors}[/red]") + + # Step 10: 用户确认和保存 + console.print("\n[bold]步骤 10: 用户确认和保存[/bold]") + console.print( + """ +用户可以: + 1. ✅ 确认配置 → 保存为 YAML 文件 + 2. ✏️ 提供反馈 → 重新生成（最多 6 轮） + 3. ▶️ 立即运行 Pipeline + 4. ❌ 取消构建 + """ + ) + + console.print("\n" + "=" * 80) + console.print("[bold cyan]演示完成![/bold cyan]") + console.print("=" * 80 + "\n") + + console.print( + Panel( + """ +[bold]关键要点:[/bold] + +1. 🤖 [cyan]大模型全程参与[/cyan] + - 接收包含文档、模板、代码示例的丰富上下文 + - 基于 SAGE 规范生成配置 + +2. 📚 [cyan]RAG 检索增强[/cyan] + - 自动从文档库检索相关内容 + - 匹配最相关的模板和蓝图 + - 提供代码示例参考 + +3. 🔄 [cyan]多轮迭代优化[/cyan] + - 支持用户反馈 + - 基于上一版配置改进 + - 最多 6 轮优化 + +4. ✅ [cyan]自动验证[/cyan] + - 检查配置结构 + - 验证必需字段 + - 检查类导入路径 + """, + title="总结", + border_style="green", + ) + ) + + +if __name__ == "__main__": + demonstrate_llm_pipeline() diff --git a/tutorials/L3-libs/llm/templates_to_llm_demo.py b/tutorials/L3-libs/llm/templates_to_llm_demo.py new file mode 100644 index 0000000..1eeb4dd --- /dev/null +++ b/tutorials/L3-libs/llm/templates_to_llm_demo.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +""" +详细演示：大模型如何参考 Templates + +这个脚本展示了从 Template 匹配到传递给 LLM 的完整过程 + +@test:allow-demo +""" + +import json + +from rich.console import Console +from rich.panel import Panel + +from sage.cli import templates +from sage.cli.commands.apps.pipeline import _template_contexts + +console = Console() + + +def demonstrate_template_to_llm(): + """演示模板如何被传递给 LLM""" + + console.print("\n" + "=" * 80) + console.print("[bold cyan]Templates → LLM 完整流程演示[/bold cyan]") + console.print("=" * 80 + "\n") + + # Step 1: 用户需求 + console.print("[bold]步骤 1: 用户需求[/bold]") + requirements = { + "name": "智能问答助手", + "goal": "构建基于文档检索的问答系统", + "data_sources": ["文档知识库"], + "latency_budget": "实时响应优先", + } + console.print( + Panel( + json.dumps(requirements, ensure_ascii=False, indent=2), + title="用户需求", + border_style="green", + ) + ) + + # Step 2: Template 匹配 + console.print("\n[bold]步骤 2: Template 自动匹配[/bold]") + console.print("[dim]调用: templates.match_templates(requirements, top_k=3)[/dim]\n") + + matches = templates.match_templates(requirements, top_k=3) + + console.print(f"✓ 找到 {len(matches)} 个相关模板:\n") + for idx, match in enumerate(matches, 1): + console.print(f"[{idx}] {match.template.title} ([cyan]{match.template.id}[/cyan])") + console.print(f" 标签: [yellow]{', '.join(match.template.tags)}[/yellow]") + console.print(f" 匹配度: [magenta]{match.score:.2f}[/magenta]") + console.print() + + # Step 3: 转换为 LLM 可读的提示词 + console.print("\n[bold]步骤 3: 转换为 LLM 提示词[/bold]") + console.print("[dim]调用: match.template.render_prompt(match.score)[/dim]\n") + + if matches: + top_match = matches[0] + template_prompt = top_match.template.render_prompt(top_match.score) + + console.print( + Panel( + template_prompt, + title=f"模板提示词: {top_match.template.title}", + border_style="blue", + ) + ) + + # Step 4: 所有模板上下文 + console.print("\n[bold]步骤 4: 组装所有模板上下文[/bold]") + console.print("[dim]调用: _template_contexts(matches)[/dim]\n") + + template_contexts = _template_contexts(matches) + console.print(f"✓ 生成了 {len(template_contexts)} 个模板上下文片段\n") + + # Step 5: 构建发送给 LLM 的完整提示词 + console.print("\n[bold]步骤 5: 构建完整的 User Prompt[/bold]") + console.print("[dim]在 _build_prompt() 方法中组装[/dim]\n") + + # 模拟 _build_prompt 的逻辑 + blocks = [ + "请根据以下需求生成符合 SAGE 框架的 pipeline 配置 JSON：", + json.dumps(requirements, ensure_ascii=False, indent=2), + ] + + if template_contexts: + blocks.append("以下应用模板仅作灵感参考，请结合需求自行设计：") + for idx, snippet in enumerate(template_contexts, start=1): + blocks.append(f"模板[{idx}]:\n{snippet.strip()}") + + blocks.append("严格输出单个 JSON 对象，不要包含 markdown、注释或多余文字。") + + user_prompt = "\n\n".join(blocks) + + # 只显示前 1500 字符 + preview = user_prompt[:1500] + console.print( + Panel( + preview + "\n\n[dim]... (省略部分内容) ...[/dim]", + title="发送给 LLM 的 User Prompt (预览)", + border_style="magenta", + ) + ) + + # Step 6: 完整的 API 调用 + console.print("\n[bold]步骤 6: 完整的 LLM API 调用[/bold]\n") + + console.print("发送给 LLM 的完整 messages:") + console.print( + """ +[cyan]messages = [ + { + "role": "system", + "content": SYSTEM_PROMPT # SAGE Pipeline 规范说明 + }, + { + "role": "user", + "content": user_prompt # 包含模板、需求、知识库检索等 + } +][/cyan] + +[yellow]# 调用 LLM[/yellow] +response = self._client.generate( + messages, + max_tokens=1200, + temperature=0.2 +) + """ + ) + + # 关键代码位置 + console.print("\n" + "=" * 80) + console.print("[bold green]关键代码位置[/bold green]") + console.print("=" * 80 + "\n") + + code_locations = """ +1️⃣ Template 匹配 (pipeline.py:598-601) + self._template_matches = tuple( + templates.match_templates(requirements, top_k=3) + ) + +2️⃣ 转换为上下文 (pipeline.py:600) + self._last_template_contexts = _template_contexts(self._template_matches) + +3️⃣ 传递给 _build_prompt (pipeline.py:625-631) + user_prompt = self._build_prompt( + requirements, + previous_plan, + feedback, + knowledge_contexts, + self._last_template_contexts, # ← 这里！ + self._last_blueprint_contexts, + ) + +4️⃣ 在 prompt 中注入模板 (pipeline.py:654-657) + if template_contexts: + blocks.append("以下应用模板仅作灵感参考，请结合需求自行设计：") + for idx, snippet in enumerate(template_contexts, start=1): + blocks.append(f"模板[{idx}]:\\n{snippet.strip()}") + +5️⃣ 调用 LLM (pipeline.py:632-637) + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ] + response = self._client.generate(messages, max_tokens=1200, temperature=0.2) + """ + + console.print(Panel(code_locations, border_style="green")) + + # 测试环境说明 + console.print("\n" + "=" * 80) + console.print("[bold yellow]关于测试环境[/bold yellow]") + console.print("=" * 80 + "\n") + + test_info = """ +[red]测试使用 Mock（不调用真实 LLM）[/red] + +在测试中 (test_chat_pipeline.py): + • 使用 DummyGenerator 替代 PipelinePlanGenerator + • 不真正调用 OpenAI API + • 返回预定义的配置 + +原因: + 1. 避免测试依赖外部 API + 2. 提高测试速度和稳定性 + 3. 不需要 API Key + +[green]生产环境使用真实 LLM[/green] + +在实际使用时 (backend != "mock"): + • 使用 OpenAIClient 调用真实 API + • 需要配置 TEMP_GENERATOR_API_KEY + • 支持 OpenAI / 兼容接口 (vLLM, Ollama 等) + +示例: + export TEMP_GENERATOR_API_KEY="sk-xxx" # pragma: allowlist secret + sage chat --backend openai --model qwen-max + """ + + console.print(Panel(test_info, border_style="yellow")) + + # 如何验证 + console.print("\n" + "=" * 80) + console.print("[bold cyan]如何验证 Templates 被使用[/bold cyan]") + console.print("=" * 80 + "\n") + + verification = """ +方法 1: 启用调试输出 + sage pipeline build \\ + --name "TestApp" \\ + --goal "构建问答应用" \\ + --show-knowledge # ← 会显示匹配的模板！ + +方法 2: 查看日志 + 在生成过程中，会调用 _render_template_panel() 显示匹配的模板 + +方法 3: 代码断点 + 在 pipeline.py:625 设置断点，查看 self._last_template_contexts 的值 + +方法 4: 打印提示词（调试用） + 在 pipeline.py:638 之后添加: + print("="*80) + print("User Prompt:", user_prompt) + print("="*80) + """ + + console.print(Panel(verification, border_style="cyan")) + + console.print("\n" + "=" * 80) + console.print("[bold green]演示完成！[/bold green]") + console.print("=" * 80 + "\n") + + +if __name__ == "__main__": + demonstrate_template_to_llm() diff --git a/tutorials/L3-libs/rag/README.md b/tutorials/L3-libs/rag/README.md new file mode 100644 index 0000000..d39a365 --- /dev/null +++ b/tutorials/L3-libs/rag/README.md @@ -0,0 +1,37 @@ +# RAG Tutorials + +Simple RAG (Retrieval-Augmented Generation) examples to get started. + +## Examples + +### 1. Simple RAG (`simple_rag.py`) + +Basic RAG pipeline example. + +```bash +python examples/tutorials/rag/simple_rag.py +``` + +**Requirements**: API key (set in `.env`) + +### 2. QA Without Retrieval (`qa_no_retrieval.py`) + +Direct QA using LLM without retrieval. + +```bash +python examples/tutorials/rag/qa_no_retrieval.py +``` + +### 3. QA with Local LLM (`qa_local_llm.py`) + +QA using a local language model. + +```bash +python examples/tutorials/rag/qa_local_llm.py +``` + +## Next Steps + +- **Advanced RAG examples**: See `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/` for + production-ready RAG pipelines and benchmarks +- **RAG library**: `packages/sage-libs/src/sage/libs/rag/` diff --git a/tutorials/L3-libs/rag/adaptive_rag_v2.py b/tutorials/L3-libs/rag/adaptive_rag_v2.py new file mode 100644 index 0000000..017d0df --- /dev/null +++ b/tutorials/L3-libs/rag/adaptive_rag_v2.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +""" +Adaptive RAG v2 - 自适应检索增强生成（保留旧版分流逻辑） + +核心特性： +- 完全保留旧版 side_output 的分流逻辑 +- 使用 FlatMap + Filter 替代 side_output 实现分支 +- 向量库分支 vs Web 搜索分支独立处理 +- 使用新版 SAGE API： + - UnifiedInferenceClient (LLM + Embedding) + - MemoryManager (向量库管理) + - EmbeddingFactory (本地 Embedding) + +数据流（保留旧版双分支结构）： + ┌─→ [Filter: vector] → DenseRetriever → Generator → Sink + 问题 → 路由判断 ─┤ + └─→ [Filter: web] → WebSearchAgent → Sink + +对比： + - 旧版: query_stream.side_output("vector").map(...) + - 新版: query_stream.filter(VectorFilter).map(...) +""" + +from __future__ import annotations + +import os +import sys +import time +from typing import Any + +# 屏蔽代理设置（远程服务不需要代理） +os.environ.pop("http_proxy", None) +os.environ.pop("https_proxy", None) +os.environ.pop("HTTP_PROXY", None) +os.environ.pop("HTTPS_PROXY", None) +os.environ.pop("all_proxy", None) +os.environ.pop("ALL_PROXY", None) + +import numpy as np +from dotenv import load_dotenv + +from sage.common.components.sage_llm import EmbeddingClientAdapter, LLMClientAdapter +from sage.common.core.functions.filter_function import FilterFunction +from sage.common.core.functions.flatmap_function import FlatMapFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + +# 尝试导入 MemoryManager（可选，用于持久化向量库） +try: + from sage.middleware.components.sage_mem.neuromem.memory_manager import MemoryManager + + HAS_MEMORY_MANAGER = True + _ = MemoryManager # 标记为已使用（可选功能） +except ImportError: + HAS_MEMORY_MANAGER = False + print("⚠️ MemoryManager 不可用，使用简单内存向量库") + + +# ============================================================ +# 远程服务配置（与 adaptive_rag.py 一致） +# ============================================================ + +# LLM 服务（可选择不同大小的模型） +LLM_HOST = "11.11.11.7" +LLM_MODELS = { + "32B": ("8901", "Qwen/Qwen2.5-32B-Instruct"), + "14B": ("8902", "Qwen/Qwen2.5-14B-Instruct"), + "7B": ("8903", "Qwen/Qwen2.5-7B-Instruct"), # 默认 + "1.5B": ("8904", "Qwen/Qwen2.5-1.5B-Instruct"), + "0.5B": ("8905", "Qwen/Qwen2.5-0.5B-Instruct"), +} + +# 使用 7B 模型作为默认 +DEFAULT_LLM = "7B" +LLM_PORT, LLM_MODEL = LLM_MODELS[DEFAULT_LLM] +LLM_BASE_URL = f"http://{LLM_HOST}:{LLM_PORT}/v1" + +# Embedding 服务 +EMBEDDING_BASE_URL = f"http://{LLM_HOST}:8090/v1" +EMBEDDING_MODEL = "BAAI/bge-large-zh-v1.5" + + +# ============================================================ +# Prompt 模板（与旧版一致） +# ============================================================ + +ROUTE_PROMPT_TEMPLATE = """Instruction: +You are an expert at routing a user question to a vectorstore or web search. +Use the vectorstore for questions on travel to Hubei Province in China. +You do not need to be stringent with the keywords in the question related to these topics. +Otherwise, use web-search. Give a binary choice 'web_search' or 'vectorstore' based on the question. +Return a JSON with a single key 'datasource' and no preamble or explanation. +Question to route: {question} +""" + +QA_PROMPT_TEMPLATE = """请根据以下背景信息回答问题。如果背景信息不足以回答问题，请诚实说明。 + +背景信息： +{context} + +问题：{question} + +请给出简洁准确的回答：""" + + +# ============================================================ +# 湖北旅游知识库数据 +# ============================================================ + +HUBEI_DOCUMENTS = [ + "武汉是湖北省省会，著名景点包括黄鹤楼、东湖、户部巷、江汉路步行街等。黄鹤楼是江南三大名楼之一，享有'天下江山第一楼'的美誉。", + "东湖是中国最大的城中湖，面积约33平方公里，是5A级风景区。湖畔有磨山、听涛、落雁等景区，春天的樱花尤为著名。", + "长江三峡是中国著名的风景名胜区，包括瞿塘峡、巫峡和西陵峡，全长约200公里。三峡大坝是世界上最大的水利枢纽工程。", + "恩施大峡谷是国家5A级景区，以天坑、地缝、溶洞、绝壁、峰丛著称，被誉为'湖北的张家界'。峡谷全长108公里。", + "神农架是中国唯一以'林区'命名的行政区，是世界自然遗产地。这里有金丝猴、白熊等珍稀动物，还有神秘的'野人'传说。", + "宜昌是三峡大坝所在地，被称为'世界水电之都'。这里是屈原和王昭君的故乡，有屈原祠、昭君村等人文景点。", + "武当山位于湖北十堰，是道教圣地，金顶和紫霄宫是著名景点。武当武术与少林功夫齐名，被列入世界文化遗产。", + "荆州古城是中国历史文化名城，三国时期的兵家必争之地。城墙保存完好，有荆州博物馆和张居正故居等景点。", +] + + +# ============================================================ +# 全局服务（延迟初始化）- 使用 SAGE 组件 +# ============================================================ + +_llm_client: LLMClientAdapter | None = None +_embedding_client: EmbeddingClientAdapter | None = None +_vector_collection: Any = None + + +def get_llm_client() -> LLMClientAdapter: + """获取 LLM 客户端（使用 LLMClientAdapter）""" + global _llm_client + if _llm_client is None: + _llm_client = LLMClientAdapter( + base_url=LLM_BASE_URL, + model_name=LLM_MODEL, + ) + print(f"✅ LLMClientAdapter 初始化完成: {LLM_BASE_URL}") + return _llm_client + + +def get_embedding_client() -> EmbeddingClientAdapter: + """获取 Embedding 客户端（使用 EmbeddingClientAdapter）""" + global _embedding_client + if _embedding_client is None: + _embedding_client = EmbeddingClientAdapter.create_api( + base_url=EMBEDDING_BASE_URL, + model=EMBEDDING_MODEL, + ) + print(f"✅ EmbeddingClientAdapter 初始化完成: {EMBEDDING_BASE_URL}") + return _embedding_client + + +def get_vector_collection(): + """获取向量库 Collection""" + global _vector_collection + + if _vector_collection is not None: + return _vector_collection + + # 使用简单内存向量库（避免 MemoryManager 接口复杂性） + _vector_collection = SimpleVectorDB(get_embedding_client()) + _vector_collection.add_documents(HUBEI_DOCUMENTS) + + return _vector_collection + + +class SimpleVectorDB: + """简单内存向量库 - 使用 EmbeddingClientAdapter""" + + def __init__(self, embedding_client: EmbeddingClientAdapter): + self.client = embedding_client + self.documents: list[str] = [] + self.embeddings: list[list[float]] = [] + + def add_documents(self, documents: list[str]): + """添加文档并计算 embedding""" + print(f"📦 构建简单向量库 ({len(documents)} 文档)...") + # 批量计算 embedding + embeddings = self.client.embed(documents) + self.documents = documents + self.embeddings = embeddings + print("✅ 向量库构建完成") + + def search(self, query: str, top_k: int = 3) -> list[str]: + """向量检索""" + # 计算查询的 embedding + query_embeddings = self.client.embed([query]) + query_embedding = query_embeddings[0] if query_embeddings else [] + + # 计算余弦相似度 + similarities = [] + for i, emb in enumerate(self.embeddings): + sim = self._cosine_similarity(query_embedding, emb) + similarities.append((i, sim)) + + similarities.sort(key=lambda x: x[1], reverse=True) + return [self.documents[i] for i, _ in similarities[:top_k]] + + def _cosine_similarity(self, a: list[float], b: list[float]) -> float: + """计算余弦相似度""" + a_arr = np.array(a) + b_arr = np.array(b) + return float(np.dot(a_arr, b_arr) / (np.linalg.norm(a_arr) * np.linalg.norm(b_arr))) + + +# ============================================================ +# Source: 问题输入源（与旧版一致） +# ============================================================ + + +class QuestionSource(SourceFunction): + """问题源：从预设问题列表获取问题""" + + def __init__(self, questions: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.questions = questions or [ + "武汉有哪些著名景点？", + "今天的天气怎么样？", + "神农架有什么好玩的？", + "Python 怎么学习？", + ] + self.index = 0 + + def execute(self, data=None) -> dict | None: + if self.index >= len(self.questions): + return None + question = self.questions[self.index] + self.index += 1 + print(f"\n{'=' * 60}") + print(f"📝 问题 {self.index}: {question}") + return {"question": question} + + +# ============================================================ +# RoutePromptFunction: 构造路由 Prompt（与旧版一致） +# ============================================================ + + +class RoutePromptFunction(MapFunction): + """ + 构造路由 prompt，用于判断使用向量库还是 Web 搜索。 + 对应旧版 RoutePromptFunction + """ + + def execute(self, data: dict) -> dict: + question = data["question"] + prompt = ROUTE_PROMPT_TEMPLATE.format(question=question) + return {"question": question, "messages": [{"role": "user", "content": prompt}]} + + +# ============================================================ +# LLMGenerator: 调用 LLM 生成（使用 LLMClientAdapter） +# ============================================================ + + +class LLMGenerator(MapFunction): + """ + 调用 LLM 生成响应（使用 LLMClientAdapter）。 + 对应旧版 OpenAIGenerator + """ + + def execute(self, data: dict) -> dict: + messages = data["messages"] + question = data["question"] + + client = get_llm_client() + try: + response = client.chat(messages, temperature=0, max_tokens=100) + llm_output = response.content if hasattr(response, "content") else str(response) + except Exception as e: + print(f"⚠️ LLM 调用失败: {e}") + llm_output = '{"datasource": "web_search"}' + + return {"question": question, "llm_output": llm_output} + + +# ============================================================ +# RouteSplitter: 使用 FlatMap 替代 side_output 打标签 +# ============================================================ + + +class RouteSplitter(FlatMapFunction): + """ + 路由分流器：根据 LLM 输出判断是走 vectorstore 还是 web_search。 + + 替代旧版 side_output 的实现： + - 旧版: self.out.collect(data, "vector") / self.out.collect(data, "web") + - 新版: 返回带 route 标签的数据，下游用 Filter 分流 + """ + + def execute(self, data: dict) -> list[dict]: + question = data["question"] + llm_output = data["llm_output"] + + print(f"🔀 RouteSplitter 收到: {llm_output}") + + # 解析路由决策 + if "vectorstore" in llm_output.lower(): + route = "vector" + else: + route = "web" + + print(f" → 路由决策: {route}") + + # 返回带路由标签的数据（替代 side_output） + return [{"question": question, "route": route}] + + +# ============================================================ +# Filter: 分流过滤器（替代 side_output） +# ============================================================ + + +class VectorRouteFilter(FilterFunction): + """过滤出走向量库的请求（替代 query_stream.side_output("vector")）""" + + def execute(self, data: dict) -> bool: + return data.get("route") == "vector" + + +class WebRouteFilter(FilterFunction): + """过滤出走 Web 搜索的请求（替代 query_stream.side_output("web")）""" + + def execute(self, data: dict) -> bool: + return data.get("route") == "web" + + +# ============================================================ +# DenseRetriever: 向量库检索（使用 EmbeddingClientAdapter） +# ============================================================ + + +class DenseRetriever(MapFunction): + """ + 向量库检索器（使用 EmbeddingClientAdapter）。 + 对应旧版 DenseRetriever + """ + + def execute(self, data: dict) -> dict: + question = data["question"] + print(f"🔍 [向量库] 检索: {question}") + + # 使用简单向量库 - search 方法内部会计算 embedding + collection = get_vector_collection() + results = collection.search(question, top_k=3) + context = "\n\n".join(results) + + print(f" → 检索到 {len(results)} 条结果") + return {"question": question, "context": context, "source": "知识库"} + + +# ============================================================ +# QAPromptor: 构造 QA Prompt +# ============================================================ + + +class QAPromptor(MapFunction): + """ + 构造 QA Prompt。 + 对应旧版 QAPromptor + """ + + def execute(self, data: dict) -> dict: + question = data["question"] + context = data["context"] + source = data["source"] + + prompt = QA_PROMPT_TEMPLATE.format(context=context, question=question) + return { + "question": question, + "messages": [{"role": "user", "content": prompt}], + "source": source, + } + + +# ============================================================ +# QAGenerator: 生成最终回答（使用 UnifiedInferenceClient） +# ============================================================ + + +class QAGenerator(MapFunction): + """ + 生成最终回答（使用 LLMClientAdapter）。 + 对应旧版 OpenAIGenerator 在 QA 阶段的使用 + """ + + def execute(self, data: dict) -> dict: + messages = data["messages"] + question = data["question"] + source = data["source"] + + client = get_llm_client() + try: + response = client.chat(messages, temperature=0.7, max_tokens=500) + answer = response.content if hasattr(response, "content") else str(response) + except Exception as e: + answer = f"生成回答时出错: {e}" + + print(f"🤖 生成回答完成 (来源: {source})") + return {"question": question, "answer": answer, "source": source} + + +# ============================================================ +# WebSearchAgent: Web 搜索代理 +# ============================================================ + + +class WebSearchAgent(MapFunction): + """ + Web 搜索代理（使用 LLMClientAdapter）。 + 对应旧版 BaseAgent + """ + + def execute(self, data: dict) -> dict: + question = data["question"] + print(f"🌐 [Web搜索] 搜索: {question}") + + # 检查是否有博查 API Key + bocha_key = os.environ.get("BOCHA_API_KEY") + if bocha_key: + answer = self._bocha_search(question, bocha_key) + else: + # 无 API 时使用 LLM 直接回答 + client = get_llm_client() + try: + response = client.chat( + [{"role": "user", "content": question}], + temperature=0.7, + max_tokens=500, + ) + answer = response.content if hasattr(response, "content") else str(response) + except Exception as e: + answer = f"回答生成失败: {e}" + + print("🤖 Web 回答完成") + return {"question": question, "answer": answer, "source": "Web搜索/LLM直答"} + + def _bocha_search(self, question: str, api_key: str) -> str: + """调用博查搜索 API""" + try: + import requests + + resp = requests.post( + "https://api.bocha.com/v1/search", + headers={"Authorization": f"Bearer {api_key}"}, + json={"query": question, "count": 3}, + timeout=10, + ) + if resp.ok: + data = resp.json() + results = data.get("results", []) + if results: + context = "\n\n".join([r.get("snippet", "") for r in results[:3]]) + # 用 LLM 生成回答 + client = get_llm_client() + response = client.chat( + [ + { + "role": "user", + "content": QA_PROMPT_TEMPLATE.format( + context=context, question=question + ), + } + ], + temperature=0.7, + max_tokens=500, + ) + return response.content if hasattr(response, "content") else str(response) + return f"未找到关于'{question}'的搜索结果。" + except Exception as e: + return f"Web 搜索失败: {e}" + + +# ============================================================ +# Sink: 结果输出（与旧版 TerminalSink 一致） +# ============================================================ + + +class TerminalSink(SinkFunction): + """ + 终端输出 Sink。 + 对应旧版 TerminalSink + """ + + def execute(self, data: dict): + question = data.get("question", "") + answer = data.get("answer", "") + source = data.get("source", "未知") + + print(f"\n{'─' * 60}") + print(f"❓ 问题: {question}") + print(f"📚 来源: {source}") + print(f"💬 回答: {answer}") + print(f"{'─' * 60}\n") + + +# ============================================================ +# 主程序：完全保留旧版的双分支结构 +# ============================================================ + + +def run_adaptive_rag_v2(): + """ + 运行 Adaptive RAG v2 流水线 + + 保留旧版的双分支结构： + - 向量库分支: RoutePrompt → LLMGenerator → RouteSplitter + → Filter(vector) → DenseRetriever → QAPromptor → QAGenerator → Sink + - Web 分支: RoutePrompt → LLMGenerator → RouteSplitter + → Filter(web) → WebSearchAgent → Sink + """ + print("🚀 启动 Adaptive RAG v2 系统") + print(f"📊 LLM 服务: {LLM_BASE_URL} ({LLM_MODEL})") + print(f"📊 Embedding 服务: {EMBEDDING_BASE_URL} ({EMBEDDING_MODEL})") + print("📊 流程: 问题 → 路由判断 → [向量库分支 | Web分支] → 回答 → 输出") + print("=" * 60) + + # 预初始化组件 + print("\n📦 初始化组件...") + get_llm_client() + get_embedding_client() + get_vector_collection() + print() + + # 创建环境 + env = LocalEnvironment("adaptive_rag_v2") + + # 预设问题列表 + questions = [ + "武汉有哪些著名景点？", # → vectorstore + "今天北京的天气怎么样？", # → web_search + "神农架有什么好玩的？", # → vectorstore + "Python 有哪些常用的 Web 框架？", # → web_search + ] + + # ======================================== + # 构建主流程（与旧版结构一致） + # ======================================== + + # 主 Query 路由流程 + # 旧版: env.from_source(FileSource).map(RoutePromptFunction).map(OpenAIGenerator).map(RouteSplitter) + query_stream = ( + env.from_source(QuestionSource, questions) + .map(RoutePromptFunction) # 构造路由 prompt + .map(LLMGenerator) # LLM 判断路由 + .flatmap(RouteSplitter) # 打上路由标签（替代 side_output） + ) + + # ======================================== + # 向量库分支（替代 query_stream.side_output("vector")） + # ======================================== + # 旧版: + # query_stream.side_output("vector") + # .map(DenseRetriever) + # .map(QAPromptor) + # .map(OpenAIGenerator) + # .sink(TerminalSink) + _vector_stream = ( + query_stream.filter(VectorRouteFilter) # 替代 .side_output("vector") + .map(DenseRetriever) # 向量检索 + .map(QAPromptor) # 构造 QA prompt + .map(QAGenerator) # 生成回答 + .sink(TerminalSink) # 输出 + ) + + # ======================================== + # Web 搜索分支（替代 query_stream.side_output("web")） + # ======================================== + # 旧版: + # query_stream.side_output("web") + # .map(BaseAgent) + # .map(TerminalSink) + _web_stream = ( + query_stream.filter(WebRouteFilter) # 替代 .side_output("web") + .map(WebSearchAgent) # Web 搜索 + LLM 回答 + .sink(TerminalSink) # 输出 + ) + + # 运行 + try: + env.submit() + time.sleep(15) # 等待处理完成 + print("\n✅ Adaptive RAG v2 处理完成") + except Exception as e: + print(f"❌ 处理出错: {e}") + import traceback + + traceback.print_exc() + finally: + env.close() + + +if __name__ == "__main__": + # 检查是否在测试模式下运行 + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + print("🧪 Test mode detected - adaptive_rag_v2 example") + print("✅ Test passed: Example structure validated") + sys.exit(0) + + CustomLogger.disable_global_console_debug() + load_dotenv(override=False) + run_adaptive_rag_v2() + + +# ============================================================ +# Pipeline 拓扑图 +# ============================================================ +# +# ┌───────────────┐ +# │ QuestionSource│ +# └───────┬───────┘ +# │ +# ▼ +# ┌─────────────────┐ +# │RoutePromptFunction│ +# └────────┬────────┘ +# │ +# ▼ +# ┌─────────────────┐ +# │ LLMGenerator │ ─────► LLM :8903 +# └────────┬────────┘ +# │ +# ▼ +# ┌─────────────────┐ +# │ RouteSplitter │ (FlatMap, 替代 side_output) +# └────────┬────────┘ +# │ +# ┌───────────────┴───────────────┐ +# │ │ +# ▼ ▼ +# ┌─────────────────┐ ┌─────────────────┐ +# │VectorRouteFilter│ │ WebRouteFilter │ +# │ route="vector" │ │ route="web" │ +# └────────┬────────┘ └────────┬────────┘ +# │ │ +# ▼ ▼ +# ┌─────────────────┐ ┌─────────────────┐ +# │ DenseRetriever │ │ WebSearchAgent │ +# │ │ │ │ +# │ Embedding :8090 │ │ LLM :8903 │ +# │ SimpleVectorDB │ └────────┬────────┘ +# └────────┬────────┘ │ +# │ │ +# ▼ │ +# ┌─────────────────┐ │ +# │ QAPromptor │ │ +# └────────┬────────┘ │ +# │ │ +# ▼ │ +# ┌─────────────────┐ │ +# │ QAGenerator │ ─────► LLM :8903 │ +# └────────┬────────┘ │ +# │ │ +# ▼ ▼ +# ┌─────────────────┐ ┌─────────────────┐ +# │ TerminalSink │ │ TerminalSink │ +# │ (知识库回答) │ │ (Web/LLM回答) │ +# └─────────────────┘ └─────────────────┘ +# +# ============================================================ +# 远程服务 (11.11.11.7) +# ============================================================ +# :8903 Qwen/Qwen2.5-7B-Instruct LLM +# :8090 BAAI/bge-large-zh-v1.5 Embedding +# +# ============================================================ +# 核心变更: side_output → FlatMap + Filter +# ============================================================ +# 旧版: query_stream.side_output("vector") +# 新版: query_stream.flatmap(RouteSplitter).filter(VectorRouteFilter) +# ============================================================ diff --git a/tutorials/L3-libs/rag/advanced_rag_topology.py b/tutorials/L3-libs/rag/advanced_rag_topology.py new file mode 100644 index 0000000..7ee37d6 --- /dev/null +++ b/tutorials/L3-libs/rag/advanced_rag_topology.py @@ -0,0 +1,779 @@ +#!/usr/bin/env python3 +""" +Advanced RAG Topology - 完整 RAG 系统拓扑结构 +============================================= + +本示例展示如何基于 SAGE 框架构建一个包含以下组件的完整 RAG 系统： +- sage_flow: 向量流处理引擎（高性能数据流） +- sage_db: 向量数据库（文档检索） +- sage_tsdb: 时序数据库（对话历史、日志、指标） +- sage_refiner: 上下文压缩/精炼器 +- LLM: 推理引擎 + +拓扑结构图： +============ + +``` +┌─────────────────────────────────────────────────────────────────────────────────────┐ +│ SAGE RAG Pipeline 拓扑 │ +└─────────────────────────────────────────────────────────────────────────────────────┘ + + ┌───────────────┐ + │ sage_flow │ ← 向量批处理加速 (C++ 高性能) + │ (向量流引擎) │ + └───────┬───────┘ + │ 加速向量计算 + ┌───────────────┐ ┌───────────▼───┐ ┌───────────────┐ ┌───────────────┐ + │ Source │────▶│ Embedder │────▶│ Retriever │────▶│ Reranker │ + │ (问题输入) │ │ (向量编码) │ │ (sage_db) │ │ (重排序) │ + │ [文本] │ │ [文本→向量] │ │ [向量→文档] │ │ [文档重排] │ + └───────────────┘ └───────────────┘ └───────┬───────┘ └───────┬───────┘ + │ │ + ▼ │ + ┌───────────────┐ │ + │ sage_tsdb │ │ + │ (查询日志) │ │ + └───────────────┘ │ + ▼ + ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ + │ Sink │◀────│ Generator │◀────│ Promptor │◀────│ Refiner │ + │ (输出) │ │ (LLM) │ │ (提示构建) │ │ (上下文压缩) │ + │ [文本] │ │ [Prompt→回答] │ │ [组装Prompt] │ │ [文本→压缩文本]│ + └───────────────┘ └───────┬───────┘ └───────────────┘ └───────────────┘ + │ (sage_refiner) + ▼ + ┌───────────────┐ + │ sage_tsdb │ + │ (响应日志) │ + └───────────────┘ +``` + +各组件数据类型说明： +- sage_flow: 向量批处理引擎，加速 Embedder 的向量计算（输入文本批次，输出向量批次） +- sage_db: 向量数据库，输入查询向量，输出相关文档列表 +- sage_tsdb: 时序数据库，记录时间戳+指标数据 +- sage_refiner: 上下文压缩器，输入长文档文本，输出压缩后的文本（文本→文本） +- LLM: 推理引擎，输入 Prompt 文本，输出回答文本 + +数据流路径： +1. Source → 接收用户问题 [文本] +2. Embedder (+ sage_flow) → 将问题编码为向量 [文本→向量]，sage_flow 加速批量计算 +3. Retriever (sage_db) → 从向量数据库检索相关文档 [向量→文档列表] +4. sage_tsdb → 记录查询历史，用于分析和优化 +5. Reranker → 对检索结果重排序 [文档列表→排序后文档列表] +6. Refiner (sage_refiner) → 压缩上下文 [长文本→压缩文本]，控制 token 预算 +7. Promptor → 构建 LLM 提示 [压缩文本+问题→Prompt] +8. Generator (LLM) → 生成回答 [Prompt→回答] +9. Sink → 输出结果 +10. sage_tsdb → 记录响应日志和指标 + +层级分布 (遵循 SAGE 架构规范): +- L1 (common): 基础函数类 (SourceFunction, MapFunction, SinkFunction) +- L2 (platform): 平台服务 +- L3 (kernel/libs): Environment, DataStream API, Embedding, Retriever 基础 +- L4 (middleware): sage_db, sage_tsdb, sage_flow, sage_refiner (C++ 扩展) +- L5 (apps): 应用编排 +- L6 (cli/studio): 用户界面和可视化 +""" + +import time +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +import numpy as np +from dotenv import load_dotenv + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger + +# ================================================================================ +# 配置数据类 +# ================================================================================ + + +@dataclass +class RAGTopologyConfig: + """RAG 拓扑配置""" + + # 向量数据库配置 (sage_db) + db_config: dict[str, Any] = field( + default_factory=lambda: { + "collection_name": "rag_knowledge_base", + "dim": 384, # 嵌入维度 + "metric": "cosine", + "top_k": 10, + } + ) + + # 时序数据库配置 (sage_tsdb) + tsdb_config: dict[str, Any] = field( + default_factory=lambda: { + "enable_query_log": True, + "enable_response_log": True, + "retention_days": 30, + "metrics_interval_ms": 1000, + } + ) + + # 向量流处理配置 (sage_flow) + flow_config: dict[str, Any] = field( + default_factory=lambda: { + "batch_size": 32, + "enable_batching": True, + "timeout_ms": 5000, + } + ) + + # Refiner 配置 (上下文压缩) + refiner_config: dict[str, Any] = field( + default_factory=lambda: { + "algorithm": "simple", # simple, long_refiner, llmlingua2 + "budget": 4000, # token 预算 + "enable_cache": True, + } + ) + + # LLM 推理配置 + llm_config: dict[str, Any] = field( + default_factory=lambda: { + "model": "Qwen/Qwen2.5-7B-Instruct", + "temperature": 0.7, + "max_tokens": 512, + } + ) + + +# ================================================================================ +# 1. Source - 问题输入源 +# ================================================================================ + + +class QuestionSource(SourceFunction): + """ + 问题数据源 - 接收用户查询 + + 在实际应用中，可替换为： + - API 接口接收 + - 消息队列消费 + - 文件批量读取 + """ + + def __init__(self, questions: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.questions = questions or [ + "SAGE 框架支持哪些 LLM 后端?", + "如何在 SAGE 中实现分布式 Pipeline?", + "sage_db 和 sage_tsdb 有什么区别?", + ] + self.index = 0 + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + def execute(self, data=None): + if self.index >= len(self.questions): + return None + + question = self.questions[self.index] + self.index += 1 + + self.logger.info(f"📝 [Source] 发送问题 #{self.index}: {question}") + + return { + "query_id": f"q_{self.index}_{int(time.time())}", + "query": question, + "timestamp": datetime.now().isoformat(), + } + + +# ================================================================================ +# 2. Embedder - 向量编码器 (与 sage_flow 集成) +# ================================================================================ + + +class EmbeddingOperator(MapFunction): + """ + 向量嵌入算子 - 将文本编码为向量 + + 集成 sage_flow 进行高效向量流处理： + - 批量编码优化 + - GPU 加速（如果可用） + - 缓存机制 + """ + + def __init__(self, dim: int = 384, **kwargs): + super().__init__(**kwargs) + self.dim = dim + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + # 尝试加载真实的 Embedding 模型 + self._embedder = None + self._init_embedder() + + def _init_embedder(self): + """初始化嵌入模型""" + try: + from sage.common.components.sage_embedding import EmbeddingFactory + + self._embedder = EmbeddingFactory.create( + "hf", + model="BAAI/bge-small-zh-v1.5", + ) + self.logger.info("✓ 已加载 HuggingFace Embedding 模型") + except Exception as e: + self.logger.warning(f"无法加载 Embedding 模型，使用模拟: {e}") + self._embedder = None + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + query = data["query"] + + if self._embedder: + # 使用真实模型编码 + query_vector = self._embedder.embed(query) + query_vector = np.array(query_vector) + else: + # 模拟向量编码（用于演示） + np.random.seed(hash(query) % 2**32) + query_vector = np.random.randn(self.dim).astype(np.float32) + query_vector = query_vector / np.linalg.norm(query_vector) + + data["query_vector"] = query_vector + self.logger.info(f"🔢 [Embedder] 向量编码完成, dim={len(query_vector)}") + + return data + + +# ================================================================================ +# 3. Retriever - 向量检索器 (基于 sage_db) +# ================================================================================ + + +class VectorRetriever(MapFunction): + """ + 向量检索算子 - 从 sage_db 检索相关文档 + + 核心功能： + - 基于 FAISS 的高效相似度搜索 + - 支持元数据过滤 + - 混合检索（向量 + 关键词） + """ + + def __init__(self, config: dict[str, Any] | None = None, **kwargs): + super().__init__(**kwargs) + self.config = config or {} + self.top_k = self.config.get("top_k", 5) + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + # 模拟知识库 + self._knowledge_base = self._init_knowledge_base() + self._db = None + self._init_db() + + def _init_knowledge_base(self) -> list[dict[str, Any]]: + """初始化模拟知识库""" + return [ + { + "id": "doc_1", + "content": "SAGE 支持多种 LLM 后端，包括 vLLM、OpenAI API、DashScope 等。通过 UnifiedInferenceClient 可以统一调用不同后端。", + "metadata": {"topic": "llm", "source": "docs"}, + }, + { + "id": "doc_2", + "content": "SAGE 基于 Ray 构建分布式执行能力。使用 RemoteEnvironment 可以在集群上运行 Pipeline，JobManager 负责任务调度。", + "metadata": {"topic": "distributed", "source": "docs"}, + }, + { + "id": "doc_3", + "content": "sage_db 是高性能向量数据库，基于 FAISS 实现，用于文档检索。sage_tsdb 是时序数据库，用于存储时间序列数据如监控指标、对话历史等。", + "metadata": {"topic": "database", "source": "docs"}, + }, + { + "id": "doc_4", + "content": "sage_flow 是向量流处理引擎，支持高效的批量向量运算。sage_refiner 提供上下文压缩功能，可以将长文档压缩到指定的 token 预算内。", + "metadata": {"topic": "components", "source": "docs"}, + }, + { + "id": "doc_5", + "content": "SAGE 的 dataflow 范式采用声明式 API：env.from_source().map().map().sink()。这种方式便于优化和分布式执行。", + "metadata": {"topic": "api", "source": "docs"}, + }, + ] + + def _init_db(self): + """尝试初始化 sage_db""" + try: + from sage.middleware.components.sage_db import SageDB + + self._db = SageDB(dim=self.config.get("dim", 384)) + self.logger.info("✓ 已初始化 sage_db") + except Exception as e: + self.logger.warning(f"无法初始化 sage_db，使用模拟检索: {e}") + self._db = None + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + query = data["query"] + _query_vector = data.get("query_vector") # 保留以备将来使用 + + # 简单的关键词匹配检索（演示用） + retrieved_docs = [] + for doc in self._knowledge_base: + # 计算简单的相关性分数 + score = sum(1 for word in query.split() if word in doc["content"]) + if score > 0: + retrieved_docs.append({"doc": doc, "score": score}) + + # 按分数排序 + retrieved_docs.sort(key=lambda x: x["score"], reverse=True) + retrieved_docs = retrieved_docs[: self.top_k] + + data["retrieved_documents"] = [item["doc"] for item in retrieved_docs] + data["retrieval_scores"] = [item["score"] for item in retrieved_docs] + + self.logger.info(f"🔍 [Retriever] 检索到 {len(retrieved_docs)} 篇相关文档") + + return data + + +# ================================================================================ +# 4. TSDB Logger - 时序数据记录 (基于 sage_tsdb) +# ================================================================================ + + +class TSDBLogger(MapFunction): + """ + 时序数据记录算子 - 使用 sage_tsdb 记录查询和响应 + + 记录内容： + - 查询时间戳和内容 + - 检索延迟 + - 生成延迟 + - 响应质量指标 + """ + + def __init__(self, config: dict[str, Any] | None = None, **kwargs): + super().__init__(**kwargs) + self.config = config or {} + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + self._tsdb = None + self._init_tsdb() + + def _init_tsdb(self): + """尝试初始化 sage_tsdb""" + try: + from sage.middleware.components.sage_tsdb import SageTSDB + + self._tsdb = SageTSDB() + self.logger.info("✓ 已初始化 sage_tsdb") + except Exception as e: + self.logger.warning(f"无法初始化 sage_tsdb，使用内存记录: {e}") + self._tsdb = None + + # 内存中的备用日志 + self._memory_log: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + # 记录时序数据 + log_entry = { + "timestamp": int(time.time() * 1000), # 毫秒时间戳 + "query_id": data.get("query_id"), + "query": data.get("query"), + "num_retrieved": len(data.get("retrieved_documents", [])), + "stage": "retrieval_complete", + } + + if self._tsdb: + try: + self._tsdb.insert( + metric_name="rag_queries", + timestamp=log_entry["timestamp"], + value=1.0, + tags={"query_id": log_entry["query_id"]}, + fields=log_entry, + ) + except Exception as e: + self.logger.warning(f"TSDB 写入失败: {e}") + + self._memory_log.append(log_entry) + self.logger.info(f"📊 [TSDB] 记录查询日志: {data.get('query_id')}") + + return data + + +# ================================================================================ +# 5. Reranker - 重排序器 +# ================================================================================ + + +class DocumentReranker(MapFunction): + """ + 文档重排序算子 + + 使用交叉编码器或其他重排序模型对检索结果进行精排。 + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + documents = data.get("retrieved_documents", []) + query = data.get("query", "") + + if not documents: + return data + + # 模拟重排序（实际应使用 BGE Reranker 等模型） + reranked_docs = [] + for doc in documents: + # 简单的相关性评分 + content = doc.get("content", "") + score = sum(1 for word in query.split() if word.lower() in content.lower()) + reranked_docs.append({"doc": doc, "rerank_score": score * 1.5}) + + reranked_docs.sort(key=lambda x: x["rerank_score"], reverse=True) + + data["retrieved_documents"] = [item["doc"] for item in reranked_docs] + data["rerank_scores"] = [item["rerank_score"] for item in reranked_docs] + + self.logger.info( + f"📋 [Reranker] 重排序完成, top doc score: {reranked_docs[0]['rerank_score'] if reranked_docs else 0}" + ) + + return data + + +# ================================================================================ +# 6. Refiner - 上下文压缩器 (基于 sage_refiner) +# ================================================================================ + + +class ContextRefiner(MapFunction): + """ + 上下文压缩算子 - 使用 sage_refiner 压缩检索到的文档 + + 支持多种压缩算法： + - simple: 简单截断 + - long_refiner: LongRefiner 算法 + - llmlingua2: LLMLingua2 快速压缩 + """ + + def __init__(self, config: dict[str, Any] | None = None, **kwargs): + super().__init__(**kwargs) + self.config = config or {} + self.budget = self.config.get("budget", 4000) + self.algorithm = self.config.get("algorithm", "simple") + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + self._refiner = None + self._init_refiner() + + def _init_refiner(self): + """初始化 refiner""" + try: + from sage.middleware.components.sage_refiner import RefinerService + + self._refiner = RefinerService( + config={ + "algorithm": self.algorithm, + "budget": self.budget, + } + ) + self.logger.info(f"✓ 已初始化 sage_refiner (algorithm={self.algorithm})") + except Exception as e: + self.logger.warning(f"无法初始化 sage_refiner，使用简单截断: {e}") + self._refiner = None + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + documents = data.get("retrieved_documents", []) + query = data.get("query", "") + + if not documents: + data["refined_context"] = "" + return data + + # 合并文档内容 + full_context = "\n\n".join(doc.get("content", "") for doc in documents) + + if self._refiner: + try: + result = self._refiner.refine(query=query, documents=[full_context]) + refined_context = result.get("compressed_text", full_context) + except Exception as e: + self.logger.warning(f"Refiner 压缩失败: {e}") + refined_context = full_context[: self.budget] + else: + # 简单截断 + refined_context = full_context[: self.budget] + + data["refined_context"] = refined_context + data["original_length"] = len(full_context) + data["refined_length"] = len(refined_context) + + compression_ratio = len(refined_context) / max(len(full_context), 1) + self.logger.info(f"🗜️ [Refiner] 压缩完成, ratio: {compression_ratio:.2%}") + + return data + + +# ================================================================================ +# 7. Promptor - 提示构建器 +# ================================================================================ + + +class RAGPromptor(MapFunction): + """ + RAG 提示构建算子 + + 将查询和压缩后的上下文组合成 LLM 可用的提示。 + """ + + def __init__(self, template: str | None = None, **kwargs): + super().__init__(**kwargs) + self.template = ( + template + or """请根据以下背景信息回答用户问题。 + +背景信息： +{context} + +用户问题：{query} + +请给出准确、简洁的回答：""" + ) + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + query = data.get("query", "") + context = data.get("refined_context", "") + + prompt = self.template.format(context=context, query=query) + data["prompt"] = prompt + + self.logger.info(f"📝 [Promptor] 提示构建完成, length: {len(prompt)}") + + return data + + +# ================================================================================ +# 8. Generator - LLM 推理引擎 +# ================================================================================ + + +class LLMGenerator(MapFunction): + """ + LLM 生成算子 - 使用 SAGE 的统一推理客户端 + + 支持多种后端： + - vLLM (本地部署) + - OpenAI API + - DashScope + """ + + def __init__(self, config: dict[str, Any] | None = None, **kwargs): + super().__init__(**kwargs) + self.config = config or {} + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + self._client = None + self._init_client() + + def _init_client(self): + """初始化 LLM 客户端""" + try: + from sage.common.components.sage_llm import UnifiedInferenceClient + + self._client = UnifiedInferenceClient.create() + self.logger.info("✓ 已初始化 UnifiedInferenceClient") + except Exception as e: + self.logger.warning(f"无法初始化 LLM 客户端，使用模拟回答: {e}") + self._client = None + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + prompt = data.get("prompt", "") + query = data.get("query", "") + + start_time = time.time() + + if self._client: + try: + response = self._client.chat( + messages=[{"role": "user", "content": prompt}], + temperature=self.config.get("temperature", 0.7), + max_tokens=self.config.get("max_tokens", 512), + ) + answer = response.get("content", "") + except Exception as e: + self.logger.warning(f"LLM 调用失败: {e}") + answer = self._mock_answer(query, data.get("refined_context", "")) + else: + answer = self._mock_answer(query, data.get("refined_context", "")) + + latency = time.time() - start_time + + data["answer"] = answer + data["generation_latency"] = latency + + self.logger.info(f"🤖 [Generator] 生成完成, latency: {latency:.2f}s") + + return data + + def _mock_answer(self, query: str, context: str) -> str: + """模拟 LLM 回答""" + if "LLM" in query or "后端" in query: + return "SAGE 支持多种 LLM 后端，包括 vLLM（本地高性能推理）、OpenAI API、DashScope 等。通过 UnifiedInferenceClient 可以统一调用。" + elif "分布式" in query: + return "SAGE 基于 Ray 构建分布式能力。使用 RemoteEnvironment 在集群运行 Pipeline，JobManager 负责调度。" + elif "sage_db" in query or "sage_tsdb" in query or "区别" in query: + return "sage_db 是向量数据库，用于文档检索；sage_tsdb 是时序数据库，用于存储时间序列数据如监控指标和对话历史。" + else: + return f"根据提供的信息，这个问题涉及：{context[:100]}..." + + +# ================================================================================ +# 9. Response TSDB Logger - 响应日志记录 +# ================================================================================ + + +class ResponseTSDBLogger(MapFunction): + """记录响应到时序数据库""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.logger = CustomLogger.get_logger(self.__class__.__name__) + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + # 记录响应指标 + log_entry = { + "timestamp": int(time.time() * 1000), + "query_id": data.get("query_id"), + "answer_length": len(data.get("answer", "")), + "generation_latency": data.get("generation_latency", 0), + "compression_ratio": data.get("refined_length", 0) + / max(data.get("original_length", 1), 1), + } + + self.logger.info(f"📊 [TSDB] 记录响应指标: latency={log_entry['generation_latency']:.2f}s") + + data["response_metrics"] = log_entry + return data + + +# ================================================================================ +# 10. Sink - 结果输出 +# ================================================================================ + + +class RAGResultSink(SinkFunction): + """ + RAG 结果输出 + + 支持多种输出方式： + - 终端打印 + - 文件保存 + - API 回调 + """ + + def __init__(self, output_file: str | None = None, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.logger = CustomLogger.get_logger(self.__class__.__name__) + self.results: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]): + query = data.get("query", "") + answer = data.get("answer", "") + metrics = data.get("response_metrics", {}) + + # 格式化输出 + print("\n" + "=" * 70) + print(f"❓ 问题: {query}") + print("-" * 70) + print(f"💬 回答: {answer}") + print("-" * 70) + print( + f"📊 指标: latency={metrics.get('generation_latency', 0):.2f}s, compression={metrics.get('compression_ratio', 0):.2%}" + ) + print("=" * 70 + "\n") + + self.results.append(data) + self.logger.info(f"✅ [Sink] 结果输出完成: {data.get('query_id')}") + + +# ================================================================================ +# Pipeline 构建与执行 +# ================================================================================ + + +def build_rag_topology(config: RAGTopologyConfig | None = None): + """ + 构建 RAG 拓扑 + + 拓扑结构： + Source → Embedder → Retriever → TSDBLogger → Reranker + → Refiner → Promptor → Generator → ResponseLogger → Sink + """ + from sage.kernel.api.local_environment import LocalEnvironment + + config = config or RAGTopologyConfig() + + # 创建执行环境 + env = LocalEnvironment("AdvancedRAGTopology") + + # 构建 dataflow pipeline + ( + env.from_source(QuestionSource) + .map(EmbeddingOperator, dim=config.db_config.get("dim", 384)) + .map(VectorRetriever, config=config.db_config) + .map(TSDBLogger, config=config.tsdb_config) + .map(DocumentReranker) + .map(ContextRefiner, config=config.refiner_config) + .map(RAGPromptor) + .map(LLMGenerator, config=config.llm_config) + .map(ResponseTSDBLogger) + .sink(RAGResultSink) + ) + + return env + + +def main(): + """运行 RAG 拓扑演示""" + load_dotenv() + + print("\n" + "=" * 70) + print("🚀 SAGE Advanced RAG Topology Demo") + print("=" * 70 + "\n") + + # 配置 + config = RAGTopologyConfig( + db_config={ + "collection_name": "rag_demo", + "dim": 384, + "top_k": 5, + }, + refiner_config={ + "algorithm": "simple", + "budget": 2000, + }, + llm_config={ + "model": "Qwen/Qwen2.5-7B-Instruct", + "temperature": 0.7, + "max_tokens": 256, + }, + ) + + # 构建拓扑 + env = build_rag_topology(config) + + # 执行 + print("📦 开始执行 RAG Pipeline...\n") + env.submit(autostop=True) + + print("\n✅ RAG Pipeline 执行完成.") + print("=" * 70 + "\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/rag/qa_local_llm.py b/tutorials/L3-libs/rag/qa_local_llm.py new file mode 100644 index 0000000..3289b66 --- /dev/null +++ b/tutorials/L3-libs/rag/qa_local_llm.py @@ -0,0 +1,143 @@ +""" +终端交互式QA无界流处理 - 本地版本 +支持终端输入问题，使用本地大模型生成回答的无界流处理示例 +""" + +import time + +from dotenv import load_dotenv + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.config.loader import load_config +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment +from sage.middleware.operators.rag import HFGenerator, QAPromptor + + +class TerminalInputSource(SourceFunction): + """终端输入源函数 - 简化版""" + + def execute(self, data=None): + try: + user_input = input().strip() + if user_input: + return user_input + return self.execute(data) + except (EOFError, KeyboardInterrupt): + raise + + +class QuestionProcessor(MapFunction): + """问题处理器""" + + def execute(self, data): + if not data or data.strip() == "": + return None + + question = data.strip() + return question + + +class AnswerFormatter(MapFunction): + """回答格式化器""" + + def execute(self, data): + if not data: + return None + + # HFGenerator返回的格式是 (user_query, generated_text) + if isinstance(data, tuple) and len(data) >= 2: + user_query = data[0] + answer = data[1] + return { + "question": user_query if user_query else "N/A", + "answer": answer, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + else: + return { + "question": "N/A", + "answer": str(data), + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + + +class ConsoleSink(SinkFunction): + """控制台输出""" + + def execute(self, data): + if not data: + return None + + if isinstance(data, dict): + print(f"\n🤖 {data.get('answer', 'N/A')}\n") + else: + print(f"\n🤖 {data}\n") + + return data + + +def create_qa_pipeline(): + """创建QA处理管道 - 使用本地模型""" + import os + + # 加载配置 + load_dotenv(override=False) + config_path = os.path.join( + os.path.dirname(__file__), "..", "config", "config_source_local.yaml" + ) + + # 检查配置文件是否存在 + if not os.path.exists(config_path): + print(f"❌ 配置文件不存在: {config_path}") + return + + config = load_config(config_path) + + # 创建本地环境 + env = LocalEnvironment() + + # 启动欢迎提示 + print("💬 QA助手已启动（本地模式）！输入问题后按回车") + + try: + # 构建无界流处理管道 - 使用本地生成器 + ( + env.from_source(TerminalInputSource) + .map(QuestionProcessor) + .map(QAPromptor, config["promptor"]) + .map(HFGenerator, config["generator"]["local"]) + .map(AnswerFormatter) + .sink(ConsoleSink) + ) + + # 提交并运行 + env.submit() + # 保持主线程运行，直到用户退出 + while True: + time.sleep(1) + + except Exception as e: + print(f"❌ 管道运行出错: {str(e)}") + finally: + try: + env.close() + print("✅ QA流处理管道已关闭") + except Exception: + pass + + +if __name__ == "__main__": + import os + import sys + + # 检查是否在测试模式下运行 + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + print("🧪 Test mode detected - qa_without_retrieval_local is interactive") + print("✅ Test passed: Interactive example structure validated") + sys.exit(0) + + CustomLogger.disable_global_console_debug() + create_qa_pipeline() diff --git a/tutorials/L3-libs/rag/qa_no_retrieval.py b/tutorials/L3-libs/rag/qa_no_retrieval.py new file mode 100644 index 0000000..4395ede --- /dev/null +++ b/tutorials/L3-libs/rag/qa_no_retrieval.py @@ -0,0 +1,135 @@ +""" +终端交互式QA无界流处理 +支持终端输入问题，使用大模型生成回答的无界流处理示例 +""" + +import time + +from dotenv import load_dotenv + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.config.loader import load_config +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment +from sage.middleware.operators.rag import OpenAIGenerator, QAPromptor + + +class TerminalInputSource(SourceFunction): + """终端输入源函数 - 简化版""" + + def execute(self, data=None): + try: + user_input = input().strip() + if user_input: + return user_input + return self.execute(data) + except (EOFError, KeyboardInterrupt): + raise + + +class QuestionProcessor(MapFunction): + """问题处理器""" + + def execute(self, data): + if not data or data.strip() == "": + return None + + question = data.strip() + return question + + +class AnswerFormatter(MapFunction): + """回答格式化器""" + + def execute(self, data): + if not data: + return None + + # OpenAIGenerator返回的格式是 (user_query, generated_text) + if isinstance(data, tuple) and len(data) >= 2: + user_query = data[0] + answer = data[1] + return { + "question": user_query if user_query else "N/A", + "answer": answer, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + else: + return { + "question": "N/A", + "answer": str(data), + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + } + + +class ConsoleSink(SinkFunction): + """控制台输出""" + + def execute(self, data): + if not data: + return None + + if isinstance(data, dict): + print(f"\n🤖 {data.get('answer', 'N/A')}\n") + else: + print(f"\n🤖 {data}\n") + + return data + + +def create_qa_pipeline(): + """创建QA处理管道""" + import os + + # 加载配置 + load_dotenv(override=False) + config_path = os.path.join(os.path.dirname(__file__), "..", "config", "config_source.yaml") + config = load_config(config_path) + + # 创建本地环境 + env = LocalEnvironment() + + # 启动欢迎提示 + print("💬 QA助手已启动！输入问题后按回车") + + try: + # 构建无界流处理管道 + ( + env.from_source(TerminalInputSource) + .map(QuestionProcessor) + .map(QAPromptor, config["promptor"]) + .map(OpenAIGenerator, config["generator"]["vllm"]) + .map(AnswerFormatter) + .sink(ConsoleSink) + ) + + # 提交并运行 + env.submit() + # 保持主线程运行，直到用户退出 + while True: + time.sleep(1) + + except Exception as e: + print(f"❌ 管道运行出错: {str(e)}") + finally: + try: + env.close() + print("✅ QA流处理管道已关闭") + except Exception: + pass + + +if __name__ == "__main__": + import os + import sys + + # 检查是否在测试模式下运行 + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + print("🧪 Test mode detected - qa_without_retrieval is interactive") + print("✅ Test passed: Interactive example structure validated") + sys.exit(0) + + CustomLogger.disable_global_console_debug() + create_qa_pipeline() diff --git a/tutorials/L3-libs/rag/simple_rag.py b/tutorials/L3-libs/rag/simple_rag.py new file mode 100644 index 0000000..1d4cd6d --- /dev/null +++ b/tutorials/L3-libs/rag/simple_rag.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +""" +简化版RAG应用 - 测试完整流程 +用于验证问题源→检索→生成→输出的完整数据流 + +支持 RemoteEnvironment + LocalSinkScheduler： +- 计算任务在远程节点执行 +- Sink 节点绑定到本地（客户端），输出可见 +""" + +import os +import socket +import sys +import time + +from dotenv import load_dotenv + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment +from sage.kernel.api.remote_environment import RemoteEnvironment +from sage.kernel.scheduler.api import BaseScheduler +from sage.kernel.scheduler.decision import PlacementDecision + + +class SimpleQuestionSource(SourceFunction): + """简单问题源：只发送一个问题进行测试""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.sent = False + + def execute(self, data=None): + if self.sent: + return None + self.sent = True + question = "张先生的手机通常放在什么地方？" + print(f"📝 发送问题: {question}") + return question + + +class SimpleRetriever(MapFunction): + """简化的检索器""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # 模拟知识库数据 + self.knowledge = { + "张先生的手机": "张先生习惯把手机放在办公桌右上角的充电座上", + "李女士的钱包": "李女士总是把钱包放在卧室梳妆台的第一个抽屉里", + "王经理的钥匙": "王经理的办公室钥匙通常挂在衣帽架上的西装口袋里", + } + + def execute(self, data): + question = data + print(f"🔍 检索问题: {question}") + + # 简单的关键词匹配 + relevant_info = [] + for key, value in self.knowledge.items(): + if any(word in question for word in key.split()): + relevant_info.append(value) + + context = "\n".join(relevant_info) if relevant_info else "没有找到相关信息" + result = {"query": question, "context": context} + print(f"✅ 检索结果: {context}") + return result + + +class SimplePromptor(MapFunction): + """简化的提示构建器""" + + def execute(self, data): + query = data["query"] + context = data["context"] + + prompt = f"""请根据以下背景信息回答问题： + +背景信息： +{context} + +问题：{query} + +请给出简洁准确的回答：""" + + result = {"query": query, "prompt": prompt} + print("✅ 构建提示完成") + return result + + +class SimpleGenerator(MapFunction): + """简化的AI生成器 - 使用模拟回答""" + + def execute(self, data): + query = data["query"] + data["prompt"] + + print("🤖 AI生成中...") + + # 模拟AI回答 + if "张先生" in query and "手机" in query: + answer = "根据提供的信息，张先生习惯把手机放在办公桌右上角的充电座上。" + elif "李女士" in query and "钱包" in query: + answer = "根据提供的信息，李女士总是把钱包放在卧室梳妆台的第一个抽屉里。" + elif "王经理" in query and "钥匙" in query: + answer = "根据提供的信息，王经理的办公室钥匙通常挂在衣帽架上的西装口袋里。" + else: + answer = "抱歉，我无法根据现有信息回答这个问题。" + + result = {"query": query, "answer": answer} + print(f"✅ AI生成完成: {answer}") + return result + + +class SimpleTerminalSink(SinkFunction): + """简化的终端输出""" + + def execute(self, data): + query = data["query"] + answer = data["answer"] + + print("\n" + "=" * 60) + print(f"❓ 问题: {query}") + print(f"💬 回答: {answer}") + print("=" * 60 + "\n") + + +class SimpleFileSink(SinkFunction): + """文件输出 - 结果写入文件，便于远程执行后查看""" + + def __init__(self, output_path: str = "/home/sage/SAGE/.sage/rag_output.txt", **kwargs): + super().__init__(**kwargs) + self.output_path = output_path + + def execute(self, data): + from datetime import datetime + + query = data["query"] + answer = data["answer"] + + # 构建输出内容 + output = { + "timestamp": datetime.now().isoformat(), + "query": query, + "answer": answer, + } + + # 追加写入文件 + with open(self.output_path, "a", encoding="utf-8") as f: + f.write("=" * 60 + "\n") + f.write(f"时间: {output['timestamp']}\n") + f.write(f"问题: {query}\n") + f.write(f"回答: {answer}\n") + f.write("=" * 60 + "\n\n") + + print(f"✅ 结果已写入: {self.output_path}") + + +# ============================================================ +# LocalSinkScheduler - 将 Sink 节点绑定到本地 +# ============================================================ + + +class LocalSinkScheduler(BaseScheduler): + """ + 本地 Sink 调度器：将 Sink 节点放到客户端本地执行 + + 工作原理： + - Sink 节点 → 绑定到本地（使用实际的 Ray 节点 ID） + - 其他节点 → 使用 Ray 默认负载均衡 + + 使用场景： + - RemoteEnvironment 远程执行计算 + - 但希望 Sink 输出在本地可见 + + 注意：需要在 Ray 集群环境中运行，会获取当前节点的真实 Ray node ID + """ + + def __init__(self): + super().__init__() + self.local_hostname = socket.gethostname() + self._local_node_id = None # 延迟获取 + + def _get_local_node_id(self): + """获取当前节点的 Ray node ID""" + if self._local_node_id is not None: + return self._local_node_id + + try: + import ray + + if not ray.is_initialized(): + # 如果 Ray 没有初始化，返回 None 使用默认调度 + return None + + # 获取当前节点的 node ID + current_node_id = ray.get_runtime_context().get_node_id() + self._local_node_id = current_node_id + return current_node_id + except Exception: + return None + + def make_decision(self, task_node): + """根据任务类型决定放置策略""" + # 导入放在方法内部，确保远程反序列化时可用 + + task_name = getattr(task_node, "name", str(task_node)) + + # 检查是否是 Sink 节点 + is_sink = "Sink" in task_name or "sink" in task_name.lower() + + if is_sink: + # 获取本地节点的真实 Ray node ID + local_node_id = self._get_local_node_id() + + if local_node_id: + # 使用真实的 Ray node ID + return PlacementDecision( + target_node=local_node_id, + placement_strategy="affinity", + reason=f"Sink bound to local node: {self.local_hostname} (node_id: {local_node_id[:8]}...)", + ) + else: + # 如果无法获取 node ID，使用默认调度 + return PlacementDecision( + placement_strategy="default", + reason="Sink: Could not get local node ID, using default scheduling", + ) + + # 其他任务使用默认调度 + return PlacementDecision( + placement_strategy="default", + reason="Default load balancing for compute tasks", + ) + + +def pipeline_run(): + """运行简化RAG管道""" + print("🚀 启动简化版RAG系统") + print("📊 流程: 问题源 → 简单检索 → 提示构建 → 模拟生成 → 终端输出") + print("=" * 60) + + # 选择环境模式 + USE_REMOTE = True # 设为 True 使用远程模式（需要先启动 JobManager） + + if USE_REMOTE: + # 远程模式：需要先启动 JobManager + # 运行: sage jobmanager start --host 0.0.0.0 --port 19001 + scheduler = LocalSinkScheduler() + print(f"📍 使用 LocalSinkScheduler，Sink 将在本地节点 ({scheduler.local_hostname}) 执行") + env = RemoteEnvironment( + "rag_simple_demo", + host="sage-node-1", + scheduler=scheduler, + ) + else: + # 本地模式：直接执行，无需额外服务 + print("📍 使用 LocalEnvironment 本地执行") + env = LocalEnvironment("rag_simple_demo") + + # 输出文件路径 + output_file = "/home/sage/SAGE/.sage/rag_output.txt" + + # 构建管道 + ( + env.from_source(SimpleQuestionSource) + .map(SimpleRetriever) + .map(SimplePromptor) + .map(SimpleGenerator) + .sink(SimpleFileSink, output_file) # 使用 FileSink + ) + + try: + print(f"🔄 开始处理... 结果将写入: {output_file}") + env.submit() + time.sleep(5) # 等待处理完成 + print("✅ 处理完成") + + # 显示输出文件内容 + if os.path.exists(output_file): + print(f"\n📄 输出文件内容 ({output_file}):") + with open(output_file, encoding="utf-8") as f: + print(f.read()) + + except Exception as e: + print(f"❌ 处理出错: {e}") + import traceback + + traceback.print_exc() + finally: + env.close() + + +if __name__ == "__main__": + # 检查是否在测试模式下运行 + if os.getenv("SAGE_EXAMPLES_MODE") == "test" or os.getenv("SAGE_TEST_MODE") == "true": + print("🧪 Test mode detected - rag_simple example") + print("✅ Test passed: Example structure validated") + sys.exit(0) + + CustomLogger.disable_global_console_debug() + load_dotenv(override=False) + pipeline_run() diff --git a/tutorials/L3-libs/rag/usage_1_direct_library.py b/tutorials/L3-libs/rag/usage_1_direct_library.py new file mode 100644 index 0000000..0978fae --- /dev/null +++ b/tutorials/L3-libs/rag/usage_1_direct_library.py @@ -0,0 +1,279 @@ +""" +Usage 1: Direct Library Usage +============================== + +最简单的方式：直接使用 unlearning 库，无需 SAGE 运行时。 + +适用场景： +- 独立脚本 +- Jupyter Notebook 实验 +- 快速原型验证 +- 研究算法开发 + +优势： +- 零依赖 SAGE 运行时 +- 代码简洁清晰 +- 易于调试 +- 适合快速实验 +""" + +import numpy as np + +from sage.libs.privacy.unlearning import UnlearningEngine +from sage.libs.privacy.unlearning.algorithms import LaplaceMechanism + + +def generate_test_data(n_vectors=50, dim=128): + """生成测试数据""" + vectors = np.random.randn(n_vectors, dim).astype(np.float32) + # L2 归一化（模拟真实 embeddings） + vectors = vectors / (np.linalg.norm(vectors, axis=1, keepdims=True) + 1e-10) + ids = [f"doc_{i}" for i in range(n_vectors)] + return vectors, ids + + +def example_basic_unlearning(): + """示例1：基础遗忘操作""" + print("=" * 70) + print("Example 1: Basic Unlearning") + print("=" * 70) + + # 1. 生成数据 + all_vectors, all_ids = generate_test_data(n_vectors=50, dim=128) + print(f"✓ Generated {len(all_vectors)} vectors") + + # 2. 选择要遗忘的向量 + forget_indices = [5, 10, 15, 20, 25] + vectors_to_forget = all_vectors[forget_indices] + ids_to_forget = [all_ids[i] for i in forget_indices] + print(f"✓ Selected {len(ids_to_forget)} vectors to forget: {ids_to_forget}") + + # 3. 创建 unlearning engine + engine = UnlearningEngine( + epsilon=1.0, delta=1e-5, total_budget_epsilon=10.0, enable_compensation=True + ) + print("✓ Created UnlearningEngine") + + # 4. 执行遗忘 + result = engine.unlearn_vectors( + vectors_to_forget=vectors_to_forget, + vector_ids_to_forget=ids_to_forget, + all_vectors=all_vectors, + all_vector_ids=all_ids, + perturbation_strategy="uniform", + ) + + # 5. 查看结果 + print("\n🎯 Unlearning Result:") + print(f" Success: {result.success}") + print(f" Vectors unlearned: {result.num_vectors_unlearned}") + print(f" Neighbors compensated: {result.num_neighbors_compensated}") + print(f" Privacy cost: ε={result.privacy_cost[0]:.4f}, δ={result.privacy_cost[1]:.6f}") + + # 6. 获取扰动后的向量 + perturbed = result.metadata["perturbed_vectors"] + print("\n📊 Vector Comparison:") + for i, (orig, pert, vec_id) in enumerate(zip(vectors_to_forget, perturbed, ids_to_forget)): + l2_dist = np.linalg.norm(orig - pert) + cos_sim = np.dot(orig, pert) / (np.linalg.norm(orig) * np.linalg.norm(pert)) + print(f" {vec_id}: L2={l2_dist:.4f}, CosSim={cos_sim:.4f}") + + print() + + +def example_custom_mechanism(): + """示例2：使用自定义隐私机制""" + print("=" * 70) + print("Example 2: Custom Privacy Mechanism") + print("=" * 70) + + # 1. 生成数据 + vectors, ids = generate_test_data(n_vectors=30, dim=64) + forget_vectors = vectors[:3] + forget_ids = ids[:3] + + # 2. 创建自定义 Laplace 机制 + custom_mechanism = LaplaceMechanism(epsilon=0.5) + print("✓ Created custom Laplace mechanism with ε=0.5") + + # 3. 使用自定义机制 + engine = UnlearningEngine( + mechanism=custom_mechanism, + total_budget_epsilon=5.0, + enable_compensation=False, # 不使用补偿 + ) + + result = engine.unlearn_vectors( + vectors_to_forget=forget_vectors, + vector_ids_to_forget=forget_ids, + perturbation_strategy="selective", + ) + + print("\n🎯 Result with custom mechanism:") + print(f" Success: {result.success}") + print(f" Privacy cost: ε={result.privacy_cost[0]:.4f}") + print() + + +def example_batch_unlearning(): + """示例3：批量遗忘操作""" + print("=" * 70) + print("Example 3: Batch Unlearning") + print("=" * 70) + + # 1. 生成数据 + all_vectors, all_ids = generate_test_data(n_vectors=100, dim=128) + + # 2. 创建 engine + engine = UnlearningEngine( + epsilon=0.5, delta=1e-5, total_budget_epsilon=20.0, enable_compensation=True + ) + + # 3. 分批遗忘 + batch_size = 5 + total_forgotten = 0 + + for batch_idx in range(3): # 遗忘3批 + start_idx = batch_idx * batch_size + end_idx = start_idx + batch_size + + forget_vectors = all_vectors[start_idx:end_idx] + forget_ids = all_ids[start_idx:end_idx] + + result = engine.unlearn_vectors( + vectors_to_forget=forget_vectors, + vector_ids_to_forget=forget_ids, + all_vectors=all_vectors, + all_vector_ids=all_ids, + perturbation_strategy="uniform", + ) + + total_forgotten += result.num_vectors_unlearned + print( + f" Batch {batch_idx + 1}: Forgotten {result.num_vectors_unlearned} vectors, " + f"Privacy cost: ε={result.privacy_cost[0]:.4f}" + ) + + # 4. 检查剩余预算 + status = engine.get_privacy_status() + remaining = status["remaining_budget"] + + print("\n📊 Summary:") + print(f" Total forgotten: {total_forgotten} vectors") + print(f" Remaining budget: ε={remaining['epsilon_remaining']:.4f}") + print(f" Budget utilization: {status['accountant_summary']['budget_utilization']:.1%}") + print() + + +def example_similarity_based_unlearning(): + """示例4：基于相似度的遗忘""" + print("=" * 70) + print("Example 4: Similarity-based Unlearning") + print("=" * 70) + + # 1. 生成数据 + all_vectors, all_ids = generate_test_data(n_vectors=80, dim=128) + + # 2. 创建一个查询向量（要遗忘的主题） + query_vector = np.random.randn(128).astype(np.float32) + query_vector = query_vector / np.linalg.norm(query_vector) + print("✓ Created query vector representing topic to forget") + + # 3. 创建 engine + engine = UnlearningEngine(epsilon=1.0, delta=1e-5) + + # 4. 遗忘所有相似的向量 + result = engine.unlearn_by_similarity( + query_vector=query_vector, + all_vectors=all_vectors, + all_vector_ids=all_ids, + similarity_threshold=0.3, # 相似度 > 0.3 的都遗忘 + max_unlearn=10, # 最多遗忘10个 + perturbation_strategy="adaptive", + ) + + print("\n🎯 Similarity-based Unlearning Result:") + print(f" Success: {result.success}") + print(f" Vectors forgotten: {result.num_vectors_unlearned}") + print(f" Privacy cost: ε={result.privacy_cost[0]:.4f}") + + if result.num_vectors_unlearned > 0: + result.metadata.get("perturbed_vectors", []) + print(f" Forgotten vector IDs: {result.metadata.get('message', 'N/A')}") + + print() + + +def example_privacy_budget_management(): + """示例5：隐私预算管理""" + print("=" * 70) + print("Example 5: Privacy Budget Management") + print("=" * 70) + + # 创建 engine 带有较小的总预算 + engine = UnlearningEngine( + epsilon=2.0, + delta=1e-5, + total_budget_epsilon=5.0, # 小预算 + enable_compensation=False, + ) + + vectors, ids = generate_test_data(n_vectors=50, dim=64) + + print("📊 Privacy Budget Tracking:") + print(f" Initial budget: ε={engine.privacy_accountant.total_epsilon_budget}") + + # 尝试多次操作直到预算耗尽 + operation_count = 0 + while True: + forget_idx = operation_count % len(vectors) + result = engine.unlearn_vectors( + vectors_to_forget=vectors[forget_idx : forget_idx + 1], + vector_ids_to_forget=[ids[forget_idx]], + perturbation_strategy="uniform", + ) + + operation_count += 1 + + if not result.success: + print(f"\n❌ Operation {operation_count} failed: {result.metadata.get('error')}") + print(f" Remaining budget: {result.metadata.get('remaining_budget')}") + break + else: + status = engine.get_privacy_status() + remaining = status["remaining_budget"] + print( + f" Operation {operation_count}: Success, " + f"Remaining ε={remaining['epsilon_remaining']:.4f}" + ) + + print() + + +def main(): + """运行所有示例""" + print("\n" + "=" * 70) + print("SAGE Unlearning Library - Direct Usage Examples") + print("=" * 70) + print("\n这些示例展示了如何直接使用 unlearning 库，无需 SAGE 运行时。") + print("适合：独立脚本、Jupyter 实验、快速原型验证\n") + + # 运行所有示例 + example_basic_unlearning() + example_custom_mechanism() + example_batch_unlearning() + example_similarity_based_unlearning() + example_privacy_budget_management() + + print("=" * 70) + print("✅ All examples completed successfully!") + print("=" * 70) + print("\n💡 Next steps:") + print(" 1. Try modifying the parameters (epsilon, delta, strategies)") + print(" 2. Implement your own privacy mechanism") + print(" 3. Test on real embeddings from your RAG system") + print(" 4. See usage_2_sage_function.py for SAGE integration\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/rag/usage_2_sage_function.py b/tutorials/L3-libs/rag/usage_2_sage_function.py new file mode 100644 index 0000000..6fe76ac --- /dev/null +++ b/tutorials/L3-libs/rag/usage_2_sage_function.py @@ -0,0 +1,226 @@ +""" +SAGE Unlearning - Function Integration + +This module demonstrates how to integrate unlearning with SAGE Functions. + +Note: This example shows architectural patterns. A complete Pipeline +requires a full SAGE runtime environment. For testing and quick verification, +use usage_1_direct_library.py. + +This shows: +1. How to wrap unlearning in a Function class +2. How to integrate with SAGE data processing +3. How to manage state in functions +4. How to compose multiple functions + +**For Students**: Study this to understand how to integrate unlearning +with SAGE's data processing framework. +""" + +import numpy as np + +from sage.libs.privacy.unlearning import UnlearningEngine + + +class UnlearningFunctionExample: + """ + Demonstrates wrapping unlearning logic in a function-like class. + + In a full SAGE Pipeline, this would inherit from BaseFunction. + Here we show the pattern without requiring the full runtime. + """ + + def __init__(self, epsilon=1.0): + """Initialize the function""" + self.engine = UnlearningEngine(total_budget_epsilon=epsilon, enable_compensation=True) + self.vectors_processed = 0 + self.vectors_forgotten = 0 + + def process_vector(self, vector_id, vector, should_forget=False): + """ + Process a single vector. + + In a real Pipeline, this would be the execute() method. + """ + self.vectors_processed += 1 + + if should_forget: + self.vectors_forgotten += 1 + # In a real scenario, this would be batched + # For demo, we just track it + return { + "action": "forgot", + "vector_id": vector_id, + "privacy_cost": 0.1, # Simplified + } + + return {"action": "kept", "vector_id": vector_id, "vector": vector} + + +def example_function_pattern(): + """Example 1: Function Pattern Demonstration""" + print("\n" + "=" * 70) + print("Example 1: Unlearning Function Pattern") + print("=" * 70) + + func = UnlearningFunctionExample(epsilon=1.0) + + # Simulate processing vectors + vectors = np.random.randn(10, 64).astype(np.float32) + vectors = vectors / (np.linalg.norm(vectors, axis=1, keepdims=True) + 1e-10) + + forget_indices = {0, 3, 7} # Which vectors to forget + + print(f"Processing {len(vectors)} vectors...") + results = [] + + for i, vector in enumerate(vectors): + should_forget = i in forget_indices + result = func.process_vector(f"doc_{i}", vector, should_forget) + results.append(result) + if should_forget: + print(f" - Vector {i}: Forgotten") + + print("\n📊 Summary:") + print(f" Vectors processed: {func.vectors_processed}") + print(f" Vectors forgotten: {func.vectors_forgotten}") + forgotten_count = sum(1 for r in results if r["action"] == "forgot") + print(f" Verified forgotten: {forgotten_count}") + + +def example_batched_unlearning(): + """Example 2: Batched Unlearning in Functions""" + print("\n" + "=" * 70) + print("Example 2: Batched Unlearning") + print("=" * 70) + + # Create function with batching + engine = UnlearningEngine(total_budget_epsilon=10.0) + + # Simulate batching vectors + all_vectors = np.random.randn(100, 128).astype(np.float32) + all_vectors = all_vectors / (np.linalg.norm(all_vectors, axis=1, keepdims=True) + 1e-10) + all_ids = [f"doc_{i}" for i in range(100)] + + # Forget vectors in batches + batch_size = 10 + for batch_idx in range(3): + start_idx = batch_idx * batch_size + end_idx = start_idx + batch_size + + forget_vectors = all_vectors[start_idx:end_idx] + forget_ids = all_ids[start_idx:end_idx] + + result = engine.unlearn_vectors( + vectors_to_forget=forget_vectors, + vector_ids_to_forget=forget_ids, + all_vectors=all_vectors, + all_vector_ids=all_ids, + perturbation_strategy="selective", + ) + + if result.success: + print( + f" Batch {batch_idx}: Forgotten {result.num_vectors_unlearned} vectors, " + f"Privacy cost: ε={result.privacy_cost[0]:.4f}" + ) + + +def example_stateful_processing(): + """Example 3: Stateful Vector Processing""" + print("\n" + "=" * 70) + print("Example 3: Stateful Vector Processing") + print("=" * 70) + + class StatefulProcessor: + """Processor that maintains state across calls""" + + def __init__(self): + self.state = { + "vectors_accumulated": [], + "ids_accumulated": [], + "total_forgotten": 0, + "privacy_spent": 0.0, + } + self.engine = UnlearningEngine(total_budget_epsilon=5.0) + + def add_vector(self, vector_id, vector): + self.state["vectors_accumulated"].append(vector) + self.state["ids_accumulated"].append(vector_id) + + def flush_and_forget(self, num_to_forget=3): + """Accumulate and then forget""" + if len(self.state["vectors_accumulated"]) < num_to_forget: + return None + + forget_vectors = np.array(self.state["vectors_accumulated"][:num_to_forget]) + forget_ids = self.state["ids_accumulated"][:num_to_forget] + all_vectors = np.array(self.state["vectors_accumulated"]) + all_ids = self.state["ids_accumulated"] + + result = self.engine.unlearn_vectors( + vectors_to_forget=forget_vectors, + vector_ids_to_forget=forget_ids, + all_vectors=all_vectors, + all_vector_ids=all_ids, + perturbation_strategy="uniform", + ) + + if result.success: + self.state["total_forgotten"] += result.num_vectors_unlearned + self.state["privacy_spent"] += result.privacy_cost[0] + # Clear processed vectors + self.state["vectors_accumulated"] = self.state["vectors_accumulated"][ + num_to_forget: + ] + self.state["ids_accumulated"] = self.state["ids_accumulated"][num_to_forget:] + + return result + + processor = StatefulProcessor() + + # Add vectors + for i in range(10): + vector = np.random.randn(64).astype(np.float32) + vector = vector / (np.linalg.norm(vector) + 1e-10) + processor.add_vector(f"doc_{i}", vector) + + print("Added 10 vectors to processor") + + # Flush and forget in batches + result = processor.flush_and_forget(num_to_forget=5) + if result: + print( + f"Batch 1: Forgotten {result.num_vectors_unlearned}, Privacy cost: ε={result.privacy_cost[0]:.4f}" + ) + + print("\n📊 Final state:") + print(f" Total vectors forgotten: {processor.state['total_forgotten']}") + print(f" Total privacy spent: ε={processor.state['privacy_spent']:.4f}") + print(f" Vectors still in buffer: {len(processor.state['vectors_accumulated'])}") + + +def main(): + """Run all examples""" + print("\n" + "=" * 70) + print("SAGE Unlearning Library - Function Integration Examples") + print("=" * 70) + print("\n这些示例展示了如何在 SAGE Function 中使用 unlearning 库。") + print("These examples show how to integrate unlearning with SAGE Functions.\n") + + # Run examples + example_function_pattern() + example_batched_unlearning() + example_stateful_processing() + + print("\n" + "=" * 70) + print("✅ All examples completed successfully!") + print("=" * 70) + print("\n💡 Next steps:") + print(" 1. Study the patterns shown here") + print(" 2. Implement custom data processing logic") + print(" 3. See usage_3_memory_service.py for service integration\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/rag/usage_3_memory_service.py b/tutorials/L3-libs/rag/usage_3_memory_service.py new file mode 100644 index 0000000..effc1e5 --- /dev/null +++ b/tutorials/L3-libs/rag/usage_3_memory_service.py @@ -0,0 +1,472 @@ +""" +Usage 3: MemoryService Integration +================================== + +将 unlearning 集成到 MemoryService 中。 + +适用场景： +- RAG 系统中的隐私遗忘 +- 需要从 VDB 中检索和更新向量 +- 与记忆管理系统集成 +- 完整的数据生命周期管理 + +优势： +- 与 VDB collection 无缝集成 +- 支持向量检索和更新 +- 遗忘操作的持久化 +- 隐私预算管理 +""" + +import os +from typing import Any + +import numpy as np + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.service.base_service import BaseService +from sage.libs.privacy.unlearning import UnlearningEngine +from sage.middleware.components.sage_mem.neuromem.memory_collection.vdb_collection import ( + VDBMemoryCollection, +) +from sage.middleware.components.sage_mem.neuromem.memory_manager import MemoryManager + + +class DPMemoryService(BaseService): + """ + 带差分隐私的内存服务 + + 支持使用 DP 遗忘操作从 VDB 中安全删除数据。 + """ + + def __init__(self, data_dir: str | None = None, epsilon: float = 1.0, delta: float = 1e-5): + super().__init__() + + # 初始化内存管理器 + if data_dir is None: + data_dir = os.path.join(os.getcwd(), "data", "dp_memory_service") + os.makedirs(data_dir, exist_ok=True) + + self.manager = MemoryManager(data_dir) + self.logger.info(f"Initialized DPMemoryService with data_dir={data_dir}") + + # 初始化 DP unlearning engine + self.unlearning_engine = UnlearningEngine( + epsilon=epsilon, + delta=delta, + total_budget_epsilon=100.0, + enable_compensation=True, + ) + + self.logger.info(f"Initialized UnlearningEngine with ε={epsilon}, δ={delta}") + + def create_collection(self, collection_name: str, config: dict | None = None) -> bool: + """创建 VDB collection""" + try: + if config is None: + config = { + "name": collection_name, + "backend_type": "VDB", + "description": f"DP-enabled collection: {collection_name}", + } + + collection = self.manager.create_collection(config) + + if collection is None: + self.logger.warning(f"Failed to create collection: {collection_name}") + return False + + # 创建默认索引 + index_config = { + "name": "global_index", + "embedding_model": "mockembedder", + "dim": 128, + "backend_type": "FAISS", + "description": "Global index for similarity search", + } + collection.create_index(index_config) # type: ignore[attr-defined] + # Note: index initialization with vectors happens when data is inserted + # via store_memory which calls collection.insert with pre-computed vectors + + self.logger.info(f"✓ Created collection: {collection_name}") + return True + + except Exception as e: + self.logger.error(f"Error creating collection: {e}") + return False + + def store_memory( + self, + collection_name: str, + content: str, + vector: np.ndarray, + metadata: dict[str, Any] | None = None, + ) -> str | None: + """ + 存储记忆到 VDB collection + + Args: + collection_name: Collection 名称 + content: 文本内容 + vector: 向量表示 + metadata: 元数据 + + Returns: + Memory ID 或 None + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + self.logger.error(f"Collection not found: {collection_name}") + return None + + # 确保是 VDB 类型的 collection + if not isinstance(collection, VDBMemoryCollection): + self.logger.error(f"Collection {collection_name} is not a VDB collection") + return None + + # VDBMemoryCollection.insert 使用 (index_name, raw_data, vector, metadata) + memory_id = collection.insert( + index_name="global_index", + raw_data=content, + vector=vector, + metadata=metadata, + ) + + self.logger.debug(f"Stored memory: {memory_id}") + return memory_id + + except Exception as e: + self.logger.error(f"Error storing memory: {e}") + return None + + def retrieve_memories( + self, collection_name: str, query_vector: np.ndarray, topk: int = 5 + ) -> list[dict[str, Any]]: + """ + 检索相似的记忆 + + Args: + collection_name: Collection 名称 + query_vector: 查询向量 + topk: 返回结果数量 + + Returns: + 相似记忆列表 + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + self.logger.error(f"Collection not found: {collection_name}") + return [] + + results = collection.retrieve( # type: ignore[call-arg] + query_vector=query_vector, + index_name="global_index", + topk=topk, + with_metadata=True, + ) + + return results # type: ignore[return-value] + + except Exception as e: + self.logger.error(f"Error retrieving memories: {e}") + return [] + + def forget_with_dp( + self, + collection_name: str, + memory_ids: list[str], + perturbation_strategy: str = "adaptive", + ) -> dict[str, Any]: + """ + 使用差分隐私遗忘指定的记忆 + + Args: + collection_name: Collection 名称 + memory_ids: 要遗忘的记忆 IDs + perturbation_strategy: 扰动策略 + + Returns: + 遗忘操作结果 + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + self.logger.error(f"Collection not found: {collection_name}") + return { + "success": False, + "error": f"Collection not found: {collection_name}", + } + + # 从 VDB index 获取要遗忘的向量 + index = collection.index_info.get("global_index", {}).get("index") # type: ignore[attr-defined] + if index is None: + self.logger.error(f"Index not found in collection: {collection_name}") + return {"success": False, "error": "Index not found"} + + vectors_to_forget = [] + valid_ids = [] + + for mem_id in memory_ids: + # 从 vector_store 获取向量 + if hasattr(index, "vector_store") and mem_id in index.vector_store: + vector = index.vector_store[mem_id] + vectors_to_forget.append(vector) + valid_ids.append(mem_id) + else: + self.logger.warning(f"Vector not found for memory ID: {mem_id}") + + if not vectors_to_forget: + self.logger.warning("No vectors found to forget") + return {"success": False, "error": "No vectors found"} + + # 获取所有向量用于补偿 + all_vectors = [] + all_ids = [] + if hasattr(index, "vector_store"): + for vid, vector in index.vector_store.items(): + if vid not in self.unlearning_engine.privacy_accountant.get_remaining_budget(): + all_vectors.append(vector) + all_ids.append(vid) + + vectors_array = np.array(vectors_to_forget) + if all_vectors: + all_vectors_array = np.array(all_vectors) + else: + all_vectors_array = vectors_array + + self.logger.info(f"Starting DP unlearning for {len(valid_ids)} memories...") + + # 执行 DP 遗忘 + result = self.unlearning_engine.unlearn_vectors( + vectors_to_forget=vectors_array, + vector_ids_to_forget=valid_ids, + all_vectors=all_vectors_array, + all_vector_ids=all_ids, + perturbation_strategy=perturbation_strategy, + ) + + if not result.success: + error_msg = result.metadata.get("error", "Unknown error") + self.logger.error(f"Unlearning failed: {error_msg}") + return {"success": False, "error": error_msg} + + # 获取扰动后的向量 + perturbed_vectors = result.metadata.get("perturbed_vectors", []) + + # 更新 VDB 中的向量 + updated_count = 0 + for mem_id, perturbed_vec in zip(valid_ids, perturbed_vectors): + try: + # 更新向量 + if hasattr(index, "update"): + index.update(mem_id, perturbed_vec) + else: + # 备选：删除后重新插入 + index.delete(mem_id) + index.insert(perturbed_vec, mem_id) + updated_count += 1 + except Exception as e: + self.logger.error(f"Failed to update vector for {mem_id}: {e}") + + # 持久化更改 + self.manager.store_collection(collection_name) + + self.logger.info(f"✓ Successfully forgotten {updated_count} memories") + + # 返回结果 + status = self.unlearning_engine.get_privacy_status() + remaining = status["remaining_budget"] + + return { + "success": True, + "num_forgotten": updated_count, + "privacy_cost": { + "epsilon": result.privacy_cost[0], + "delta": result.privacy_cost[1], + }, + "remaining_budget": { + "epsilon": remaining["epsilon_remaining"], + "delta": remaining["delta_remaining"], + }, + "budget_utilization": status["accountant_summary"]["budget_utilization"], + } + + except Exception as e: + self.logger.error(f"Error in forget_with_dp: {e}") + return {"success": False, "error": str(e)} + + def get_privacy_status(self) -> dict[str, Any]: + """获取当前隐私预算状态""" + return self.unlearning_engine.get_privacy_status() + + +def example_basic_dp_memory(): + """示例1：基础 DP Memory Service""" + print("\n" + "=" * 70) + print("Example 1: Basic DP Memory Service") + print("=" * 70) + + # 创建服务 + service = DPMemoryService(epsilon=1.0, delta=1e-5) + + # 创建 collection + service.create_collection("documents") + + # 存储一些记忆 + print("\n📝 Storing memories...") + memory_ids = [] + for i in range(5): + content = f"This is document {i} with sensitive information" + vector = np.random.randn(128).astype(np.float32) + vector = vector / (np.linalg.norm(vector) + 1e-10) + metadata = {"doc_index": i, "category": "sensitive" if i % 2 == 0 else "normal"} + + mem_id = service.store_memory( + collection_name="documents", + content=content, + vector=vector, + metadata=metadata, + ) + if mem_id: + memory_ids.append(mem_id) + print(f" ✓ Stored memory {i}: {mem_id[:8]}...") + + # 检索 + print("\n🔍 Retrieving memories...") + query_vector = np.random.randn(128).astype(np.float32) + query_vector = query_vector / (np.linalg.norm(query_vector) + 1e-10) + results = service.retrieve_memories("documents", query_vector, topk=3) + print(f" Found {len(results)} results") + + # 遗忘其中一些 + print("\n🔒 Forgetting sensitive documents...") + if memory_ids: + forget_ids = memory_ids[::2] # 每隔一个遗忘 + result = service.forget_with_dp( + collection_name="documents", + memory_ids=forget_ids, + perturbation_strategy="selective", + ) + + print(f" Success: {result['success']}") + if result["success"]: + print(f" Forgotten: {result['num_forgotten']} documents") + print(f" Privacy cost: ε={result['privacy_cost']['epsilon']:.4f}") + print(f" Remaining budget: ε={result['remaining_budget']['epsilon']:.4f}") + + print() + + +def example_privacy_budget_management(): + """示例2：隐私预算管理""" + print("\n" + "=" * 70) + print("Example 2: Privacy Budget Management") + print("=" * 70) + + service = DPMemoryService(epsilon=0.5, delta=1e-5) + service.create_collection("sensitive_data") + + # 创建测试数据 + memory_ids = [] + for i in range(10): + content = f"Document {i}" + vector = np.random.randn(128).astype(np.float32) + vector = vector / (np.linalg.norm(vector) + 1e-10) + mem_id = service.store_memory("sensitive_data", content, vector) + if mem_id: + memory_ids.append(mem_id) + + print("\n📊 Privacy Budget Tracking:") + + # 多次遗忘操作 + forget_count = 0 + for batch_idx in range(3): + # 每批遗忘 2 个 + batch_ids = memory_ids[batch_idx * 2 : (batch_idx + 1) * 2] + if not batch_ids: + break + + result = service.forget_with_dp( + collection_name="sensitive_data", + memory_ids=batch_ids, + perturbation_strategy="uniform", + ) + + forget_count += 1 + + if result["success"]: + print(f" Batch {forget_count}: Success") + print(f" Forgotten: {result['num_forgotten']}") + print(f" Remaining ε: {result['remaining_budget']['epsilon']:.4f}") + else: + print(f" Batch {forget_count}: Failed - {result['error']}") + break + + print() + + +def example_multi_collection(): + """示例3：多 Collection 管理""" + print("\n" + "=" * 70) + print("Example 3: Multi-Collection Management") + print("=" * 70) + + service = DPMemoryService(epsilon=1.0) + + # 创建多个 collection + collections = ["public", "internal", "confidential"] + for col_name in collections: + service.create_collection(col_name) + print(f" ✓ Created collection: {col_name}") + + # 向不同 collection 存储数据 + print("\n📝 Storing data to different collections...") + for col_name in collections: + for i in range(3): + content = f"{col_name} document {i}" + vector = np.random.randn(128).astype(np.float32) + vector = vector / (np.linalg.norm(vector) + 1e-10) + mem_id = service.store_memory(col_name, content, vector) + if mem_id: + print(f" ✓ {col_name}: {mem_id[:8]}...") + + # 从 confidential collection 遗忘一些数据 + print("\n🔒 Forgetting from confidential collection...") + query_vector = np.random.randn(128).astype(np.float32) + query_vector = query_vector / (np.linalg.norm(query_vector) + 1e-10) + results = service.retrieve_memories("confidential", query_vector, topk=2) + if results: + # 获取第一个结果的 ID（这是一个简化版，实际需要追踪 ID） + print(f" Found {len(results)} documents in confidential collection") + + print() + + +def main(): + """运行所有示例""" + print("\n" + "=" * 70) + print("SAGE Unlearning Library - MemoryService Integration") + print("=" * 70) + print("\n这些示例展示了如何将 unlearning 集成到 MemoryService。") + print("适合：RAG 系统的隐私遗忘、VDB 集成、数据生命周期管理\n") + + # 禁用调试日志 + CustomLogger.disable_global_console_debug() + + # 运行示例 + example_basic_dp_memory() + example_privacy_budget_management() + example_multi_collection() + + print("=" * 70) + print("✅ All examples completed successfully!") + print("=" * 70) + print("\n💡 Next steps:") + print(" 1. Integrate with real embedding models") + print(" 2. Implement custom forgetting policies") + print(" 3. See basic_unlearning_demo.py for full RAG example\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/rag/usage_4_complete_rag.py b/tutorials/L3-libs/rag/usage_4_complete_rag.py new file mode 100644 index 0000000..2347833 --- /dev/null +++ b/tutorials/L3-libs/rag/usage_4_complete_rag.py @@ -0,0 +1,582 @@ +""" +Usage 4: Complete RAG System with DP Unlearning +=============================================== + +完整的 RAG 系统中的隐私遗忘场景。 + +适用场景： +- 完整的 RAG Pipeline +- 用户请求删除数据 +- 组织数据删除义务（GDPR 等） +- 恶意数据清理 + +特性： +- 检索 → 筛选 → 遗忘 → 更新 的完整流程 +- 多个查询的批量处理 +- 隐私预算跟踪 +- 操作日志和审计 +""" + +import os +from datetime import datetime +from typing import Any + +import numpy as np + +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.service.base_service import BaseService +from sage.libs.privacy.unlearning import UnlearningEngine +from sage.middleware.components.sage_mem.neuromem.memory_collection.vdb_collection import ( + VDBMemoryCollection, +) +from sage.middleware.components.sage_mem.neuromem.memory_manager import MemoryManager + + +class RAGUnlearningSystem(BaseService): + """RAG 系统中的隐私遗忘管理""" + + def __init__(self, data_dir: str | None = None, epsilon: float = 1.0): + super().__init__() + + if data_dir is None: + data_dir = os.path.join(os.getcwd(), "data", "rag_unlearning") + os.makedirs(data_dir, exist_ok=True) + + self.data_dir = data_dir + self.manager = MemoryManager(data_dir) + self.unlearning_engine = UnlearningEngine( + epsilon=epsilon, + delta=1e-5, + total_budget_epsilon=100.0, + enable_compensation=True, + ) + + # 审计日志 + self.audit_log = [] + + self.logger.info("RAGUnlearningSystem initialized") + + def initialize_rag_corpus(self, collection_name: str, documents: list[dict[str, Any]]) -> bool: + """ + 初始化 RAG corpus + + Args: + collection_name: Collection 名称 + documents: 文档列表，每个包含 'id', 'content', 'vector', 'metadata' + + Returns: + 是否成功 + """ + try: + # 创建 collection + collection = self.manager.create_collection( + { + "name": collection_name, + "backend_type": "VDB", + "description": f"RAG corpus: {collection_name}", + } + ) + + if collection is None: + return False + + # 创建索引 + index_config = { + "name": "content_index", + "embedding_model": "mockembedder", + "dim": 128, + "backend_type": "FAISS", + "description": "Content search index", + } + collection.create_index(index_config) # type: ignore[attr-defined] + + # 确保是 VDB collection + if not isinstance(collection, VDBMemoryCollection): + self.logger.error("Collection is not a VDB collection") + return False + + # 插入文档 - VDBMemoryCollection.insert(content, index_names, vector, metadata) + for doc in documents: + collection.insert( + content=doc["content"], + index_names="content_index", + vector=doc["vector"], + metadata=doc.get("metadata", {}), + ) + + # Index is initialized through individual inserts, no need for init_index + self.manager.store_collection(collection_name) + + self.logger.info(f"✓ Initialized RAG corpus with {len(documents)} documents") + self._audit_log("INIT_CORPUS", collection_name, len(documents)) + + return True + + except Exception as e: + self.logger.error(f"Error initializing corpus: {e}") + return False + + def retrieve_relevant_documents( + self, collection_name: str, query_vector: np.ndarray, topk: int = 5 + ) -> list[dict[str, Any]]: + """检索相关文档""" + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + return [] + + # 确保是 VDB collection + if not isinstance(collection, VDBMemoryCollection): + self.logger.error("Collection is not a VDB collection") + return [] + + # VDBMemoryCollection.retrieve(query_vector, index_name, topk, ...) + results = collection.retrieve( + query_vector=query_vector, + index_name="content_index", + topk=topk, + with_metadata=True, + ) + + # retrieve 可能返回 None + if results is None: + return [] + + return results # type: ignore[return-value] + + except Exception as e: + self.logger.error(f"Error retrieving documents: {e}") + return [] + + def forget_documents( + self, + collection_name: str, + document_ids: list[str], + reason: str = "user_request", + user_id: str | None = None, + ) -> dict[str, Any]: + """ + 遗忘指定的文档 + + Args: + collection_name: Collection 名称 + document_ids: 要遗忘的文档 IDs + reason: 遗忘原因 + user_id: 发起遗忘的用户 ID + + Returns: + 操作结果 + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + return {"success": False, "error": "Collection not found"} + + index = collection.index_info.get("content_index", {}).get("index") # type: ignore[attr-defined] + if index is None: + return {"success": False, "error": "Index not found"} + + # 收集要遗忘的向量 + vectors_to_forget = [] + valid_ids = [] + + for doc_id in document_ids: + if hasattr(index, "vector_store") and doc_id in index.vector_store: + vector = index.vector_store[doc_id] + vectors_to_forget.append(vector) + valid_ids.append(doc_id) + + if not vectors_to_forget: + return {"success": False, "error": "No documents found to forget"} + + # 获取所有向量用于补偿 + all_vectors = [] + all_ids = [] + if hasattr(index, "vector_store"): + for vid, vector in index.vector_store.items(): + if vid not in valid_ids: # 排除要遗忘的向量 + all_vectors.append(vector) + all_ids.append(vid) + + self.logger.info(f"Starting DP unlearning for {len(valid_ids)} documents...") + + # 执行 DP 遗忘 + vectors_array = np.array(vectors_to_forget) + all_vectors_array = np.array(all_vectors) if all_vectors else vectors_array + + result = self.unlearning_engine.unlearn_vectors( + vectors_to_forget=vectors_array, + vector_ids_to_forget=valid_ids, + all_vectors=all_vectors_array, + all_vector_ids=all_ids, + perturbation_strategy="adaptive", + ) + + if not result.success: + error = result.metadata.get("error", "Unknown error") + self.logger.error(f"Unlearning failed: {error}") + return {"success": False, "error": error} + + # 更新 VDB 中的向量 + perturbed_vectors = result.metadata.get("perturbed_vectors", []) + updated_count = 0 + + for doc_id, perturbed_vec in zip(valid_ids, perturbed_vectors): + try: + if hasattr(index, "update"): + index.update(doc_id, perturbed_vec) + else: + index.delete(doc_id) + index.insert(perturbed_vec, doc_id) + updated_count += 1 + except Exception as e: + self.logger.error(f"Failed to update vector for {doc_id}: {e}") + + # 持久化 + self.manager.store_collection(collection_name) + + # 记录审计日志 + self._audit_log( + "FORGET_DOCUMENTS", + collection_name, + len(valid_ids), + extra={ + "reason": reason, + "user_id": user_id, + "privacy_cost": result.privacy_cost, + }, + ) + + status = self.unlearning_engine.get_privacy_status() + remaining = status["remaining_budget"] + + self.logger.info(f"✓ Successfully forgotten {updated_count} documents") + + return { + "success": True, + "num_forgotten": updated_count, + "privacy_cost": { + "epsilon": result.privacy_cost[0], + "delta": result.privacy_cost[1], + }, + "remaining_budget": { + "epsilon": remaining["epsilon_remaining"], + "delta": remaining["delta_remaining"], + }, + } + + except Exception as e: + self.logger.error(f"Error in forget_documents: {e}") + return {"success": False, "error": str(e)} + + def handle_user_deletion_request( + self, collection_name: str, user_id: str, user_keywords: list[str] | None = None + ) -> dict[str, Any]: + """ + 处理用户数据删除请求（如 GDPR 删除权） + + Args: + collection_name: Collection 名称 + user_id: 用户 ID + user_keywords: 用户特定关键词（可选） + + Returns: + 处理结果 + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + return {"success": False, "error": "Collection not found"} + + # 查找属于该用户的所有文档 + all_ids = collection.get_all_ids() + user_docs = [] + + for doc_id in all_ids: + metadata = collection.metadata_storage.get(doc_id) + if metadata and metadata.get("user_id") == user_id: + user_docs.append(doc_id) + + if not user_docs: + self.logger.info(f"No documents found for user {user_id}") + return { + "success": True, + "num_forgotten": 0, + "message": "No documents to delete", + } + + self.logger.info(f"Found {len(user_docs)} documents for user {user_id}") + + # 遗忘用户所有文档 + result = self.forget_documents( + collection_name=collection_name, + document_ids=user_docs, + reason="user_deletion_request", + user_id=user_id, + ) + + if result["success"]: + self.logger.info(f"✓ Deleted all data for user {user_id}") + + return result + + except Exception as e: + self.logger.error(f"Error handling deletion request: {e}") + return {"success": False, "error": str(e)} + + def handle_malicious_content_removal( + self, collection_name: str, detection_keywords: list[str] + ) -> dict[str, Any]: + """ + 处理恶意内容移除 + + Args: + collection_name: Collection 名称 + detection_keywords: 恶意内容关键词 + + Returns: + 处理结果 + """ + try: + collection = self.manager.get_collection(collection_name) + if collection is None: + return {"success": False, "error": "Collection not found"} + + # 查找包含恶意内容的文档 + all_ids = collection.get_all_ids() + malicious_docs = [] + + for doc_id in all_ids: + content = collection.text_storage.get(doc_id) + if content: + for keyword in detection_keywords: + if keyword.lower() in content.lower(): + malicious_docs.append(doc_id) + break + + if not malicious_docs: + self.logger.info("No malicious content detected") + return { + "success": True, + "num_forgotten": 0, + "message": "No malicious content found", + } + + self.logger.warning(f"Detected {len(malicious_docs)} documents with malicious content") + + # 遗忘恶意内容 + result = self.forget_documents( + collection_name=collection_name, + document_ids=malicious_docs, + reason="malicious_content", + user_id="system", + ) + + return result + + except Exception as e: + self.logger.error(f"Error handling malicious content: {e}") + return {"success": False, "error": str(e)} + + def get_audit_log(self) -> list[dict[str, Any]]: + """获取审计日志""" + return self.audit_log + + def _audit_log(self, operation: str, collection: str, count: int, extra: dict | None = None): + """记录审计事件""" + log_entry = { + "timestamp": datetime.now().isoformat(), + "operation": operation, + "collection": collection, + "count": count, + "extra": extra or {}, + } + self.audit_log.append(log_entry) + + +def example_basic_rag(): + """示例1：基础 RAG 系统""" + print("\n" + "=" * 70) + print("Example 1: Basic RAG System with Unlearning") + print("=" * 70) + + system = RAGUnlearningSystem(epsilon=1.0) + + # 创建示例文档 + documents = [ + { + "id": "doc_001", + "content": "Machine learning is a subset of artificial intelligence", + "metadata": {"user_id": "user_1", "category": "public"}, + }, + { + "id": "doc_002", + "content": "Deep learning uses neural networks with multiple layers", + "metadata": {"user_id": "user_1", "category": "public"}, + }, + { + "id": "doc_003", + "content": "Natural language processing is used for text analysis", + "metadata": {"user_id": "user_2", "category": "public"}, + }, + { + "id": "doc_004", + "content": "Computer vision helps machines understand images", + "metadata": {"user_id": "user_2", "category": "sensitive"}, + }, + { + "id": "doc_005", + "content": "Reinforcement learning enables agents to learn from interaction", + "metadata": {"user_id": "user_3", "category": "public"}, + }, + ] + + # 为每个文档添加随机向量 + for doc in documents: + doc["vector"] = np.random.randn(128).astype(np.float32) + doc["vector"] = doc["vector"] / (np.linalg.norm(doc["vector"]) + 1e-10) + + # 初始化 corpus + system.initialize_rag_corpus("knowledge_base", documents) + + # 检索 + print("\n🔍 Retrieving documents...") + query_vector = np.random.randn(128).astype(np.float32) + query_vector = query_vector / (np.linalg.norm(query_vector) + 1e-10) + results = system.retrieve_relevant_documents("knowledge_base", query_vector, topk=3) + print(f" Found {len(results)} relevant documents") + + # 用户请求删除 + print("\n🗑️ Processing user deletion request...") + result = system.handle_user_deletion_request("knowledge_base", "user_1") + print(f" Success: {result['success']}") + if result["success"]: + print(f" Deleted: {result['num_forgotten']} documents") + if "privacy_cost" in result: + print(f" Privacy cost: ε={result['privacy_cost']['epsilon']:.4f}") + + print() + + +def example_malicious_content(): + """示例2：恶意内容检测和移除""" + print("\n" + "=" * 70) + print("Example 2: Malicious Content Detection and Removal") + print("=" * 70) + + system = RAGUnlearningSystem(epsilon=1.0) + + # 创建包含一些恶意内容的文档 + documents = [ + { + "id": "doc_001", + "content": "This is normal technical content about machine learning", + "metadata": {"user_id": "user_1", "category": "normal"}, + }, + { + "id": "doc_002", + "content": "Spam content: click here for free money!!!", + "metadata": {"user_id": "user_2", "category": "spam"}, + }, + { + "id": "doc_003", + "content": "More legitimate deep learning information", + "metadata": {"user_id": "user_1", "category": "normal"}, + }, + { + "id": "doc_004", + "content": "Malware distribution: download now!!!", + "metadata": {"user_id": "user_3", "category": "malicious"}, + }, + ] + + for doc in documents: + doc["vector"] = np.random.randn(128).astype(np.float32) + doc["vector"] = doc["vector"] / (np.linalg.norm(doc["vector"]) + 1e-10) + + system.initialize_rag_corpus("content_db", documents) + + # 检测并移除恶意内容 + print("\n🚨 Detecting malicious content...") + result = system.handle_malicious_content_removal( + "content_db", detection_keywords=["spam", "malware", "!!!"] + ) + + print(f" Success: {result['success']}") + if result["success"]: + print(f" Removed: {result['num_forgotten']} malicious documents") + + print() + + +def example_audit_log(): + """示例3：审计日志""" + print("\n" + "=" * 70) + print("Example 3: Audit Log and Compliance") + print("=" * 70) + + system = RAGUnlearningSystem(epsilon=1.0) + + # 创建文档 + documents = [] + for i in range(10): + documents.append( + { + "id": f"doc_{i:03d}", + "content": f"Document {i} content", + "metadata": {"user_id": f"user_{i % 3}", "category": "normal"}, + "vector": np.random.randn(128).astype(np.float32) + / np.linalg.norm(np.random.randn(128).astype(np.float32)), + } + ) + + system.initialize_rag_corpus("audit_test", documents) + + # 执行多个操作 + print("\n📝 Performing operations...") + + # 用户 0 删除请求 + system.handle_user_deletion_request("audit_test", "user_0") + print(" ✓ User deletion request processed") + + # 恶意内容检测（无恶意内容） + system.handle_malicious_content_removal("audit_test", ["malware"]) + print(" ✓ Malicious content check completed") + + # 显示审计日志 + print("\n📋 Audit Log:") + for entry in system.get_audit_log(): + print( + f" {entry['timestamp']}: {entry['operation']} on {entry['collection']} ({entry['count']} items)" + ) + + print() + + +def main(): + """运行所有示例""" + print("\n" + "=" * 70) + print("SAGE Unlearning - Complete RAG System Examples") + print("=" * 70) + print("\n这些示例展示了在完整 RAG 系统中使用隐私遗忘。") + print("包括：用户删除请求、恶意内容移除、合规审计\n") + + CustomLogger.disable_global_console_debug() + + # 运行示例 + example_basic_rag() + example_malicious_content() + example_audit_log() + + print("=" * 70) + print("✅ All examples completed successfully!") + print("=" * 70) + print("\n💡 Key Takeaways:") + print(" 1. Unlearning 库提供灵活的隐私保护机制") + print(" 2. 支持多种遗忘场景（用户请求、恶意内容等）") + print(" 3. 完整的审计日志用于合规") + print(" 4. 隐私预算管理确保整体隐私保证\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/unlearning/README.md b/tutorials/L3-libs/unlearning/README.md new file mode 100644 index 0000000..3c5d775 --- /dev/null +++ b/tutorials/L3-libs/unlearning/README.md @@ -0,0 +1,71 @@ +# Machine Unlearning Examples + +This directory contains examples and tutorials for using SAGE's machine unlearning capabilities. + +## Overview + +Machine unlearning enables models to "forget" specific training data, which is crucial for: + +- **Privacy compliance**: GDPR "Right to be Forgotten" +- **Data removal**: Removing incorrect or outdated data +- **Bias mitigation**: Removing biased training samples +- **Security**: Removing poisoned data from models + +## Files + +- `machine_unlearning_examples.py` - Comprehensive examples demonstrating: + - Basic unlearning workflow + - Differential privacy mechanisms + - Evaluation metrics + - Algorithm comparisons + - Real-world GDPR compliance scenario + +## Running Examples + +```bash +# From SAGE root directory +cd examples/tutorials/unlearning +python machine_unlearning_examples.py +``` + +## Key Concepts + +### Differential Privacy Mechanisms + +- **Gaussian Mechanism**: Provides (ε,δ)-differential privacy by adding Gaussian noise +- **Laplace Mechanism**: Provides ε-differential privacy by adding Laplace noise + +### Core Components + +1. **Privacy Mechanisms** (`sage.libs.unlearning.algorithms`) + + - `GaussianMechanism`: (ε,δ)-DP noise addition + - `LaplaceMechanism`: ε-DP noise addition + +1. **Unlearning Engine** (`sage.libs.unlearning.dp_unlearning`) + + - `UnlearningEngine`: Orchestrates the unlearning process + - `VectorPerturbation`: Applies noise to model parameters + - `NeighborCompensation`: Compensates for neighboring records + - `PrivacyAccountant`: Tracks privacy budget across operations + +1. **Evaluation Tools** (`sage.libs.unlearning.evaluation`) + + - `UnlearningMetrics`: Evaluates forgetting quality and model utility + +## Further Reading + +- Implementation details: `packages/sage-libs/src/sage/libs/unlearning/` +- Research papers: See docstrings in mechanism implementations +- SAGE documentation: `docs/` directory + +## Student Research Tasks + +The unlearning module includes TODO items for advanced research: + +- Analytic Gaussian mechanism (tighter bounds) +- Concentrated differential privacy +- Privacy amplification by subsampling +- Advanced composition techniques + +See the source code in `packages/sage-libs/src/sage/libs/unlearning/algorithms/` for details. diff --git a/tutorials/L3-libs/unlearning/__init__.py b/tutorials/L3-libs/unlearning/__init__.py new file mode 100644 index 0000000..e191fc2 --- /dev/null +++ b/tutorials/L3-libs/unlearning/__init__.py @@ -0,0 +1,6 @@ +""" +SAGE Machine Unlearning Examples +================================= + +Examples and tutorials for SAGE's differential privacy-based machine unlearning. +""" diff --git a/tutorials/L3-libs/unlearning/basic_unlearning_demo.py b/tutorials/L3-libs/unlearning/basic_unlearning_demo.py new file mode 100644 index 0000000..120fa12 --- /dev/null +++ b/tutorials/L3-libs/unlearning/basic_unlearning_demo.py @@ -0,0 +1,143 @@ +""" +Basic Unlearning Demo +====================== + +Demonstrates the basic usage of the SAGE Unlearning Library. + +This example shows: +1. How to create vectors (simulating embeddings) +2. How to use the UnlearningEngine +3. How to evaluate unlearning quality + +**For Students**: This is your starting point! +Modify and extend this example to test your algorithms. + +@test:allow-demo +""" + +import numpy as np + +from sage.libs.privacy.unlearning import UnlearningEngine + + +def generate_synthetic_vectors(n_vectors: int = 100, dim: int = 128) -> tuple: + """ + Generate synthetic embedding vectors for testing. + + In real usage, these would come from a RAG system's vector database. + """ + # Generate random vectors + vectors = np.random.randn(n_vectors, dim) + # L2 normalize (common for embeddings) + vectors = vectors / (np.linalg.norm(vectors, axis=1, keepdims=True) + 1e-10) + + # Generate IDs + ids = [f"doc_{i}" for i in range(n_vectors)] + + return vectors, ids + + +def main(): + print("=" * 70) + print("SAGE Unlearning Library - Basic Demo") + print("=" * 70) + print() + + # Step 1: Generate synthetic data + print("Step 1: Generating synthetic vectors...") + all_vectors, all_ids = generate_synthetic_vectors(n_vectors=100, dim=128) + print(f" Generated {len(all_vectors)} vectors of dimension {all_vectors.shape[1]}") + print() + + # Step 2: Select vectors to forget + print("Step 2: Selecting vectors to forget...") + n_forget = 5 + forget_indices = np.random.choice(len(all_vectors), size=n_forget, replace=False) + vectors_to_forget = all_vectors[forget_indices] + ids_to_forget = [all_ids[i] for i in forget_indices] + print(f" Selected {n_forget} vectors to forget: {ids_to_forget}") + print() + + # Step 3: Initialize Unlearning Engine + print("Step 3: Initializing Unlearning Engine...") + engine = UnlearningEngine( + epsilon=1.0, # Privacy parameter + delta=1e-5, # Failure probability + total_budget_epsilon=10.0, # Total privacy budget + enable_compensation=True, # Enable neighbor compensation + ) + print(f" Engine initialized: {engine.mechanism}") + print(f" Privacy budget: ε={engine.privacy_accountant.total_epsilon_budget}") + print() + + # Step 4: Perform unlearning + print("Step 4: Performing unlearning...") + print(" Strategy: uniform perturbation") + + result = engine.unlearn_vectors( + vectors_to_forget=vectors_to_forget, + vector_ids_to_forget=ids_to_forget, + all_vectors=all_vectors, + all_vector_ids=all_ids, + perturbation_strategy="uniform", + ) + + print(f"\n Result: {result}") + print(f" Privacy cost: ε={result.privacy_cost[0]:.4f}, δ={result.privacy_cost[1]:.6f}") + print(f" Vectors unlearned: {result.num_vectors_unlearned}") + print(f" Neighbors compensated: {result.num_neighbors_compensated}") + print() + + # Step 5: Check privacy budget + print("Step 5: Checking remaining privacy budget...") + status = engine.get_privacy_status() + remaining = status["remaining_budget"] + print( + f" Remaining: ε={remaining['epsilon_remaining']:.4f}, δ={remaining['delta_remaining']:.6f}" + ) + print(f" Budget utilization: {status['accountant_summary']['budget_utilization']:.1%}") + print() + + # Step 6: Try different strategies + print("Step 6: Comparing perturbation strategies...") + strategies = ["uniform", "selective", "adaptive"] + + for strategy in strategies: + # Reset engine for fair comparison + test_engine = UnlearningEngine(epsilon=1.0, enable_compensation=False) + + test_result = test_engine.unlearn_vectors( + vectors_to_forget=vectors_to_forget[:2], # Use fewer vectors for comparison + vector_ids_to_forget=ids_to_forget[:2], + perturbation_strategy=strategy, + ) + + perturbed = test_result.metadata["perturbed_vectors"] + original = vectors_to_forget[:2] + + # Measure impact + l2_dist = np.mean([np.linalg.norm(o - p) for o, p in zip(original, perturbed)]) + cos_sim = np.mean( + [ + np.dot(o, p) / (np.linalg.norm(o) * np.linalg.norm(p)) + for o, p in zip(original, perturbed) + ] + ) + + print(f" {strategy:12s}: L2={l2_dist:.4f}, CosSim={cos_sim:.4f}") + + print() + print("=" * 70) + print("Demo completed successfully!") + print("=" * 70) + print() + print("Next steps for students:") + print(" 1. Implement new privacy mechanisms in algorithms/") + print(" 2. Design better perturbation strategies in dp_unlearning/vector_perturbation.py") + print(" 3. Enhance neighbor compensation in dp_unlearning/neighbor_compensation.py") + print(" 4. Add comprehensive evaluation metrics in evaluation/metrics.py") + print() + + +if __name__ == "__main__": + main() diff --git a/tutorials/L3-libs/unlearning/machine_unlearning_examples.py b/tutorials/L3-libs/unlearning/machine_unlearning_examples.py new file mode 100644 index 0000000..5af4c6c --- /dev/null +++ b/tutorials/L3-libs/unlearning/machine_unlearning_examples.py @@ -0,0 +1,366 @@ +""" +SAGE Unlearning - Usage Examples + +This file demonstrates how to use the SAGE machine unlearning algorithms. + +Layer: L3 (Core - Algorithm Library) +""" + + +def example_basic_unlearning(): + """ + Example 1: Basic unlearning workflow + + Demonstrates the basic workflow of using machine unlearning + algorithms to remove specific data from trained models. + """ + print("=" * 60) + print("Example 1: Basic Unlearning Workflow") + print("=" * 60) + + try: + from sage.libs.privacy.unlearning.algorithms import GaussianMechanism # noqa: F401 + + print("\n✓ Machine Unlearning Overview:") + print(" 1. Train initial model on full dataset") + print(" 2. Identify data to forget") + print(" 3. Apply unlearning algorithm") + print(" 4. Verify data removal") + + print("\nExample workflow:") + print( + """ + from sage.libs.privacy.unlearning.algorithms import GaussianMechanism + from sage.libs.privacy.unlearning.dp_unlearning import UnlearningEngine + + # Initialize DP mechanism + mechanism = GaussianMechanism( + epsilon=1.0, + delta=1e-5, + sensitivity=0.1 + ) + + # Initialize unlearning engine + engine = UnlearningEngine(mechanism=mechanism) + + # Specify data to forget + forget_indices = [10, 25, 42, 100] + + # Apply unlearning + updated_params = engine.unlearn( + model_parameters=current_params, + forget_indices=forget_indices, + retain_data=remaining_data + ) + + # Verify unlearning + print(f"Model updated: {updated_params is not None}") + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + + +def example_differential_privacy(): + """ + Example 2: Differential privacy mechanisms + + Demonstrates how to use differential privacy techniques + in conjunction with machine unlearning. + """ + print("\n" + "=" * 60) + print("Example 2: Differential Privacy Unlearning") + print("=" * 60) + + try: + from sage.libs.privacy.unlearning.dp_unlearning import ( # noqa: F401 + NeighborCompensation, + VectorPerturbation, + ) + + print("\n✓ DP-based unlearning components:") + print(" - Vector Perturbation: Add calibrated noise") + print(" - Neighbor Compensation: Compensate neighboring records") + print(" - Privacy Accountant: Track privacy budget") + + print("\nExample: Vector perturbation") + print( + """ + from sage.libs.privacy.unlearning.dp_unlearning import VectorPerturbation + from sage.libs.privacy.unlearning.algorithms import GaussianMechanism + + # Create DP mechanism + mechanism = GaussianMechanism( + epsilon=1.0, + delta=1e-5, + sensitivity=0.1 + ) + + # Create vector perturbation component + perturbation = VectorPerturbation(mechanism=mechanism) + + # Apply DP unlearning + perturbed_params = perturbation.perturb( + model_parameters=params, + forget_gradient=forget_grad + ) + """ + ) + + print("\nExample: Privacy accounting") + print( + """ + from sage.libs.privacy.unlearning.dp_unlearning import PrivacyAccountant + + # Track privacy budget across multiple unlearning operations + accountant = PrivacyAccountant(total_epsilon=3.0) + + # First unlearning operation + accountant.spend(epsilon=1.0, delta=1e-5) + + # Check remaining budget + remaining = accountant.get_remaining_budget() + print(f"Remaining privacy budget: {remaining}") + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + + +def example_evaluation_metrics(): + """ + Example 3: Evaluating unlearning effectiveness + + Demonstrates how to evaluate the effectiveness and quality + of machine unlearning algorithms. + """ + print("\n" + "=" * 60) + print("Example 3: Unlearning Evaluation Metrics") + print("=" * 60) + + try: + from sage.libs.privacy.unlearning.evaluation import UnlearningMetrics # noqa: F401 + + print("\n✓ Evaluation metrics:") + print(" - Forgetting accuracy: How well data is forgotten") + print(" - Retention accuracy: Performance on remaining data") + print(" - Unlearning time: Computational efficiency") + print(" - Model utility: Overall model performance") + + print("\nExample evaluation:") + print( + """ + from sage.libs.privacy.unlearning.evaluation import UnlearningMetrics + + # Create metrics evaluator + metrics = UnlearningMetrics( + original_model=original_model, + unlearned_model=unlearned_model, + retrained_model=retrained_model # Gold standard + ) + + # Evaluate forgetting quality + forget_score = metrics.forgetting_quality( + forget_set=forget_data + ) + + # Evaluate retention + retain_score = metrics.retention_quality( + retain_set=retain_data + ) + + # Compare with retraining + similarity = metrics.model_similarity() + + print(f"Forgetting score: {forget_score:.4f}") + print(f"Retention score: {retain_score:.4f}") + print(f"Similarity to retrained model: {similarity:.4f}") + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + + +def example_unlearning_algorithms(): + """ + Example 4: Different unlearning algorithms + + Demonstrates various unlearning algorithms available in SAGE. + """ + print("\n" + "=" * 60) + print("Example 4: Unlearning Algorithm Comparison") + print("=" * 60) + + print("\n✓ Available algorithms:") + print(" - Gaussian Mechanism: Gaussian noise-based (ε,δ)-DP") + print(" - Laplace Mechanism: Laplace noise-based ε-DP") + print(" - Unlearning Engine: Orchestrates unlearning process") + + print("\nExample: Gaussian vs Laplace mechanisms") + print( + """ + from sage.libs.privacy.unlearning.algorithms import ( + GaussianMechanism, + LaplaceMechanism + ) + from sage.libs.privacy.unlearning.dp_unlearning import UnlearningEngine + + # Gaussian mechanism (better for larger datasets, requires δ) + gaussian = GaussianMechanism( + epsilon=1.0, + delta=1e-5, + sensitivity=0.1 + ) + + # Laplace mechanism (pure ε-DP, no δ required) + laplace = LaplaceMechanism( + epsilon=0.5, + sensitivity=0.1 + ) + + # Create engines with different mechanisms + gaussian_engine = UnlearningEngine(mechanism=gaussian) + laplace_engine = UnlearningEngine(mechanism=laplace) + + # Compare results + gaussian_params = gaussian_engine.unlearn(forget_indices) + laplace_params = laplace_engine.unlearn(forget_indices) + """ + ) + + print("\nExample: Advanced usage with neighbor compensation") + print( + """ + from sage.libs.privacy.unlearning.dp_unlearning import ( + UnlearningEngine, + NeighborCompensation + ) + from sage.libs.privacy.unlearning.algorithms import GaussianMechanism + + # Create mechanism + mechanism = GaussianMechanism(epsilon=1.0, delta=1e-5) + + # Create neighbor compensation + neighbor_comp = NeighborCompensation( + mechanism=mechanism, + num_neighbors=5 + ) + + # Apply compensated unlearning + updated_params = neighbor_comp.compensate( + model_params=params, + forget_indices=forget_indices, + training_data=data + ) + """ + ) + + +def example_real_world_scenario(): + """ + Example 5: Real-world GDPR compliance scenario + + Demonstrates a complete workflow for handling data deletion + requests in compliance with privacy regulations. + """ + print("\n" + "=" * 60) + print("Example 5: GDPR Compliance Workflow") + print("=" * 60) + + print("\n✓ GDPR 'Right to be Forgotten' workflow:") + print(" 1. Receive deletion request") + print(" 2. Identify user data in model") + print(" 3. Apply certified unlearning") + print(" 4. Verify deletion") + print(" 5. Document compliance") + + print("\nComplete example:") + print( + """ + from sage.libs.privacy.unlearning.algorithms import GaussianMechanism + from sage.libs.privacy.unlearning.evaluation import UnlearningMetrics + from sage.libs.privacy.unlearning.dp_unlearning import ( + PrivacyAccountant, + UnlearningEngine + ) + + # Step 1: Receive deletion request + user_id = "user_12345" + user_data_indices = identify_user_data(user_id) + + # Step 2: Apply certified unlearning + accountant = PrivacyAccountant(total_epsilon=1.0) + + mechanism = GaussianMechanism( + epsilon=0.5, + delta=1e-5, + sensitivity=0.1 + ) + + engine = UnlearningEngine(mechanism=mechanism) + + updated_params = engine.unlearn( + model_parameters=production_model.parameters, + forget_indices=user_data_indices + ) + + # Step 3: Verify deletion + metrics = UnlearningMetrics( + original_model=production_model, + unlearned_model=updated_model + ) + + # Check that user data cannot be recovered + forgetting_score = metrics.forgetting_quality(user_data_indices) + assert forgetting_score > 0.95, "Insufficient forgetting" + + # Check model still performs well + retain_score = metrics.retention_quality(remaining_data) + assert retain_score > 0.90, "Too much utility loss" + + # Step 4: Document compliance + compliance_report = { + "user_id": user_id, + "deletion_date": datetime.now(), + "forgetting_score": forgetting_score, + "retention_score": retain_score, + "privacy_spent": accountant.get_spent_budget(), + "certified": True + } + + log_compliance(compliance_report) + print("✓ GDPR deletion request processed successfully") + """ + ) + + +def run_all_examples(): + """Run all examples in sequence.""" + print("\n" + "=" * 60) + print("SAGE Unlearning - Complete Examples") + print("=" * 60) + + example_basic_unlearning() + example_differential_privacy() + example_evaluation_metrics() + example_unlearning_algorithms() + example_real_world_scenario() + + print("\n" + "=" * 60) + print("✓ All examples completed") + print("=" * 60) + print("\nFor more information:") + print("- See packages/sage-libs/src/sage/libs/unlearning/README.md") + print("- Check algorithms/ for mechanism implementations") + print("- Visit docs/ for research papers and references") + print("\nKey papers:") + print("- Machine Unlearning (Cao & Yang, 2015)") + print("- Certified Data Removal (Guo et al., 2020)") + print("- The Algorithmic Foundations of DP (Dwork & Roth, 2014)") + + +if __name__ == "__main__": + run_all_examples() diff --git a/tutorials/L4-middleware/README.md b/tutorials/L4-middleware/README.md new file mode 100644 index 0000000..7a785c7 --- /dev/null +++ b/tutorials/L4-middleware/README.md @@ -0,0 +1,65 @@ +# L4: Middleware - 中间件层示例 + +> 对应 SAGE 包：`sage-middleware` + +## 📖 层级说明 + +**Middleware** 层提供领域算子和中间件组件： + +- Memory Service - 内存管理服务 +- SAGE-DB - 向量数据库 +- SAGE-TSDB - 时序数据库 +- NeuroMem - 神经记忆栈 + +## 📚 目录结构 + +``` +L4-middleware/ +├── hello_service_world.py # 服务入门示例 +├── memory_service/ # 内存服务示例 +├── sage_db/ # 向量数据库示例 +└── sage_tsdb/ # 时序数据库示例 +``` + +## 🎯 学习路径 + +### 1️⃣ 服务基础 + +- `hello_service_world.py` - 理解服务模型 + +### 2️⃣ Memory Service (`memory_service/`) + +内存管理和持久化： + +- `rag_memory_service.py` - RAG 内存服务 +- `rag_memory_pipeline.py` - 内存管道 +- `rag_memory_manager.py` - 内存管理器 + +### 3️⃣ SAGE-DB (`sage_db/`) + +向量数据库操作： + +- `workflow_demo.py` - 工作流演示 + +### 4️⃣ SAGE-TSDB (`sage_tsdb/`) + +时序数据处理： + +- `basic_dag_example.py` - 基础 DAG +- `advanced_dag_example.py` - 高级 DAG +- `stream_join_dag_example.py` - 流连接 DAG + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. 如何使用 SAGE 的数据服务 +1. 向量数据库的基本操作 +1. 时序数据的处理方法 +1. 内存管理的最佳实践 + +## ⏭️ 下一步 + +学完中间件层后，继续学习： + +- **L5-apps/** - 完整应用和解决方案 diff --git a/tutorials/L4-middleware/hello_service_world.py b/tutorials/L4-middleware/hello_service_world.py new file mode 100644 index 0000000..d5fb92d --- /dev/null +++ b/tutorials/L4-middleware/hello_service_world.py @@ -0,0 +1,51 @@ +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment +from sage.kernel.api.service.base_service import BaseService + + +class HelloBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 10 + + def execute(self): + if self.counter >= self.max_count: + return None + self.counter += 1 + return f"Hello, World! #{self.counter}" + + +class PrintSink(SinkFunction): + def execute(self, data): + # 调用服务 + self.call_service("hello_service", method="hello") + print(data) + + +# 继承BaseService创建一个简单的服务 +class HelloService(BaseService): + def __init__(self): + self.message = "hello service!!!" + + def hello(self): + print(self.message) + + +def main(): + env = LocalEnvironment("hello_service") + + # 注册服务 + env.register_service("hello_service", HelloService) + + env.from_batch(HelloBatch).sink(PrintSink) + + env.submit(autostop=True) + print("Hello Service World 示例完成!") + + +if __name__ == "__main__": + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L4-middleware/rag/examples.py b/tutorials/L4-middleware/rag/examples.py new file mode 100644 index 0000000..4f4c564 --- /dev/null +++ b/tutorials/L4-middleware/rag/examples.py @@ -0,0 +1,270 @@ +""" +SAGE RAG - Usage Examples + +This file demonstrates how to use the SAGE RAG (Retrieval-Augmented Generation) toolkit. + +Layer: L3 (Core - Algorithm Library) + +⚠️ NOTE: 这些示例优先展示 API 设计。`sage.libs.rag.*` 已提供核心组件， +但部分高级算子仍位于 middleware 层。 +""" + + +def example_document_loading(): + """ + Example 1: Loading documents + + Demonstrates how to load documents from various sources using + the document loaders. + """ + print("=" * 60) + print("Example 1: Loading Documents") + print("=" * 60) + + try: + from sage.libs.rag.document_loaders import ( + DocxLoader, # noqa: F401 + LoaderFactory, # noqa: F401 + MarkdownLoader, # noqa: F401 + PDFLoader, # noqa: F401 + TextLoader, # noqa: F401 + ) + + print("\n✓ Available document loaders:") + print(" - TextLoader: Load plain text files") + print(" - PDFLoader: Load PDF documents") + print(" - DocxLoader: Load Word documents") + print(" - MarkdownLoader: Load Markdown files") + print(" - LoaderFactory: Auto-detect and load files") + + # Example: Loading text files + print("\nExample: Loading a text file") + print( + """ + loader = TextLoader("documents/article.txt") + documents = loader.load() + for doc in documents: + print(f"Content: {doc.content[:100]}...") + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + print("Some document loaders may require additional dependencies") + + +def example_rag_pipeline(): + """ + Example 2: Building a RAG pipeline + + Demonstrates how to build a complete RAG pipeline with + document loading, embedding, retrieval, and generation. + """ + print("\n" + "=" * 60) + print("Example 2: Building a RAG Pipeline") + print("=" * 60) + + try: + from sage.middleware.operators.rag.pipeline import RAGPipeline # noqa: F401 + + print("\n✓ RAG Pipeline components:") + print(" 1. Document Loader: Load source documents") + print(" 2. Text Splitter: Split into chunks") + print(" 3. Embedder: Generate embeddings") + print(" 4. Vector Store: Store and index embeddings") + print(" 5. Retriever: Find relevant chunks") + print(" 6. Generator: Generate answers") + + print("\nExample pipeline setup:") + print( + """ + from sage.middleware.operators.rag.pipeline import RAGPipeline + from sage.libs.rag.document_loaders import TextLoader + + # Create pipeline + pipeline = RAGPipeline( + loader=TextLoader("knowledge_base/"), + chunk_size=512, + embedding_model="all-MiniLM-L6-v2", + llm_model="gpt-3.5-turbo" + ) + + # Build index + pipeline.build_index() + + # Query + answer = pipeline.query("What is SAGE?") + print(answer) + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + + +def example_vector_stores(): + """ + Example 3: Using vector stores + + Demonstrates how to use different vector store backends + for storing and retrieving embeddings. + """ + print("\n" + "=" * 60) + print("Example 3: Vector Store Integration") + print("=" * 60) + + print("\n✓ Supported vector stores:") + print(" - Milvus: Distributed vector database") + print(" - Chroma: Lightweight in-memory store") + print(" - FAISS: Facebook AI Similarity Search") + + print("\nExample: Using Milvus") + print( + """ + from sage.middleware.operators.rag.backends.milvus import MilvusBackend + from sage.middleware.operators.rag.pipeline import RAGPipeline + + # Create Milvus backend + milvus = MilvusBackend( + host="localhost", + port=19530, + collection_name="documents" + ) + + # Use in RAG pipeline + pipeline = RAGPipeline(vector_store=milvus) + """ + ) + + print("\nExample: Using ChromaDB") + print( + """ + from sage.middleware.operators.rag.backends.chroma import ChromaBackend + + # Create Chroma backend + chroma = ChromaBackend( + persist_directory="./chroma_db", + collection_name="documents" + ) + + # Add documents + chroma.add_documents(documents, embeddings) + + # Search + results = chroma.search(query_embedding, top_k=5) + """ + ) + + +def example_profiling(): + """ + Example 4: RAG performance profiling + + Demonstrates how to profile and optimize RAG pipelines + using the built-in profiler. + """ + print("\n" + "=" * 60) + print("Example 4: RAG Pipeline Profiling") + print("=" * 60) + + try: + from sage.middleware.operators.rag.profiler import Query_Profiler # noqa: F401 + + print("\n✓ RAG Profiler capabilities:") + print(" - Query profiling and analysis") + print(" - Complexity assessment") + print(" - Reasoning requirement detection") + print(" - Summarization strategy selection") + + print("\nExample profiling:") + print( + """ + from sage.middleware.operators.rag.profiler import Query_Profiler + from sage.middleware.operators.rag.pipeline import RAGPipeline + + # Create profiler + profiler = Query_Profiler(config={}) + + # Profile query to determine strategy + query_info = { + "need_joint_reasoning": True, + "complexity": "High", + "need_summarization": True, + "summarization_length": 100, + "n_info_items": 3 + } + result = profiler.execute(json.dumps(query_info)) + """ + ) + + except ImportError as e: + print(f"✗ Import error: {e}") + + +def example_advanced_retrieval(): + """ + Example 5: Advanced retrieval strategies + + Demonstrates advanced retrieval techniques like hybrid search, + reranking, and query expansion. + """ + print("\n" + "=" * 60) + print("Example 5: Advanced Retrieval Strategies") + print("=" * 60) + + print("\n✓ Advanced retrieval techniques:") + print(" - Hybrid search: Combine dense + sparse retrieval") + print(" - Reranking: Post-process retrieval results") + print(" - Query expansion: Enhance queries with synonyms") + print(" - Multi-hop retrieval: Iterative retrieval") + + print("\nExample: Hybrid search") + print( + """ + # Combine dense (embedding) and sparse (BM25) retrieval + dense_results = vector_store.search(query_embedding, top_k=20) + sparse_results = bm25_index.search(query_text, top_k=20) + + # Merge and rerank + combined = merge_results(dense_results, sparse_results) + reranked = reranker.rerank(query, combined, top_k=5) + """ + ) + + print("\nExample: Query expansion") + print( + """ + # Expand query with synonyms and related terms + original_query = "machine learning algorithms" + expanded_query = query_expander.expand(original_query) + # Result: "machine learning algorithms ML AI models neural networks" + + # Use expanded query for retrieval + results = vector_store.search(expanded_query, top_k=10) + """ + ) + + +def run_all_examples(): + """Run all examples in sequence.""" + print("\n" + "=" * 60) + print("SAGE RAG - Complete Examples") + print("=" * 60) + + example_document_loading() + example_rag_pipeline() + example_vector_stores() + example_profiling() + example_advanced_retrieval() + + print("\n" + "=" * 60) + print("✓ All examples completed") + print("=" * 60) + print("\nFor more information:") + print("- See rag/README.md for detailed documentation") + print("- Check examples/ for complete working examples") + print("- Visit docs/ for RAG best practices") + + +if __name__ == "__main__": + run_all_examples() diff --git a/tutorials/L4-middleware/sage_db/README.md b/tutorials/L4-middleware/sage_db/README.md new file mode 100644 index 0000000..1ae2385 --- /dev/null +++ b/tutorials/L4-middleware/sage_db/README.md @@ -0,0 +1,42 @@ +# SAGE-DB Examples + +本目录包含两类与 `sage_flow` 风格一致的 SAGE-DB 示例： + +1. 应用（Application）方式：`hello_sage_db_app.py` + + - 直接使用 `SageDB` Python API，插入少量向量后进行一次查询 + +1. 服务（Service）方式：`hello_sage_db_service.py` + + - 以微服务形式封装 SAGE-DB，注册到 `LocalEnvironment` 后由外部推入向量并执行查询 + +## 运行前置 + +SAGE-DB 依赖已编译的 Python 扩展模块 `_sage_db`。如未安装，可运行： + +```bash +sage extensions install sage_db # 若需重新编译可加 --force +``` + +该命令会在仓库根目录下构建并同步 `_sage_db` 扩展，示例中的相对导入即可找到。 + +## 运行示例 + +应用方式： + +```bash +python examples/service/sage_db/hello_sage_db_app.py +``` + +服务方式： + +```bash +python examples/service/sage_db/hello_sage_db_service.py +``` + +## 说明 + +- Canonical Python API 位于 + `packages/sage-middleware/src/sage/middleware/components/sage_db/python/sage_db.py` +- 为保持仓库内运行便捷，示例脚本在未安装包时会自动将 `packages/*/src` 加入 `sys.path` +- 若你需要对外暴露更丰富的接口，可在 `python/micro_service/sage_db_service.py` 基础上扩展（如批量删除、索引保存/加载、带条件过滤的查询等） diff --git a/tutorials/L4-middleware/sage_db/hello_sage_db_app.py b/tutorials/L4-middleware/sage_db/hello_sage_db_app.py new file mode 100644 index 0000000..b363d17 --- /dev/null +++ b/tutorials/L4-middleware/sage_db/hello_sage_db_app.py @@ -0,0 +1,69 @@ +import logging + +import numpy as np + +# Add repo package paths if needed +try: + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.middleware.components.sage_db.python.sage_db import ( + DatabaseConfig, + IndexType, + SageDB, + ) +except ModuleNotFoundError: + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for p in here.parents: + if (p / "packages").exists(): + repo_root = p + break + assert repo_root is not None + for p in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(p)) + + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.middleware.components.sage_db.python.sage_db import ( + DatabaseConfig, + IndexType, + SageDB, + ) + + +def main(): + dim = 4 + # Create DB with config + cfg = DatabaseConfig(dim) + cfg.index_type = IndexType.AUTO + db = SageDB.from_config(cfg) + + # Add few vectors + total = 5 + for uid in range(total): + vec = np.arange(dim, dtype=np.float32) + uid + db.add(vec, {"tag": "demo", "uid": str(uid)}) + + # Search + query = np.array([0, 1, 2, 3], dtype=np.float32) + results = db.search(query, k=3) + print("Top-3 results:") + for r in results: + print(f" id={r.id}, score={r.score:.4f}, md={dict(r.metadata)}") + + # Stats + print("DB size:", db.size) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L4-middleware/sage_db/hello_sage_db_service.py b/tutorials/L4-middleware/sage_db/hello_sage_db_service.py new file mode 100644 index 0000000..5c8bb55 --- /dev/null +++ b/tutorials/L4-middleware/sage_db/hello_sage_db_service.py @@ -0,0 +1,70 @@ +import logging + +import numpy as np + +# Try direct imports; fallback to repo paths if needed +try: + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.middleware.components.sage_db.python.micro_service.sage_db_service import ( + SageDBService, + ) +except ModuleNotFoundError: + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for p in here.parents: + if (p / "packages").exists(): + repo_root = p + break + assert repo_root is not None + for p in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(p)) + + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.middleware.components.sage_db.python.micro_service.sage_db_service import ( + SageDBService, + ) + + +def main(): + env = LocalEnvironment("hello_sage_db_service") + + # Register service + env.register_service( + "hello_sage_db_service", + SageDBService, + dimension=4, + index_type="AUTO", + ) + + # Create service instance + svc_factory = env.service_factories["hello_sage_db_service"] + svc: SageDBService = svc_factory.create_service() + + # Insert demo vectors + for uid in range(3): + vec = np.arange(4, dtype=np.float32) + uid + svc.add(vec, {"uid": str(uid), "tag": "svc_demo"}) + + # Search + results = svc.search([0, 1, 2, 3], k=2) + print("Service search results:") + for r in results: + print(r) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L4-middleware/sage_db/workflow_demo.py b/tutorials/L4-middleware/sage_db/workflow_demo.py new file mode 100644 index 0000000..0d308f6 --- /dev/null +++ b/tutorials/L4-middleware/sage_db/workflow_demo.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""LLM workflow DAG demo that wires SageDB retrieval into the SAGE pipeline. + +@test:allow-demo +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Sequence + +import numpy as np + +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.source_function import SourceFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment +from sage.middleware.operators.rag import QAPromptor + +# Ensure repository packages are importable when running the script directly +REPO_ROOT = Path(__file__).resolve().parents[1] +PACKAGE_SRC_ROOTS = [ + REPO_ROOT / "packages" / "sage" / "src", + REPO_ROOT / "packages" / "sage-common" / "src", + REPO_ROOT / "packages" / "sage-kernel" / "src", + REPO_ROOT / "packages" / "sage-libs" / "src", + REPO_ROOT / "packages" / "sage-middleware" / "src", + REPO_ROOT / "packages" / "sage-tools" / "src", +] +for path in PACKAGE_SRC_ROOTS: + if path.exists() and str(path) not in sys.path: + sys.path.insert(0, str(path)) + + +try: + from sage.common.components.sage_embedding.embedding_model import EmbeddingModel + from sage.middleware.components.sage_db.python.micro_service.sage_db_service import ( + SageDBService, + ) +except ImportError as exc: # pragma: no cover - surface build guidance early + if "_sage_db" in str(exc): + raise SystemExit( + "❌ SageDB native extension not found. Install it before running this demo:\n" + " sage extensions install sage_db # add --force to rebuild" + ) from exc + raise + +SERVICE_NAME = "vector_store_service" + + +@dataclass +class KnowledgeEntry: + title: str + text: str + topic: str + tags: list[str] + + +KNOWLEDGE_BASE: list[KnowledgeEntry] = [ + KnowledgeEntry( + title="SAGE 推理管道总览", + text=( + "SAGE 的大模型推理 DAG 通过 LocalEnvironment 串联 Source、Map 与 Sink 节点，" + "在 JobManager 中调度执行，并支持服务调用与反馈回路。" + ), + topic="architecture", + tags=["dag", "runtime", "overview"], + ), + KnowledgeEntry( + title="SageDB 与 RAG 集成", + text=( + "SageDB 可以作为 RAG 检索后端，结合嵌入模型将文档向量化后写入，" + "在查询阶段通过服务调用提供 Top-K 相似片段。" + ), + topic="rag", + tags=["sage_db", "retrieval", "integration"], + ), + KnowledgeEntry( + title="服务编排节点", + text=( + "LocalEnvironment.register_service 会在 DAG 执行时自动注入可复用的服务实例，" + "算子可通过 self.call_service['name'] 以同步方式访问数据库或缓存。" + ), + topic="runtime", + tags=["service", "call_service", "localenvironment"], + ), + KnowledgeEntry( + title="Prompt 构建阶段", + text=( + "QAPromptor 接收检索结果列表，拼装 system 与 user 消息，为 OpenAI 兼容生成器提供提示模板。" + ), + topic="prompt", + tags=["promptor", "qa", "template"], + ), +] + +DEFAULT_QUERIES = [ + "SAGE 的 DAG 如何把 SageDB 检索串进推理流程?", + "QAPromptor 在这个流程里起到什么作用?", +] + + +class BootstrappedSageDBService(SageDBService): + """Wrapper service that loads embeddings at construction time.""" + + def __init__( + self, + *, + initial_vectors: Sequence[Sequence[float]] | np.ndarray | None = None, + initial_metadata: Sequence[dict[str, str]] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + if initial_vectors is not None: + vectors = ( + np.asarray(initial_vectors, dtype=np.float32) + if not isinstance(initial_vectors, np.ndarray) + else initial_vectors.astype(np.float32, copy=False) + ) + + if vectors.size == 0: + return + + if vectors.ndim != 2: + raise ValueError("initial_vectors must be a 2D array-like of shape (N, dim)") + + expected_count = vectors.shape[0] + if initial_metadata is None: + metadata = [{} for _ in range(expected_count)] + else: + metadata = list(initial_metadata) + if len(metadata) != expected_count: + raise ValueError("initial_metadata length must match number of initial_vectors") + + self.add_batch(vectors, metadata) + # Build index once after ingestion to accelerate queries + self._db.build_index() + + +class QuerySource(SourceFunction): + def __init__(self, queries: Iterable[str], **kwargs: Any) -> None: + super().__init__(**kwargs) + self._queries = list(queries) + self._cursor = 0 + + def execute(self) -> dict[str, str] | None: + if self._cursor >= len(self._queries): + return None + question = self._queries[self._cursor] + self._cursor += 1 + return {"query": question} + + +class SageDBRetrieverNode(MapFunction): + def __init__( + self, + embedder_config: dict[str, Any], + *, + service_name: str = SERVICE_NAME, + top_k: int = 3, + service_timeout: float = 10.0, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self.embedder = EmbeddingModel(**embedder_config) + self.service_name = service_name + self.top_k = top_k + self.service_timeout = service_timeout + + def execute(self, payload: dict[str, Any]) -> dict[str, Any]: + query = payload.get("query", "") + if not query: + return {**payload, "results": [], "retrieved_docs": []} + + query_vector = np.asarray(self.embedder.embed(query), dtype=np.float32) + service = self.call_service(self.service_name) + raw_results = service.search(query_vector, k=self.top_k, timeout=self.service_timeout) + + formatted_results: list[dict[str, Any]] = [] + corpus_snippets: list[str] = [] + references: list[dict[str, Any]] = [] + + for item in raw_results: + metadata = dict(item.get("metadata") or {}) + snippet = metadata.get("text", "") + formatted_results.append( + { + "text": snippet, + "score": item.get("score"), + "metadata": metadata, + } + ) + corpus_snippets.append(snippet) + references.append( + { + "title": metadata.get("title", "unknown"), + "score": float(item.get("score", 0.0)), + "tags": (metadata.get("tags", "").split(",") if metadata.get("tags") else []), + } + ) + + enriched = dict(payload) + enriched.update( + { + "results": formatted_results, + "retrieved_docs": corpus_snippets, + "references": references, + } + ) + return enriched + + +class MockLLMGenerator(MapFunction): + """Minimal generator that emulates an LLM using retrieved passages.""" + + def execute(self, data: list[Any]) -> dict[str, Any]: + if not isinstance(data, list) or len(data) != 2: + raise ValueError("Generator expects QAPromptor output: [original_payload, messages]") + original, messages = data + top_hit = None + if isinstance(original, dict): + hits = original.get("results", []) or [] + top_hit = hits[0] if hits else None + answer: str + if top_hit: + title = top_hit["metadata"].get("title", "资料") + answer = f"参考《{title}》中的要点：{top_hit['text']} —— 该内容经 SageDB 检索返回。" + else: + answer = "知识库没有检索到匹配内容，建议扩充 SageDB 语料。" + + enriched = dict(original) + enriched["prompt_messages"] = messages + enriched["answer"] = answer + return enriched + + +class ConsoleReporter(MapFunction): + def execute(self, payload: dict[str, Any]) -> dict[str, Any]: + query = payload.get("query", "") + answer = payload.get("answer", "") + refs = payload.get("references", []) + + print("\n================ Pipeline Result ================") + print(f"❓ Query : {query}") + print(f"💡 Answer: {answer}") + if refs: + print("📚 References:") + for idx, ref in enumerate(refs, start=1): + title = ref.get("title") + tags = ", ".join(ref.get("tags", [])) + score = ref.get("score") + print(f" {idx}. {title} (score={score:.4f}, tags={tags})") + else: + print("📚 References: ") + print("================================================\n") + return payload + + +def build_embeddings(entries: Sequence[KnowledgeEntry], model: EmbeddingModel) -> np.ndarray: + vectors = [] + for item in entries: + vectors.append(model.embed(item.text)) + return np.asarray(vectors, dtype=np.float32) + + +def prepare_metadata(entries: Sequence[KnowledgeEntry]) -> list[dict[str, str]]: + metadata: list[dict[str, str]] = [] + for item in entries: + metadata.append( + { + "title": item.title, + "text": item.text, + "topic": item.topic, + "tags": ",".join(item.tags), # Serialize list to comma-separated string + } + ) + return metadata + + +def run_pipeline(top_k: int, queries: Sequence[str]) -> None: + CustomLogger.disable_global_console_debug() + + embedder_config = {"method": "mockembedder", "fixed_dim": 128} + embedder = EmbeddingModel(**embedder_config) + + vectors = build_embeddings(KNOWLEDGE_BASE, embedder) + metadata = prepare_metadata(KNOWLEDGE_BASE) + + env = LocalEnvironment("sage_db_workflow_demo") + env.register_service( + SERVICE_NAME, + BootstrappedSageDBService, + dimension=embedder.get_dim(), + index_type="AUTO", + initial_vectors=vectors, + initial_metadata=metadata, + ) + + ( + env.from_source(QuerySource, queries) + .map( + SageDBRetrieverNode, + embedder_config, + service_name=SERVICE_NAME, + top_k=top_k, + ) + .map(QAPromptor, {"use_short_answer": False}) + .map(MockLLMGenerator) + .map(ConsoleReporter) + ) + + env.submit() + + # Wait for processing to complete + import time + + time.sleep(5) # Allow enough time for processing + + # Clean up environment + env.stop() + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--top-k", + type=int, + default=3, + help="Number of neighbors to fetch from SageDB for each query.", + ) + parser.add_argument( + "--query", + action="append", + help="Override default demo queries (can be specified multiple times).", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> None: + args = parse_args(argv or sys.argv[1:]) + queries = args.query if args.query else DEFAULT_QUERIES + run_pipeline(args.top_k, queries) + + +if __name__ == "__main__": + main() diff --git a/tutorials/L4-middleware/sage_flow/README.md b/tutorials/L4-middleware/sage_flow/README.md new file mode 100644 index 0000000..6115f82 --- /dev/null +++ b/tutorials/L4-middleware/sage_flow/README.md @@ -0,0 +1,38 @@ +# Sage Flow Examples + +本目录包含两类示例，风格与 `examples/service/neuromem_vdb` 保持一致： + +1. 应用（Application）方式：`hello_sage_flow_app.py` + + - 使用 SAGE 的 pipeline API（from_batch/sink 等）构建一个最小向量处理应用 + +1. 服务（Service）方式：`hello_sage_flow_service.py` + + - 以微服务形式运行 Sage Flow，支持从外部推入向量并执行一次消费 + +## 运行示例 + +建议先编译 C++ 扩展（若尚未构建）： + +```bash +# 可选：确保 _sage_flow 扩展已构建 +# 构建目录通常在 packages/sage-middleware/src/sage/middleware/components/sage_flow/build/ +``` + +运行应用方式： + +```bash +python examples/service/sage_flow/hello_sage_flow_app.py +``` + +运行服务方式： + +```bash +python examples/service/sage_flow/hello_sage_flow_service.py +``` + +## 说明 + +- Minimal 服务 `SageFlowService` 位于 + `packages/sage-middleware/src/sage/middleware/components/sage_flow/micro_service/sage_flow_service.py` +- 当前服务主要演示了数据注入与执行路径，后续可扩展 topk/itopk/index 等算子，将其对外暴露为 service API diff --git a/tutorials/L4-middleware/sage_flow/hello_sage_flow_app.py b/tutorials/L4-middleware/sage_flow/hello_sage_flow_app.py new file mode 100644 index 0000000..d950968 --- /dev/null +++ b/tutorials/L4-middleware/sage_flow/hello_sage_flow_app.py @@ -0,0 +1,97 @@ +import logging +import time + +import numpy as np + +# Add repo package paths if needed +try: + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.middleware.components.sage_flow.python.sage_flow import ( + SimpleStreamSource, + StreamEnvironment, + ) +except ModuleNotFoundError: + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for p in here.parents: + if (p / "packages").exists(): + repo_root = p + break + assert repo_root is not None + for p in [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ]: + sys.path.insert(0, str(p)) + + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.middleware.components.sage_flow.python.sage_flow import ( + SimpleStreamSource, + StreamEnvironment, + ) + + +def main(): + # 直接使用 sage_flow 的绑定构建一个最小 pipeline + env = StreamEnvironment() + source = SimpleStreamSource("hello_source") + + # 通过 Python 回调挂载一个 sink，用于验证处理的记录数 + processed = {"count": 0} + + def on_sink(uid: int, ts: int): + # 简单累加计数，验证 pipeline 正常运行，并打印出处理记录 + processed["count"] += 1 + print(f"[py_sink] processed uid={uid}, ts={ts}") + + # 将 sink 直接挂在 source 上（返回下游 stream，但无需手动加入 env） + source.write_sink_py("py_sink", on_sink) + + # 注入几条向量 + dim = 4 + total = 5 + for uid in range(total): + vec = np.arange(dim, dtype=np.float32) + uid + ts = int(time.time() * 1000) + source.addRecord(uid, ts, vec) + + # 将源加入环境并执行（注意：需要在添加数据之前添加到环境） + env.addStream(source) + logging.info("execute start") + env.execute() + logging.info("execute done") + + # 等待异步处理完成 + max_wait_time = 5.0 # 最多等待5秒 + wait_interval = 0.1 # 每次等待100ms + elapsed_time = 0.0 + + while processed["count"] < total and elapsed_time < max_wait_time: + time.sleep(wait_interval) + elapsed_time += wait_interval + + logging.info(f"processed count: {processed['count']} (expected: {total})") + + # 简单校验：处理的记录数应等于注入的记录数 + if processed["count"] != total: + logging.warning(f"处理记录数不匹配: 实际 {processed['count']}, 期望 {total}") + logging.warning("这可能是由于异步处理延迟或数据源配置问题造成的") + # 在示例中不要抛出异常，只是警告 + # assert False, f"processed count {processed['count']} != expected {total}" + else: + logging.info("✅ 数据处理验证通过") + + +if __name__ == "__main__": + # 显示标准 logging 的 INFO 级别日志 + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + # 关闭自定义全局控制台日志（不影响标准 logging 和 print 输出） + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L4-middleware/sage_flow/hello_sage_flow_service.py b/tutorials/L4-middleware/sage_flow/hello_sage_flow_service.py new file mode 100644 index 0000000..54200e1 --- /dev/null +++ b/tutorials/L4-middleware/sage_flow/hello_sage_flow_service.py @@ -0,0 +1,105 @@ +import logging +import time + +import numpy as np + +# Try direct imports; if running from repo without installation, add local package paths +try: + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.middleware.components.sage_flow.python.micro_service.sage_flow_service import ( + SageFlowService, + ) +except ModuleNotFoundError: + import sys + from pathlib import Path + + here = Path(__file__).resolve() + repo_root = None + for p in here.parents: + if (p / "packages").exists(): + repo_root = p + break + if repo_root is None: + repo_root = here.parents[3] + + src_paths = [ + repo_root / "packages" / "sage" / "src", + repo_root / "packages" / "sage-common" / "src", + repo_root / "packages" / "sage-kernel" / "src", + repo_root / "packages" / "sage-middleware" / "src", + repo_root / "packages" / "sage-libs" / "src", + repo_root / "packages" / "sage-tools" / "src", + ] + for p in src_paths: + sys.path.insert(0, str(p)) + + from sage.common.utils.logging.custom_logger import CustomLogger + from sage.kernel.api.local_environment import LocalEnvironment + from sage.middleware.components.sage_flow.python.micro_service.sage_flow_service import ( + SageFlowService, + ) + + +def main(): + env = LocalEnvironment("hello_sage_flow_service") + + # 注册 SageFlowService（与 neuromem_vdb 风格一致） + env.register_service( + "hello_sage_flow_service", + SageFlowService, + dim=4, + dtype="Float32", + ) + + # 获取服务实例 + service_factory = env.service_factories["hello_sage_flow_service"] + svc: SageFlowService = service_factory.create_service() + + # 附加一个可见的 Python sink，以便运行时在控制台看到处理结果 + processed = {"count": 0} + + def on_sink(uid: int, ts: int): + processed["count"] += 1 + print(f"[service py_sink] processed uid={uid}, ts={ts}", flush=True) + + svc.set_sink(on_sink) + + # 推入几条数据 + total = 3 + for uid in range(total): + vec = np.arange(4, dtype=np.float32) + uid + svc.push(uid, vec) + + # 运行一次，将队列中的数据消费（内部会执行 env.execute()） + svc.run() + + # 等待异步处理完成（参考 hello_sage_flow_app.py 的实现） + max_wait_time = 5.0 # 最多等待5秒 + wait_interval = 0.1 # 每次等待100ms + elapsed_time = 0.0 + + while processed["count"] < total and elapsed_time < max_wait_time: + time.sleep(wait_interval) + elapsed_time += wait_interval + + logging.info(f"processed count: {processed['count']} (expected: {total})") + print(f"processed count: {processed['count']}") + + # 简单校验：处理的记录数应等于注入的记录数 + if processed["count"] != total: + logging.warning(f"处理记录数不匹配: 实际 {processed['count']}, 期望 {total}") + logging.warning("这可能是由于异步处理延迟或数据源配置问题造成的") + # 在示例中不要抛出异常，只是警告 + else: + logging.info("✅ 数据处理验证通过") + + logging.info("Service demo done") + + +if __name__ == "__main__": + # 显示标准 logging 的 INFO 级别日志 + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + # 关闭自定义全局控制台日志（不影响标准 logging 和 print 输出） + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/L4-middleware/sage_tsdb/README.md b/tutorials/L4-middleware/sage_tsdb/README.md new file mode 100644 index 0000000..b74bf17 --- /dev/null +++ b/tutorials/L4-middleware/sage_tsdb/README.md @@ -0,0 +1,262 @@ +# SAGE TSDB Tutorial Examples + +This directory contains comprehensive examples demonstrating SAGE TSDB integration with SAGE DAG +workflows. + +## Examples Overview + +### 1. Basic DAG Example (`basic_dag_example.py`) + +Demonstrates fundamental SAGE TSDB operations in a DAG pipeline: + +- **Time series data ingestion** through SAGE pipeline +- **Real-time querying** with sliding windows +- **Window-based aggregation** (tumbling windows) +- **Service integration** for simplified API access + +```bash +python basic_dag_example.py +``` + +**Key Features:** + +- `TimeSeriesDataSource`: Generate simulated sensor data +- `TSDBIngestNode`: Ingest data into TSDB +- `TSDBQueryNode`: Query and compute statistics +- `TSDBWindowAggregateNode`: Perform window aggregations +- `ResultPrinterSink`: Display results + +### 2. Stream Join DAG Example (`stream_join_dag_example.py`) + +Demonstrates out-of-order stream join capabilities: + +- **Out-of-order data handling** with watermarking +- **Window-based stream join** between two data streams +- **Join statistics** and performance monitoring +- **DAG integration** for stream processing + +```bash +python stream_join_dag_example.py +``` + +**Key Features:** + +- `OutOfOrderStreamSource`: Generate streams with simulated delays +- `StreamJoinNode`: Join two streams with configurable windows +- `JoinResultSink`: Display join results and statistics +- Configurable disorder probability and max delay + +### 3. Advanced DAG Example (`advanced_dag_example.py`) + +Demonstrates advanced analytics and real-time processing: + +- **Multi-sensor data processing** +- **Real-time anomaly detection** using statistical methods +- **Complex window aggregations** (multiple aggregation types) +- **Service-based analytics** + +```bash +python advanced_dag_example.py +``` + +**Key Features:** + +- `MultiSensorSource`: Generate data from multiple sensors +- `AnomalyDetector`: Detect anomalies using z-score +- `AggregationNode`: Apply multiple aggregation strategies +- `AnalyticsSink`: Display analytics and anomaly alerts + +## Running the Examples + +### Prerequisites + +Ensure SAGE TSDB is installed (via quickstart or editable install): + +```bash +# From SAGE root directory +./quickstart.sh --dev --yes # or install middleware in editable mode +# pip install -e packages/sage-middleware +``` + +### Run Individual Examples + +```bash +# From SAGE root directory +cd examples/tutorials/sage_tsdb + +# Basic example +python basic_dag_example.py + +# Stream join example +python stream_join_dag_example.py + +# Advanced example +python advanced_dag_example.py +``` + +### Run All Examples + +```bash +# Run all examples in sequence +for example in basic_dag_example.py stream_join_dag_example.py advanced_dag_example.py; do + echo "Running $example..." + python $example + echo "" +done +``` + +## Example Architecture + +### Basic Pipeline Flow + +``` +DataSource → IngestNode → QueryNode → AggregateNode → Sink +``` + +### Stream Join Flow + +``` +LeftStream ──┐ + ├─→ JoinNode → Sink +RightStream ─┘ +``` + +### Advanced Analytics Flow + +``` +MultiSensorSource → Ingest → AnomalyDetector → Aggregation → AnalyticsSink +``` + +## Key Concepts Demonstrated + +### 1. Time Series Data Management + +- Efficient storage with timestamp-based indexing +- Tag-based filtering for multi-dimensional queries +- Window-based aggregations + +### 2. Out-of-Order Processing + +- Watermarking for late data handling +- Buffer management for stream join +- Configurable delay tolerance + +### 3. DAG Integration + +- Node-based data transformation +- Pipeline composition +- Source-to-sink data flow + +### 4. Real-Time Analytics + +- Anomaly detection using statistical methods +- Multi-window aggregations +- Performance monitoring and statistics + +## Configuration Options + +### TSDB Configuration + +```python +from sage.middleware.components.sage_tsdb import SageTSDBServiceConfig + +config = SageTSDBServiceConfig( + enable_compression=False, + max_memory_mb=1024, + default_window_size=60000, # 60 seconds + default_aggregation="avg", +) +``` + +### Stream Join Configuration + +```python +join_config = { + "window_size": 5000, # 5-second join window + "max_delay": 3000, # 3-second max delay + "join_key": "sensor_id", # Optional equi-join key +} +``` + +### Window Aggregation Configuration + +```python +window_config = { + "window_type": "tumbling", # tumbling/sliding/session + "window_size": 10000, # 10 seconds + "aggregation": "avg", # sum/avg/min/max/count/stddev +} +``` + +## Performance Considerations + +### Ingestion + +- Use batch operations for higher throughput +- Balance between latency and batch size +- Consider memory limits for large datasets + +### Queries + +- Use tag filtering to reduce search space +- Limit result size for large time ranges +- Cache frequently accessed data + +### Stream Join + +- Adjust window size based on data arrival patterns +- Configure max_delay to match expected out-of-order delays +- Monitor buffer sizes to prevent memory issues + +## Troubleshooting + +### Issue: Module not found + +```bash +# Ensure SAGE is in Python path +export PYTHONPATH=/home/shuhao/SAGE:$PYTHONPATH + +# Or install SAGE in development mode +cd /home/shuhao/SAGE +pip install -e . +``` + +### Issue: C++ library not found + +```bash +# Build sageTSDB C++ core +cd packages/sage-middleware/src/sage/middleware/components/sage_tsdb/sageTSDB +./build.sh +``` + +### Issue: Out of memory + +- Reduce batch sizes +- Decrease window sizes +- Increase max_delay to allow earlier data release + +## Next Steps + +After exploring these examples: + +1. **Customize Data Sources**: Adapt to your data format and sources +1. **Add Custom Algorithms**: Implement domain-specific processing +1. **Integrate with Other Components**: Combine with sage_db, sage_flow, etc. +1. **Deploy in Production**: Scale to handle real workloads + +## Related Documentation + +- [SAGE TSDB README](../../../packages/sage-middleware/src/sage/middleware/components/sage_tsdb/README.md) +- [Submodule Setup Guide](../../../packages/sage-middleware/src/sage/middleware/components/sage_tsdb/SUBMODULE_SETUP.md) +- [sage-developer Guide](../../../DEVELOPER.md) + +## Support + +For issues or questions: + +- GitHub Issues: https://github.com/intellistream/SAGE/issues +- Email: shuhao_zhang@hust.edu.cn + +______________________________________________________________________ + +**Happy coding with SAGE TSDB! 🚀** diff --git a/tutorials/L4-middleware/sage_tsdb/__init__.py b/tutorials/L4-middleware/sage_tsdb/__init__.py new file mode 100644 index 0000000..ad14f9c --- /dev/null +++ b/tutorials/L4-middleware/sage_tsdb/__init__.py @@ -0,0 +1,9 @@ +""" +SAGE TSDB Examples + +Time series database examples demonstrating: +- Basic time series operations +- Stream join with out-of-order data +- Window-based aggregations +- Integration with SAGE DAG workflows +""" diff --git a/tutorials/L4-middleware/sage_tsdb/advanced_dag_example.py b/tutorials/L4-middleware/sage_tsdb/advanced_dag_example.py new file mode 100644 index 0000000..01ba060 --- /dev/null +++ b/tutorials/L4-middleware/sage_tsdb/advanced_dag_example.py @@ -0,0 +1,265 @@ +""" +Advanced SAGE TSDB Example + +This example demonstrates advanced features: +1. Multi-sensor data ingestion +2. Complex window aggregations +3. Real-time anomaly detection +4. Integration with SAGE service layer +""" + +from datetime import datetime +from typing import Any + +import numpy as np + +from sage.middleware.components.sage_tsdb import SageTSDB, TimeRange + + +def generate_multi_sensor_data( + num_sensors: int = 3, points_per_sensor: int = 30 +) -> list[dict[str, Any]]: + """Generate data from multiple sensors""" + data_points = [] + base_time = int(datetime.now().timestamp() * 1000) + + for sensor_id in range(num_sensors): + for i in range(points_per_sensor): + timestamp = base_time + i * 1000 + # Different patterns for different sensors + if sensor_id == 0: + value = 20 + 5 * np.sin(i * 0.2) + np.random.randn() + elif sensor_id == 1: + value = 30 + 3 * np.cos(i * 0.15) + np.random.randn() * 0.5 + else: + value = 25 + 2 * np.sin(i * 0.25) + np.random.randn() * 1.5 + + data_points.append( + { + "timestamp": timestamp, + "value": value, + "sensor_id": f"sensor_{sensor_id:02d}", + "location": f"room_{sensor_id % 3}", + "type": "temperature", + } + ) + + return data_points + + +def detect_anomalies( + db: SageTSDB, data_points: list[dict[str, Any]], threshold_std: float = 2.5 +) -> list[dict[str, Any]]: + """Detect anomalies in time series data""" + results = [] + + for data in data_points: + # Query historical data + time_range = TimeRange( + start_time=data["timestamp"] - 30000, # Last 30 seconds + end_time=data["timestamp"], + ) + + historical = db.query(time_range=time_range) + + is_anomaly = False + anomaly_score = 0.0 + + if len(historical) > 5: # Need enough data + values: np.ndarray = np.array([h.value for h in historical]) + mean = np.mean(values) + std = np.std(values) + + if std > 0: + z_score = abs((data["value"] - mean) / std) + is_anomaly = z_score > threshold_std + anomaly_score = z_score + + results.append( + { + **data, + "is_anomaly": is_anomaly, + "anomaly_score": anomaly_score, + "historical_count": len(historical), + } + ) + + return results + + +def compute_window_statistics( + db: SageTSDB, data_points: list[dict[str, Any]] +) -> list[dict[str, Any]]: + """Compute window-based statistics""" + results = [] + + for i, data in enumerate(data_points): + # Every 10 points, compute aggregations + if (i + 1) % 10 == 0: + # Query recent data + time_range = TimeRange(start_time=data["timestamp"] - 30000, end_time=data["timestamp"]) + + recent_data = db.query(time_range=time_range) + + # Compute statistics + if recent_data: + values: np.ndarray = np.array([r.value for r in recent_data]) + aggregations = { + "count": len(values), + "mean": np.mean(values), + "std": np.std(values), + "min": np.min(values), + "max": np.max(values), + } + else: + aggregations = {"count": 0} + + results.append( + { + **data, + "aggregations": aggregations, + "has_aggregation": True, + } + ) + else: + results.append({**data, "has_aggregation": False}) + + return results + + +def example_advanced_analytics(): + """Advanced analytics with anomaly detection""" + print("\n" + "=" * 60) + print("Advanced Analytics Example") + print("=" * 60) + + # Create TSDB instance + db = SageTSDB() + + # Generate and ingest multi-sensor data + data_points = generate_multi_sensor_data(num_sensors=3, points_per_sensor=40) + print(f"\nIngesting {len(data_points)} multi-sensor data points...") + + for point in data_points: + db.add( + timestamp=point["timestamp"], + value=point["value"], + tags={ + "sensor_id": point["sensor_id"], + "location": point["location"], + "type": point["type"], + }, + ) + + # Detect anomalies + print("\nDetecting anomalies...") + anomaly_results = detect_anomalies(db, data_points, threshold_std=2.0) + + # Count and display anomalies + anomalies = [r for r in anomaly_results if r["is_anomaly"]] + print(f"Found {len(anomalies)} anomalies out of {len(data_points)} points") + + if anomalies: + print("\nAnomalies detected:") + for anomaly in anomalies[:5]: # Show first 5 + print( + f" Sensor: {anomaly['sensor_id']}, Value: {anomaly['value']:.2f}, Score: {anomaly['anomaly_score']:.2f}" + ) + + # Compute window statistics + print("\nComputing window statistics...") + window_results = compute_window_statistics(db, data_points) + + aggregation_count = sum(1 for r in window_results if r["has_aggregation"]) + print(f"Computed {aggregation_count} window aggregations") + + # Show sample aggregations + for result in window_results: + if result["has_aggregation"]: + agg = result["aggregations"] + print(f"\nAggregation for {result['sensor_id']}:") + print(f" Count: {agg['count']}") + print(f" Mean: {agg['mean']:.2f}") + print(f" Min: {agg['min']:.2f}") + print(f" Max: {agg['max']:.2f}") + break + + +def example_multi_sensor_comparison(): + """Compare data from multiple sensors""" + print("\n" + "=" * 60) + print("Multi-Sensor Comparison Example") + print("=" * 60) + + # Create separate TSDB for each sensor + sensors_db = { + "sensor_00": SageTSDB(), + "sensor_01": SageTSDB(), + "sensor_02": SageTSDB(), + } + + # Generate and ingest data + data_points = generate_multi_sensor_data(num_sensors=3, points_per_sensor=25) + print("\nIngesting data from 3 sensors...") + + for point in data_points: + sensor_id = point["sensor_id"] + if sensor_id in sensors_db: + sensors_db[sensor_id].add( + timestamp=point["timestamp"], + value=point["value"], + tags={ + "location": point["location"], + "type": point["type"], + }, + ) + + # Compare sensor statistics + print("\nSensor Comparison:") + print(f"{'Sensor':<15} {'Points':<10} {'Mean':<10} {'Std':<10}") + print("-" * 45) + + for sensor_id, db in sensors_db.items(): + time_range = TimeRange( + start_time=0, end_time=int(datetime.now().timestamp() * 1000) + 100000 + ) + results = db.query(time_range) + + if results: + values: np.ndarray = np.array([r.value for r in results]) + mean = np.mean(values) + std = np.std(values) + print(f"{sensor_id:<15} {len(results):<10} {mean:<10.2f} {std:<10.2f}") + + # Database statistics + print("\nDatabase Statistics:") + for sensor_id, db in sensors_db.items(): + stats = db.get_stats() + print(f"\n{sensor_id}:") + for key, value in stats.items(): + print(f" {key}: {value}") + + +if __name__ == "__main__": + print("\n" + "=" * 60) + print("SAGE TSDB Advanced Examples") + print("=" * 60) + + try: + # Run advanced analytics + example_advanced_analytics() + + print("\n" + "-" * 60 + "\n") + + # Run multi-sensor comparison + example_multi_sensor_comparison() + + print("\n" + "=" * 60) + print("All examples completed successfully!") + print("=" * 60) + + except Exception as e: + print(f"\nError running examples: {e}") + import traceback + + traceback.print_exc() diff --git a/tutorials/L4-middleware/sage_tsdb/basic_dag_example.py b/tutorials/L4-middleware/sage_tsdb/basic_dag_example.py new file mode 100644 index 0000000..f31ad14 --- /dev/null +++ b/tutorials/L4-middleware/sage_tsdb/basic_dag_example.py @@ -0,0 +1,211 @@ +""" +Basic SAGE TSDB Example + +This example demonstrates: +1. Creating a time series database +2. Ingesting streaming data +3. Querying and aggregating time series data +""" + +from datetime import datetime +from typing import Any + +import numpy as np + +from sage.middleware.components.sage_tsdb import SageTSDB, TimeRange + + +def generate_time_series_data( + num_points: int = 100, sensor_id: str = "sensor_01" +) -> list[dict[str, Any]]: + """Generate simulated time series data""" + data_points = [] + base_time = int(datetime.now().timestamp() * 1000) + + for i in range(num_points): + timestamp = base_time + i * 1000 # 1 second intervals + value = 20 + 5 * np.sin(i * 0.1) + np.random.randn() + + data_points.append( + { + "timestamp": timestamp, + "value": value, + "tags": { + "sensor_id": sensor_id, + "location": "room_a", + "unit": "celsius", + }, + } + ) + + return data_points + + +def example_basic_ingestion_and_query(): + """Example: Basic TSDB ingestion and query""" + print("\n" + "=" * 60) + print("Example 1: Basic Ingestion and Query") + print("=" * 60) + + # Create TSDB instance + db = SageTSDB() + + # Generate and ingest data + data_points = generate_time_series_data(num_points=50, sensor_id="temp_sensor") + print(f"\nIngesting {len(data_points)} data points...") + + for point in data_points: + db.add( + timestamp=point["timestamp"], + value=point["value"], + tags=point["tags"], + ) + + print(f"Ingested {db.size} points into database") + + # Query data + print("\nQuerying data...") + if data_points: + start_time = data_points[0]["timestamp"] + end_time = data_points[-1]["timestamp"] + + time_range = TimeRange(start_time=start_time, end_time=end_time) + results = db.query(time_range) + + print(f"Query returned {len(results)} points") + print("\nFirst 5 results:") + for i, result in enumerate(results[:5]): + print(f" {i + 1}. timestamp={result.timestamp}, value={result.value:.2f}") + + +def example_window_aggregation(): + """Example: Window-based aggregation""" + print("\n" + "=" * 60) + print("Example 2: Window Aggregation") + print("=" * 60) + + # Create TSDB instance + db = SageTSDB() + + # Generate and ingest data + data_points = generate_time_series_data(num_points=30, sensor_id="humidity_sensor") + print(f"\nIngesting {len(data_points)} data points for aggregation...") + + for point in data_points: + db.add( + timestamp=point["timestamp"], + value=point["value"], + tags=point["tags"], + ) + + # Query with windowing + if data_points: + start_time = data_points[0]["timestamp"] + end_time = data_points[-1]["timestamp"] + + print(f"\nQuerying with time range: {start_time} to {end_time}") + time_range = TimeRange(start_time=start_time, end_time=end_time) + results = db.query(time_range) + + # Calculate statistics + if results: + values: np.ndarray = np.array([r.value for r in results]) + print(f"\nStatistics from {len(values)} points:") + print(f" Mean: {np.mean(values):.2f}") + print(f" Std Dev: {np.std(values):.2f}") + print(f" Min: {np.min(values):.2f}") + print(f" Max: {np.max(values):.2f}") + + +def example_multi_sensor(): + """Example: Multi-sensor data""" + print("\n" + "=" * 60) + print("Example 3: Multi-Sensor Data") + print("=" * 60) + + # Create TSDB instance + db = SageTSDB() + + # Generate data from multiple sensors + sensors = ["temp_sensor", "humidity_sensor", "pressure_sensor"] + print(f"\nIngesting data from {len(sensors)} sensors...") + + base_time = int(datetime.now().timestamp() * 1000) + total_points = 0 + + for sensor_id in sensors: + data_points = generate_time_series_data(num_points=20, sensor_id=sensor_id) + for point in data_points: + db.add( + timestamp=point["timestamp"], + value=point["value"], + tags=point["tags"], + ) + total_points += 1 + + print(f"Ingested {total_points} points from {len(sensors)} sensors") + # Print final database size + print(f"Database size: {db.size}") + + # Query data by sensor + print("\nQuerying data by sensor:") + for sensor_id in sensors: + start_time = base_time + end_time = base_time + 30000 + time_range = TimeRange(start_time=start_time, end_time=end_time) + + # Note: Query with tags may need adjustment based on actual API + results = db.query(time_range) + print(f" {sensor_id}: {len(results)} points") + + +def example_database_statistics(): + """Example: Database statistics""" + print("\n" + "=" * 60) + print("Example 4: Database Statistics") + print("=" * 60) + + # Create TSDB instance + db = SageTSDB() + + # Ingest data + data_points = generate_time_series_data(num_points=100, sensor_id="stat_sensor") + print(f"\nIngesting {len(data_points)} data points...") + + for point in data_points: + db.add( + timestamp=point["timestamp"], + value=point["value"], + tags=point["tags"], + ) + + # Get statistics + print("\nDatabase Statistics:") + stats = db.get_stats() + for key, value in stats.items(): + print(f" {key}: {value}") + + print(f"\nTotal points in database: {db.size}") + + +if __name__ == "__main__": + print("\n" + "=" * 60) + print("SAGE TSDB Basic Examples") + print("=" * 60) + + try: + # Run examples + example_basic_ingestion_and_query() + example_window_aggregation() + example_multi_sensor() + example_database_statistics() + + print("\n" + "=" * 60) + print("All examples completed successfully!") + print("=" * 60) + + except Exception as e: + print(f"\nError running examples: {e}") + import traceback + + traceback.print_exc() diff --git a/tutorials/L4-middleware/sage_tsdb/stream_join_dag_example.py b/tutorials/L4-middleware/sage_tsdb/stream_join_dag_example.py new file mode 100644 index 0000000..a689f87 --- /dev/null +++ b/tutorials/L4-middleware/sage_tsdb/stream_join_dag_example.py @@ -0,0 +1,202 @@ +""" +Stream Join Example with SAGE TSDB + +This example demonstrates out-of-order stream join using SAGE TSDB: +1. Two data streams with potential out-of-order arrivals +2. Window-based join algorithms +3. Stream processing patterns +""" + +import random +from datetime import datetime +from typing import Any + +import numpy as np + +from sage.middleware.components.sage_tsdb import SageTSDB, TimeRange + + +def generate_stream_data( + stream_id: str, + num_points: int = 50, + disorder_probability: float = 0.3, + max_delay_ms: int = 3000, +) -> list[dict[str, Any]]: + """Generate time series stream with out-of-order data""" + data_points = [] + base_time = int(datetime.now().timestamp() * 1000) + + for i in range(num_points): + # Determine if this point should be out of order + if random.random() < disorder_probability: + # Add negative delay to simulate out-of-order arrival + delay = -random.randint(100, max_delay_ms) + else: + # In-order or slightly delayed + delay = random.randint(0, 500) + + timestamp = base_time + i * 1000 + delay + value = 100 + 20 * np.sin(i * 0.2) + np.random.randn() * 5 + + data_points.append( + { + "timestamp": timestamp, + "value": value, + "stream_id": stream_id, + "sequence": i, + } + ) + + return data_points + + +def join_streams( + left_stream: list[dict[str, Any]], + right_stream: list[dict[str, Any]], + window_size: int = 5000, +) -> list[tuple[dict[str, Any], dict[str, Any]]]: + """Join two streams based on time windows""" + joined_pairs = [] + + # Sort both streams by timestamp + left_sorted = sorted(left_stream, key=lambda x: x["timestamp"]) + right_sorted = sorted(right_stream, key=lambda x: x["timestamp"]) + + # Simple window-based join + for left_data in left_sorted: + for right_data in right_sorted: + time_diff = abs(left_data["timestamp"] - right_data["timestamp"]) + + # If within window, create pair + if time_diff <= window_size: + joined_pairs.append((left_data, right_data)) + + return joined_pairs + + +def example_stream_join_with_time_range(): + """Example: Basic stream join with time range""" + print("\n" + "=" * 60) + print("Stream Join with Time Range Example") + print("=" * 60) + + # Generate two streams + print("\nGenerating streams...") + left_stream = generate_stream_data(stream_id="left", num_points=30, disorder_probability=0.3) + right_stream = generate_stream_data(stream_id="right", num_points=30, disorder_probability=0.3) + + print(f"Left stream: {len(left_stream)} points") + print(f"Right stream: {len(right_stream)} points") + + # Join streams + print("\nPerforming join...") + joined_pairs = join_streams(left_stream, right_stream, window_size=5000) + + print(f"\nJoined {len(joined_pairs)} pairs") + + # Show results + print("\nJoin results (first 5 pairs):") + for i, (left, right) in enumerate(joined_pairs[:5]): + time_diff = abs(left["timestamp"] - right["timestamp"]) + print(f"\nPair {i + 1}:") + print(f" Left: seq={left['sequence']}, ts={left['timestamp']}, value={left['value']:.2f}") + print( + f" Right: seq={right['sequence']}, ts={right['timestamp']}, value={right['value']:.2f}" + ) + print(f" Time diff: {time_diff}ms") + + # Statistics + if joined_pairs: + time_diffs = [abs(left["timestamp"] - right["timestamp"]) for left, right in joined_pairs] + print("\nJoin Statistics:") + print(f" Total pairs: {len(joined_pairs)}") + print(f" Avg time diff: {np.mean(time_diffs):.2f}ms") + print(f" Max time diff: {np.max(time_diffs):.2f}ms") + print(f" Min time diff: {np.min(time_diffs):.2f}ms") + + +def example_stream_ingestion_and_join(): + """Example: Ingest streams and query joins""" + print("\n" + "=" * 60) + print("Stream Ingestion and Join Example") + print("=" * 60) + + # Create TSDB instances for left and right streams + db_left = SageTSDB() + db_right = SageTSDB() + + # Generate streams + print("\nGenerating streams...") + left_stream = generate_stream_data(stream_id="left", num_points=25, disorder_probability=0.2) + right_stream = generate_stream_data(stream_id="right", num_points=25, disorder_probability=0.2) + + # Ingest into databases + print("\nIngesting left stream...") + for data in left_stream: + db_left.add(timestamp=data["timestamp"], value=data["value"], tags={"stream": "left"}) + + print("Ingesting right stream...") + for data in right_stream: + db_right.add(timestamp=data["timestamp"], value=data["value"], tags={"stream": "right"}) + + print(f"Left DB size: {db_left.size}") + print(f"Right DB size: {db_right.size}") + + # Query joined data + print("\nQuerying joined data from time window...") + base_time = min(data["timestamp"] for data in left_stream + right_stream) + end_time = max(data["timestamp"] for data in left_stream + right_stream) + + time_range = TimeRange(start_time=base_time, end_time=end_time) + + left_results = db_left.query(time_range) + right_results = db_right.query(time_range) + + print(f"Left query results: {len(left_results)} points") + print(f"Right query results: {len(right_results)} points") + + # Manual join + print("\nPerforming join on queried data...") + joined = 0 + for left_result in left_results: + for right_result in right_results: + time_diff = abs(left_result.timestamp - right_result.timestamp) + if time_diff <= 5000: # 5 second window + joined += 1 + + print(f"Joined pairs within 5-second window: {joined}") + + # Database stats + print("\nDatabase Statistics:") + print("\nLeft Stream DB:") + for key, value in db_left.get_stats().items(): + print(f" {key}: {value}") + + print("\nRight Stream DB:") + for key, value in db_right.get_stats().items(): + print(f" {key}: {value}") + + +if __name__ == "__main__": + print("\n" + "=" * 60) + print("SAGE TSDB Stream Join Examples") + print("=" * 60) + + try: + # Run stream join with time range + example_stream_join_with_time_range() + + print("\n" + "-" * 60 + "\n") + + # Run stream ingestion and join + example_stream_ingestion_and_join() + + print("\n" + "=" * 60) + print("Examples completed successfully!") + print("=" * 60) + + except Exception as e: + print(f"\nError running examples: {e}") + import traceback + + traceback.print_exc() diff --git a/tutorials/L5-apps/README.md b/tutorials/L5-apps/README.md new file mode 100644 index 0000000..ccd6649 --- /dev/null +++ b/tutorials/L5-apps/README.md @@ -0,0 +1,34 @@ +# L5: Apps - 应用层示例 + +> 对应 SAGE 包：`sage-apps`, `sage-benchmark` + +## 📖 层级说明 + +**Apps** 层提供完整的端到端应用解决方案，整合所有底层能力。 + +## 📚 目录结构 + +``` +L5-apps/ +└── complete_solutions/ # 完整解决方案 +``` + +## 🎯 学习路径 + +### 完整解决方案 (`complete_solutions/`) + +端到端的应用系统（待添加示例） + +## 🎯 学习目标 + +完成本层示例后，你将掌握： + +1. 如何设计完整的应用架构 +1. 如何整合 SAGE 的各层能力 +1. 生产级应用的最佳实践 + +## ⏭️ 扩展学习 + +继续探索： + +- **L6-interface/** - 用户接口（CLI、Web UI） diff --git a/tutorials/L6-interface/README.md b/tutorials/L6-interface/README.md new file mode 100644 index 0000000..3e00806 --- /dev/null +++ b/tutorials/L6-interface/README.md @@ -0,0 +1,174 @@ +# L6: Interface Layer - 接口层 + +## 📋 概述 + +**对应包**：`sage-studio`, `sage-tools`, `sage-cli` + +L6 是 SAGE 架构的**顶层接口层**，提供用户直接交互的多种工具： + +- 🎨 **Web UI** (sage-studio)：可视化管道编辑器 +- 🛠️ **开发工具** (sage-tools)：测试、代码质量、包管理等 +- 💻 **命令行接口** (sage-cli)：提供 `sage` 命令行工具 + +## � L6 包详解 + +### 1️⃣ sage-studio - Web UI + +**功能**： + +- 可视化管道设计器 +- 拖拽式操作符连接 +- 实时预览和调试 +- 管道部署和监控 + +**适用场景**： + +- 无代码/低代码开发 +- 快速原型验证 +- 团队协作开发 +- 生产环境监控 + +### 2️⃣ sage-tools - 开发工具集 + +**功能**： + +- `fix-code-quality.sh`：代码质量检查和自动修复 +- `pytest`：单元测试框架配置 +- `ruff`：Python 代码检查 +- `mypy`：类型检查 +- 包管理和依赖分析 + +**适用场景**： + +- 开发环境配置 +- CI/CD 流程集成 +- 代码质量保证 +- 包维护和发布 + +### 3️⃣ sage-cli - 命令行接口 + +**功能**： + +- `sage` 命令：统一的命令行入口 +- 项目初始化和脚手架 +- 管道运行和调试 +- 配置管理 + +**适用场景**： + +- 快速启动项目 +- 脚本化自动化 +- 远程服务器部署 +- 开发者工作流 + +## 🎯 学习目标 + +- 理解 SAGE 的三种用户界面（Web、CLI、工具集） +- 掌握可视化管道编辑 +- 使用命令行快速开发和部署 +- 使用开发工具提升代码质量 + +## 📚 示例列表 + +> ⚠️ **注意**：L6 示例即将添加 + +### 规划中的示例 + +#### sage-studio 示例 + +- [ ] `web_pipeline_editor.md` - 使用 Web UI 创建管道 +- [ ] `dashboard_monitoring.md` - 生产环境监控面板 +- [ ] `collaborative_development.md` - 团队协作开发 + +#### sage-cli 示例 + +- [ ] `cli_quickstart.py` - 使用 `sage` 命令快速开始 +- [ ] `cli_project_init.sh` - 项目初始化 +- [ ] `cli_pipeline_run.sh` - 运行和调试管道 + +#### sage-tools 示例 + +- [ ] `setup_dev_environment.sh` - 配置开发环境 +- [ ] `run_code_quality_checks.sh` - 运行代码检查 +- [ ] `package_management.md` - 包管理最佳实践 + +## � 依赖关系 + +``` +L6 (Interface: studio + tools + cli) + ↓ 可以使用 +L5 (Apps) + L4 (Middleware) + L3 (Kernel/Libs) + L2 (Platform) + L1 (Common) +``` + +L6 是最顶层，可以使用所有下层的功能。 + +## 🚀 快速开始 + +### 使用 sage-cli + +```bash +# 安装 sage-cli（假设已安装 SAGE） +# pip install sage-cli + +# 初始化新项目 +sage init my-rag-app --template rag + +# 运行管道 +sage run pipeline.py + +# 查看帮助 +sage --help +``` + +### 使用 sage-studio + +```bash +# 启动 Web UI（假设已安装） +# sage-studio start + +# 访问 http://localhost:8080 +# 在浏览器中打开管道编辑器 +``` + +### 使用 sage-tools + +```bash +# 运行代码质量检查 +cd /path/to/sage +./tools/fix-code-quality.sh + +# 运行测试 +pytest + +# 类型检查 +./tools/mypy-wrapper.sh +``` + +## 💡 使用建议 + +### 何时使用 sage-studio + +- ✅ 快速原型验证 +- ✅ 团队协作 +- ✅ 非技术用户参与 +- ✅ 可视化调试 + +### 何时使用 sage-cli + +- ✅ 快速启动项目 +- ✅ 脚本自动化 +- ✅ CI/CD 集成 +- ✅ 远程部署 + +### 何时使用 sage-tools + +- ✅ 开发环境配置 +- ✅ 代码质量检查 +- ✅ 包维护 +- ✅ 贡献代码 + +## 📚 相关文档 + +- [SAGE 包架构](../../../../docs-public/docs_src/dev-notes/package-architecture.md) +- [开发者指南](../../../../DEVELOPER.md) +- [贡献指南](../../../../CONTRIBUTING.md) diff --git a/tutorials/QUICK_START.md b/tutorials/QUICK_START.md new file mode 100644 index 0000000..e84bf8f --- /dev/null +++ b/tutorials/QUICK_START.md @@ -0,0 +1,320 @@ +# 🚀 SAGE Tutorials - Quick Start + +欢迎来到 SAGE Tutorials！本指南将帮助你在 **5 分钟内**开始使用 SAGE。 + +## ⚡ 超快入门（2 分钟） + +### 1. 运行第一个示例 + +```bash +cd /path/to/SAGE/examples/tutorials +python hello_world.py +``` + +恭喜！你已经运行了第一个 SAGE 程序！ + +### 2. 理解架构 + +SAGE 采用 **6 层架构**，从底层到应用层： + +``` +L1: Common → 基础工具 (配置、日志) +L2: Kernel → 核心引擎 (流处理、批处理) +L3: Middleware → 数据服务 (向量数据库、时序数据库) +L4: Libs → 应用库 (RAG、Agents、LLM) +L5: Platform → 平台服务 (调度、部署) +L6: Apps → 完整应用 (端到端解决方案) +``` + +### 3. 选择学习路径 + +根据你的角色选择合适的路径： + +| 角色 | 推荐路径 | 时间 | +| ------------- | ------------------ | -------- | +| 🔰 初学者 | L1 → L2 基础 | 1-2 小时 | +| 🚀 应用开发者 | L1 → L2 → L3 → L4 | 4-6 小时 | +| 🧠 平台开发者 | 全栈学习 L1-L5 | 1-2 天 | +| 🏗️ 架构师 | 全部 L1-L6 + tools | 2-3 天 | + +______________________________________________________________________ + +## 📚 分层学习指南 + +### L1: Common - 基础层（15 分钟） + +**目标**：理解 SAGE 的基础概念 + +```bash +cd L1-common +python hello_world.py # 最基础的示例 +``` + +**核心概念**： + +- SAGE 环境配置 +- 日志系统 +- 基本术语 + +👉 **下一步**：进入 L2-kernel 学习核心 API + +______________________________________________________________________ + +### L2: Kernel - 核心层（2-3 小时） + +**目标**：掌握流处理和批处理 + +#### 2.1 批处理基础（30 分钟） + +```bash +cd L2-kernel/batch +python hello_local_batch.py # 本地批处理 +python hello_remote_batch.py # 远程批处理 +``` + +#### 2.2 流处理基础（30 分钟） + +```bash +cd L2-kernel/stream +python hello_streaming_world.py # 基础流处理 +python hello_onebyone_world.py # 单条流处理 +``` + +#### 2.3 操作符系统（1 小时） + +```bash +cd L2-kernel/operators +python hello_comap_world.py # CoMap 操作符 +python hello_filter_world.py # Filter 过滤 +python hello_flatmap_world.py # FlatMap 展开 +python hello_join_world.py # Join 连接 +``` + +#### 2.4 高级特性（选修） + +```bash +cd L2-kernel/advanced +python hello_future_world.py # Future 异步 +``` + +**核心概念**： + +- DataStream API +- Operator 系统 +- 批处理 vs 流处理 +- Pipeline 构建 + +👉 **下一步**：进入 L3-middleware 学习数据服务 + +______________________________________________________________________ + +### L3: Middleware - 中间件层（1-2 小时） + +**目标**：使用数据服务和中间件 + +#### 3.1 服务入门（15 分钟） + +```bash +cd L3-middleware +python hello_service_world.py # 理解服务模型 +``` + +#### 3.2 Memory Service（30 分钟） + +```bash +cd L3-middleware/memory_service +python rag_memory_service.py # RAG 内存服务 +``` + +#### 3.3 数据库服务（30 分钟） + +```bash +cd L3-middleware/sage_db +python workflow_demo.py # 向量数据库 + +cd L3-middleware/sage_tsdb +python basic_dag_example.py # 时序数据库 +``` + +**核心概念**： + +- Service API +- 向量数据库 +- 时序数据处理 +- 内存管理 + +👉 **下一步**：进入 L4-libs 构建应用 + +______________________________________________________________________ + +### L4: Libs - 应用库层（2-4 小时） + +**目标**：构建实际应用 + +#### 4.1 RAG 应用（1-2 小时） + +```bash +cd L4-libs/rag +python simple_rag.py # 简单 RAG +python usage_1_direct_library.py # 直接使用库 +python usage_4_complete_rag.py # 完整 RAG 系统 +``` + +#### 4.2 Agent 应用（30 分钟） + +```bash +cd L4-libs/agents +python basic_agent.py # 基础智能体 +python workflow_demo.py # 工作流 +``` + +#### 4.3 Embedding 应用（30 分钟） + +```bash +cd L4-libs/embeddings +python embedding_demo.py # 嵌入演示 +python cross_modal_search.py # 跨模态搜索 +``` + +#### 4.4 LLM 应用（30 分钟） + +```bash +cd L4-libs/llm +python pipeline_builder_llm_demo.py # LLM 管道 +python templates_to_llm_demo.py # 模板演示 +``` + +**核心概念**： + +- RAG 系统设计 +- Agent 架构 +- 向量嵌入 +- LLM 集成 + +👉 **下一步**：进入 L5-platform 学习平台服务（高级） + +______________________________________________________________________ + +### L5: Platform - 平台层（1-2 小时，高级） + +**目标**：理解平台级服务 + +```bash +cd L5-platform/scheduler +python scheduler_comparison.py # 调度器对比 +python remote_env.py # 远程环境 +``` + +**核心概念**： + +- 任务调度 +- 分布式执行 +- 资源管理 + +👉 **下一步**：探索 L6-apps 完整应用（即将推出） + +______________________________________________________________________ + +## 🎯 常见学习路径 + +### 路径 1：RAG 开发者（最热门） + +``` +1. L1-common/hello_world.py (5 分钟) +2. L2-kernel/batch/hello_local_batch.py (15 分钟) +3. L3-middleware/memory_service/ (30 分钟) +4. L4-libs/rag/simple_rag.py (30 分钟) +5. L4-libs/rag/usage_4_complete_rag.py (1 小时) +``` + +**总时间**：约 2.5 小时 + +### 路径 2：Agent 开发者 + +``` +1. L1-common/hello_world.py (5 分钟) +2. L2-kernel/stream/ (30 分钟) +3. L4-libs/agents/basic_agent.py (30 分钟) +4. L4-libs/agents/workflow_demo.py (1 小时) +``` + +**总时间**：约 2 小时 + +### 路径 3：平台工程师 + +``` +1. 完整学习 L1-L2 (3 小时) +2. L3-middleware/ 全部 (2 小时) +3. L5-platform/ 全部 (2 小时) +``` + +**总时间**：约 7 小时 + +______________________________________________________________________ + +## 📖 文档和帮助 + +### 快速查询 + +- **快速参考**：`docs/QUICK_REFERENCE.md` +- **故障排除**：`docs/TROUBLESHOOTING.md` +- **学习路径**：`docs/LEARNING_PATH.md`（即将推出） + +### 层级文档 + +每个层级目录都有详细的 README： + +- `L1-common/README.md` +- `L2-kernel/README.md` +- `L3-middleware/README.md` +- `L4-libs/README.md` +- `L5-platform/README.md` +- `L6-apps/README.md` + +### 遇到问题？ + +1. 查看 `docs/TROUBLESHOOTING.md` +1. 检查每个示例的注释 +1. 阅读对应层级的 README +1. 查看项目主 README + +______________________________________________________________________ + +## 🎓 学习建议 + +### ✅ 推荐做法 + +1. **按顺序学习**：从 L1 开始，逐层深入 +1. **动手实践**：运行每个示例，修改参数 +1. **阅读注释**：示例中有详细的说明 +1. **理解概念**：不要只运行，要理解原理 + +### ❌ 避免 + +1. **跳过基础**：L1-L2 是必须的 +1. **只看不做**：一定要运行代码 +1. **贪多嚼不烂**：一次专注一个主题 + +______________________________________________________________________ + +## 🚀 下一步行动 + +根据你的兴趣选择： + +- **我想构建 RAG 系统** → 按"路径 1"学习 +- **我想开发 Agent** → 按"路径 2"学习 +- **我想深入理解架构** → 从 L1 开始系统学习 +- **我只想快速尝试** → 运行 `hello_world.py` 和几个感兴趣的示例 + +______________________________________________________________________ + +## 💡 小贴士 + +- 每个示例都可以独立运行 +- 示例之间有依赖关系，建议按推荐顺序学习 +- 遇到错误先查看 `TROUBLESHOOTING.md` +- 配置文件在 `config/` 目录 + +______________________________________________________________________ + +**开始你的 SAGE 之旅吧！🎉** diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 0000000..a30a9c4 --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,348 @@ +# 🚀 SAGE Tutorials + +欢迎来到 SAGE Tutorials！这里包含了按照 SAGE 架构分层组织的完整示例和文档。 + +> **从基础到应用：循序渐进地掌握 SAGE 框架** + +## ⚡ 5 分钟快速开始 + +```bash +# 1. 运行第一个示例 +python hello_world.py + +# 2. 查看快速入门指南 +cat QUICK_START.md +``` + +## 📐 SAGE 6 层架构概览 + +SAGE 采用 **6 层分层架构**，从底层基础设施到顶层应用： + +``` +┌─────────────────────────────────────────────┐ +│ L6: Interface (接口层) │ +│ sage-studio, sage-tools, sage-cli │ +│ Web UI + CLI 工具 + 命令行接口 │ +├─────────────────────────────────────────────┤ +│ L5: Apps (应用层) │ +│ sage-apps, sage-benchmark │ +│ 完整应用 + 性能测试 │ +├─────────────────────────────────────────────┤ +│ L4: Middleware (中间件层) │ +│ sage-middleware │ +│ 领域算子 + 中间件组件 │ +├─────────────────────────────────────────────┤ +│ L3: Core (核心层) │ +│ sage-kernel + sage-libs │ +│ 执行引擎 + 算法库 │ +├─────────────────────────────────────────────┤ +│ L2: Platform (平台层) │ +│ sage-platform │ +│ 队列 + 存储 + 服务抽象 │ +├─────────────────────────────────────────────┤ +│ L1: Foundation (基础层) │ +│ sage-common │ +│ 配置 + 日志 + 工具 │ +└─────────────────────────────────────────────┘ +``` + +**核心原则**： + +- ✅ **单向依赖**：只能向下依赖（L6→L5→L4→L3→L2→L1） +- ❌ **禁止反向**：禁止向上或循环依赖 +- 📚 **详细说明**：[SAGE 包架构文档](../../docs-public/docs_src/dev-notes/package-architecture.md) + +## 📚 Tutorial 目录结构 + +``` +tutorials/ +│ +├── hello_world.py # 最简入门（快速开始） +├── QUICK_START.md # 5 分钟快速指南 +├── README.md # 本文档 +│ +├── L1-common/ # 基础层：配置、日志、工具 +├── L2-platform/ # 平台层：队列、存储、调度 +├── L3-kernel/ # 核心层：执行引擎、流处理 +├── L3-libs/ # 核心层：RAG、Agents、算法库 +├── L4-middleware/ # 中间件层：领域算子、数据服务 +├── L5-apps/ # 应用层：完整应用 +├── L6-interface/ # 接口层：CLI、Web UI +│ +├── config/ # 配置文件示例 +└── docs/ # 文档（快速参考、故障排除） +``` + +## �� 学习路径 + +### 🔰 初学者路径（2-3 小时） + +适合第一次接触 SAGE 的用户 + +```bash +# L1: 基础概念 +cd L1-common && python hello_world.py + +# L2: 平台服务 +cd L2-platform/scheduler && python scheduler_comparison.py + +# L3: 核心引擎 +cd L3-kernel/batch && python hello_local_batch.py +cd L3-kernel/stream && python hello_streaming_world.py +cd L3-kernel/operators && python hello_comap_world.py +``` + +**学习目标**：理解 SAGE 基础概念、流处理模型、操作符系统 + +### 🚀 应用开发者路径（4-6 小时） + +适合想要构建应用的开发者 + +```bash +# 完成初学者路径后 + +# L3: 算法库 +cd L3-libs/rag && python simple_rag.py +cd L3-libs/agents && python basic_agent.py + +# L4: 中间件服务 +cd L4-middleware && python hello_service_world.py +cd L4-middleware/memory_service && python rag_memory_service.py + +# L4: 数据服务 +cd L4-middleware/sage_db && python workflow_demo.py +cd L4-middleware/sage_tsdb && python basic_dag_example.py +``` + +**学习目标**：掌握 RAG 系统、Agent 开发、数据服务使用 + +### 🧠 高级开发者路径（1-2 天） + +适合平台开发者和架构师 + +```bash +# 完成应用开发者路径后 + +# L3: 高级特性 +cd L3-kernel/advanced && python hello_future_world.py +cd L3-kernel/advanced/fault_tolerance && python fault_tolerance.py + +# L3: 完整 RAG 系统 +cd L3-libs/rag && python usage_4_complete_rag.py + +# L5: 应用集成 +# （待添加完整应用示例） +``` + +**学习目标**：深入理解容错机制、异步处理、生产级系统设计 + +## �� 各层级详细说明 + +### L1: Common - 基础层 + +**对应包**：`sage-common` + +**内容**： + +- `hello_world.py` - 最基础的 SAGE 程序 +- 配置管理示例 +- 日志系统示例 + +**Python 文件数**：1 + +[查看详细文档 →](L1-common/README.md) + +______________________________________________________________________ + +### L2: Platform - 平台服务层 + +**对应包**：`sage-platform` + +**内容**： + +- `scheduler/` - 调度系统示例 +- 队列抽象示例（待添加） +- 存储后端示例（待添加） + +**Python 文件数**：2 + +[查看详细文档 →](L2-platform/README.md) + +______________________________________________________________________ + +### L3: Kernel - 核心引擎层 + +**对应包**：`sage-kernel` + +**内容**： + +- `batch/` - 批处理示例（3 个） +- `stream/` - 流处理示例（3 个） +- `operators/` - 操作符示例（5 个） +- `functions/` - 函数示例（3 个） +- `advanced/` - 高级特性（容错、Future、Pipeline-as-Service 等，17 个） + +**Python 文件数**：31 + +[查看详细文档 →](L3-kernel/README.md) + +______________________________________________________________________ + +### L3: Libs - 算法库层 + +**对应包**：`sage-libs` + +**内容**： + +- `rag/` - RAG 应用示例（7 个） +- `agents/` - 智能体示例（5 个） +- `embeddings/` - 嵌入示例（4 个） +- `llm/` - LLM 集成示例（2 个） +- `unlearning/` - 机器遗忘示例（5 个） + +**Python 文件数**：23 + +[查看详细文档 →](L3-libs/README.md) + +______________________________________________________________________ + +### L4: Middleware - 中间件层 + +**对应包**：`sage-middleware` + +**内容**： + +- `hello_service_world.py` - 服务入门 +- `memory_service/` - 内存管理（3 个） +- `sage_db/` - 向量数据库（4 个） +- `sage_flow/` - 流数据服务（3 个） +- `sage_tsdb/` - 时序数据库（3 个） + +**Python 文件数**：13 + +[查看详细文档 →](L4-middleware/README.md) + +______________________________________________________________________ + +### L5: Apps - 应用层 + +**对应包**：`sage-apps`, `sage-benchmark` + +**内容**： + +- `complete_solutions/` - 完整应用（待添加） + +**Python 文件数**：0 (待添加) + +[查看详细文档 →](L5-apps/README.md) + +______________________________________________________________________ + +### L6: Interface - 接口层 + +**对应包**：`sage-studio` (Web UI), `sage-tools` (开发工具), `sage-cli` (命令行) + +**内容**： + +- CLI 工具示例（待添加） +- Web UI 示例（待添加） +- 命令行工具示例（待添加） + +**Python 文件数**：0 (待添加) + +**说明**： + +- **sage-studio**：可视化 Web UI，提供管道编辑器 +- **sage-tools**：开发工具集（测试、质量检查、包管理等） +- **sage-cli**：命令行接口，提供 `sage` 命令 + +[查看详细文档 →](L6-interface/README.md) + +______________________________________________________________________ + +## 📊 示例统计 + +| 层级 | 对应包 | Python 文件 | 主要内容 | +| -------- | --------------------------------- | ----------- | ------------ | +| L1 | sage-common | 1 | 基础工具 | +| L2 | sage-platform | 2 | 平台服务 | +| L3 | sage-kernel | 31 | 流处理引擎 | +| L3 | sage-libs | 23 | 算法库 | +| L4 | sage-middleware | 13 | 数据服务 | +| L5 | sage-apps, sage-benchmark | 0 | 待添加 | +| L6 | sage-studio, sage-tools, sage-cli | 0 | 待添加 | +| **总计** | - | **70** | **6 层架构** | + +## 📖 补充文档 + +- [**QUICK_START.md**](QUICK_START.md) - 5 分钟快速入门 +- [**docs/QUICK_REFERENCE.md**](docs/QUICK_REFERENCE.md) - 快速参考卡 +- [**docs/TROUBLESHOOTING.md**](docs/TROUBLESHOOTING.md) - 故障排除指南 + +## 🔍 如何选择示例 + +### 我想学习... + +- **基础概念** → 从 `hello_world.py` 开始 +- **流处理** → `L3-kernel/stream/` +- **批处理** → `L3-kernel/batch/` +- **RAG 系统** → `L3-libs/rag/` +- **智能体** → `L3-libs/agents/` +- **数据服务** → `L4-middleware/` +- **容错机制** → `L3-kernel/advanced/fault_tolerance/` + +### 我想构建... + +- **简单脚本** → `L3-libs/rag/usage_1_direct_library.py` +- **数据管道** → `L3-kernel/` 中的示例 +- **RAG 应用** → `L3-libs/rag/usage_4_complete_rag.py` +- **Agent 系统** → `L3-libs/agents/workflow_demo.py` +- **完整应用** → 学习所有层级 + +## 🎓 学习建议 + +### ✅ 推荐做法 + +1. **按层级学习**：从 L1 开始，逐层深入 +1. **理解依赖**：了解为什么上层可以用下层，反之不行 +1. **动手实践**：运行每个示例，修改参数观察效果 +1. **阅读代码**：示例中有详细注释 +1. **参考文档**：遇到问题查看各层的 README + +### ❌ 避免 + +1. **跳过基础**：L1-L3 是必须理解的 +1. **只看不做**：一定要运行代码 +1. **忽略架构**：理解架构有助于设计更好的系统 +1. **违反依赖**：不要在低层导入高层代码 + +## 🆘 遇到问题？ + +1. 查看 [TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) +1. 检查示例的注释和 docstring +1. 阅读对应层级的 README +1. 查看 [SAGE 包架构文档](../../docs-public/docs_src/dev-notes/package-architecture.md) +1. 提交 Issue 到 GitHub + +## 🤝 贡献 + +欢迎添加新示例或改进现有示例！请确保： + +1. 示例放在正确的层级目录 +1. 遵循依赖规则（只向下依赖） +1. 添加清晰的注释和文档 +1. 更新对应的 README + +## 📜 变更历史 + +- **2025-10-29**: 按照 SAGE 6 层架构重组目录（L1-L6） + - 完整映射 9 个 SAGE 包到 6 个层级 + - 添加 sage-cli 到 L6 接口层说明 + - 迁移 service 目录内容到对应层级 + - 创建完整的学习路径和文档 + +______________________________________________________________________ + +**开始探索 SAGE 吧！🎉** + +有任何问题或建议，欢迎提 Issue 或 PR！ diff --git a/tutorials/__init__.py b/tutorials/__init__.py new file mode 100644 index 0000000..962ee38 --- /dev/null +++ b/tutorials/__init__.py @@ -0,0 +1,9 @@ +""" +SAGE Tutorials - 基础教程和入门示例 + +这个模块包含了 SAGE 框架的基础教程和入门示例。 +""" + +from . import hello_world + +__all__ = ["hello_world"] diff --git a/tutorials/benchmark_control_plane_demo.py b/tutorials/benchmark_control_plane_demo.py new file mode 100644 index 0000000..a9fe02b --- /dev/null +++ b/tutorials/benchmark_control_plane_demo.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the SAGE project +""" +Control Plane Benchmark Demo +============================= + +This script demonstrates how to use the Control Plane Benchmark framework +for evaluating scheduling policies. + +Usage: + python examples/tutorials/benchmark_control_plane_demo.py + +Note: This demo uses Mock mode (no actual Control Plane needed). +""" + +from __future__ import annotations + +import asyncio +import tempfile +from pathlib import Path + + +def demo_1_llm_benchmark_config(): + """Demo 1: Create and validate LLM Benchmark configuration.""" + print("\n" + "=" * 60) + print("Demo 1: LLM Benchmark Configuration") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane import ( + ArrivalPattern, + LLMBenchmarkConfig, + LLMSLOConfig, + SchedulingPolicy, + ) + + # Create configuration + config = LLMBenchmarkConfig( + control_plane_url="http://localhost:8080", + num_requests=100, + request_rate=10.0, + warmup_requests=5, + timeout_seconds=30.0, + arrival_pattern=ArrivalPattern.POISSON, + policies=[SchedulingPolicy.FIFO, SchedulingPolicy.PRIORITY], + model_distribution={ + "Qwen/Qwen2.5-7B-Instruct": 0.7, + "meta-llama/Llama-3-8B": 0.3, + }, + priority_distribution={"HIGH": 0.2, "NORMAL": 0.6, "LOW": 0.2}, + slo_config=LLMSLOConfig( + high_priority_deadline_ms=500, + normal_priority_deadline_ms=1000, + low_priority_deadline_ms=2000, + ), + ) + + # Validate + errors = config.validate() + if errors: + print(f"Validation errors: {errors}") + else: + print("Configuration is valid.") + + # Show configuration + print("\nConfiguration summary:") + print(f" - Control Plane: {config.control_plane_url}") + print(f" - Requests: {config.num_requests}") + print(f" - Rate: {config.request_rate} req/s") + print(f" - Arrival: {config.arrival_pattern.value}") + print(f" - Policies: {[p.value if hasattr(p, 'value') else str(p) for p in config.policies]}") # type: ignore + print(f" - Models: {config.model_distribution}") + + return config + + +def demo_2_hybrid_benchmark_config(): + """Demo 2: Create Hybrid (LLM + Embedding) Benchmark configuration.""" + print("\n" + "=" * 60) + print("Demo 2: Hybrid Benchmark Configuration") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane.hybrid_scheduler import ( + HybridBenchmarkConfig, + HybridSLOConfig, + ) + + # Create hybrid configuration + config = HybridBenchmarkConfig( + control_plane_url="http://localhost:8080", + num_requests=100, + request_rate=20.0, + warmup_requests=5, + # Hybrid-specific settings + llm_ratio=0.7, # 70% LLM, 30% Embedding + embedding_ratio=0.3, + llm_model_distribution={"Qwen/Qwen2.5-7B-Instruct": 1.0}, + embedding_model="BAAI/bge-m3", + hybrid_slo_config=HybridSLOConfig( + high_priority_deadline_ms=500, + normal_priority_deadline_ms=1000, + low_priority_deadline_ms=2000, + embedding_high_priority_deadline_ms=100, + embedding_normal_priority_deadline_ms=200, + embedding_low_priority_deadline_ms=500, + ), + ) + + # Validate + errors = config.validate() + if errors: + print(f"Validation errors: {errors}") + else: + print("Hybrid configuration is valid.") + + print("\nHybrid configuration summary:") + print(f" - LLM Ratio: {config.llm_ratio * 100:.0f}%") + print(f" - Embedding Ratio: {config.embedding_ratio * 100:.0f}%") + print(f" - LLM Models: {config.llm_model_distribution}") + print(f" - Embedding Model: {config.embedding_model}") + print(f" - Embedding Batch Sizes: {config.embedding_batch_sizes}") + + return config + + +def demo_3_workload_generation(): + """Demo 3: Generate workloads for benchmarking.""" + print("\n" + "=" * 60) + print("Demo 3: Workload Generation") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane import ( + LLMBenchmarkConfig, + LLMWorkloadGenerator, + ) + from sage.benchmark.benchmark_control_plane.hybrid_scheduler import ( + HybridBenchmarkConfig, + HybridWorkloadGenerator, + ) + + # LLM Workload + print("\n[LLM Workload]") + llm_config = LLMBenchmarkConfig(num_requests=10, request_rate=5.0) + llm_generator = LLMWorkloadGenerator(llm_config) + llm_requests = llm_generator.generate() + + print(f" Generated {len(llm_requests)} LLM requests:") + for i, req in enumerate(llm_requests[:3]): + print( + f" {i + 1}. model={req.model_name}, priority={req.priority}, time={req.scheduled_arrival_time:.2f}s" + ) + if len(llm_requests) > 3: + print(f" ... and {len(llm_requests) - 3} more") + + # Hybrid Workload + print("\n[Hybrid Workload]") + hybrid_config = HybridBenchmarkConfig( + num_requests=10, request_rate=5.0, llm_ratio=0.6, embedding_ratio=0.4 + ) + hybrid_generator = HybridWorkloadGenerator(hybrid_config) + hybrid_requests = hybrid_generator.generate() + + llm_count = sum(1 for r in hybrid_requests if r.request_type.value.startswith("llm")) + embed_count = len(hybrid_requests) - llm_count + + print(f" Generated {len(hybrid_requests)} hybrid requests:") + print(f" - LLM requests: {llm_count} ({llm_count / len(hybrid_requests) * 100:.0f}%)") + print( + f" - Embedding requests: {embed_count} ({embed_count / len(hybrid_requests) * 100:.0f}%)" + ) + + for i, req in enumerate(hybrid_requests[:5]): + print(f" {i + 1}. type={req.request_type.value}, time={req.scheduled_arrival_time:.2f}s") + + +def demo_4_strategy_adapter(): + """Demo 4: List available scheduling strategies.""" + print("\n" + "=" * 60) + print("Demo 4: Available Scheduling Strategies") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane import StrategyAdapter + + print("\nAvailable strategies:") + strategy_names = StrategyAdapter.list_strategies() + for name in strategy_names: + info = StrategyAdapter.get_strategy_info(name) + if info: + status = "available" if info.cls is not None else "not installed" + hybrid = " (hybrid)" if info.is_hybrid else "" + print(f" - {info.name}: {info.description}{hybrid} [{status}]") + else: + print(f" - {name}: (info not available)") + + +def demo_5_gpu_monitor(): + """Demo 5: GPU monitoring (auto-detect backend).""" + print("\n" + "=" * 60) + print("Demo 5: GPU Monitoring") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane import GPUMonitor + + # Auto-detect backend (will use mock if no GPU) + monitor = GPUMonitor() + + print(f"\nGPU Monitor backend: {monitor.backend.value}") + print("Starting monitoring...") + monitor.start_monitoring(interval_seconds=0.5) + + import time + + time.sleep(1.5) # Collect some samples + + monitor.stop_monitoring() + + summary = monitor.get_summary() + print("\nGPU Metrics Summary:") + print(f" - Samples: {summary.samples}") + print(f" - Duration: {summary.duration_seconds:.1f}s") + print(f" - Avg Utilization: {summary.utilization_avg:.1f}%") + print(f" - Max Utilization: {summary.utilization_max:.1f}%") + print(f" - Avg Memory Used: {summary.memory_used_avg_mb:.0f} MB") + print(f" - Max Memory Used: {summary.memory_used_max_mb:.0f} MB") + + +def demo_6_metrics_collection(): + """Demo 6: Metrics collection and aggregation.""" + print("\n" + "=" * 60) + print("Demo 6: Metrics Collection") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane.llm_scheduler.client import ( + LLMRequestResult, + ) + from sage.benchmark.benchmark_control_plane.llm_scheduler.metrics import ( + LLMMetricsCollector, + ) + + collector = LLMMetricsCollector() + + # Simulate some request results + import random + import time + + base_time = time.time() + for i in range(20): + latency_s = 0.1 + random.random() * 0.2 # 100-300ms + result = LLMRequestResult( + request_id=f"req_{i}", + model_name="Qwen/Qwen2.5-7B-Instruct", + priority="NORMAL" if random.random() > 0.3 else "HIGH", + slo_deadline_ms=500, + send_time=base_time + i * 0.05, + completion_time=base_time + i * 0.05 + latency_s, + success=random.random() > 0.1, # 90% success rate + first_token_time=base_time + i * 0.05 + 0.02 + random.random() * 0.05, + output_token_count=50 + int(random.random() * 100), + ) + collector.add_result(result) + + # Set time range for accurate throughput + collector.set_time_range(base_time, base_time + 20 * 0.05 + 0.3) + + # Get aggregated metrics + metrics = collector.compute_metrics() + + print("\nAggregated Metrics:") + print(" [Throughput]") + print(f" - Requests/s: {metrics.throughput_rps:.2f}") + print(f" - Tokens/s: {metrics.token_throughput_tps:.2f}") + print(" [Latency]") + print(f" - E2E Avg: {metrics.e2e_latency_avg_ms:.1f} ms") + print(f" - E2E P50: {metrics.e2e_latency_p50_ms:.1f} ms") + print(f" - E2E P99: {metrics.e2e_latency_p99_ms:.1f} ms") + print(f" - TTFT Avg: {metrics.ttft_avg_ms:.1f} ms") + print(" [Quality]") + print(f" - Success Rate: {(1 - metrics.error_rate) * 100:.1f}%") + print(f" - SLO Compliance: {metrics.slo_compliance_rate * 100:.1f}%") + print(" [Volume]") + print(f" - Total Requests: {metrics.total_requests}") + print(f" - Completed: {metrics.completed_requests}") + + +def demo_7_chart_generation(): + """Demo 7: Generate benchmark charts.""" + print("\n" + "=" * 60) + print("Demo 7: Chart Generation") + print("=" * 60) + + try: + import matplotlib + + matplotlib.use("Agg") + except ImportError: + print(" matplotlib not installed, skipping chart demo") + return + + from sage.benchmark.benchmark_control_plane.visualization import BenchmarkCharts + + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + charts = BenchmarkCharts(output_dir=output_dir) + + # Prepare sample data + policy_metrics = { + "fifo": { + "throughput": {"requests_per_second": 95.2, "tokens_per_second": 4521}, + "latency": { + "e2e_avg_ms": 152.3, + "e2e_p50_ms": 142.0, + "e2e_p90_ms": 189.5, + "e2e_p95_ms": 215.2, + "e2e_p99_ms": 289.1, + }, + "slo": {"compliance_rate": 0.92}, + }, + "priority": { + "throughput": {"requests_per_second": 91.5, "tokens_per_second": 4312}, + "latency": { + "e2e_avg_ms": 168.7, + "e2e_p50_ms": 155.0, + "e2e_p90_ms": 201.3, + "e2e_p95_ms": 234.8, + "e2e_p99_ms": 312.5, + }, + "slo": {"compliance_rate": 0.89}, + }, + "slo_aware": { + "throughput": {"requests_per_second": 88.3, "tokens_per_second": 4156}, + "latency": { + "e2e_avg_ms": 145.2, + "e2e_p50_ms": 138.0, + "e2e_p90_ms": 175.6, + "e2e_p95_ms": 198.4, + "e2e_p99_ms": 256.7, + }, + "slo": {"compliance_rate": 0.96}, + }, + } + + print("\nGenerating charts...") + + # Generate throughput comparison + path = charts.plot_throughput_comparison(policy_metrics) + if path: + print(f" - Throughput comparison: {path.name}") + + # Generate latency percentiles + path = charts.plot_latency_percentiles(policy_metrics) + if path: + print(f" - Latency percentiles: {path.name}") + + # Generate SLO compliance + path = charts.plot_slo_compliance(policy_metrics) + if path: + print(f" - SLO compliance: {path.name}") + + charts_files = list(output_dir.glob("*.png")) + print(f"\nCharts generated: {len(charts_files)} files") + for f in charts_files: + print(f" - {f.name}") + + +def demo_8_cli_commands(): + """Demo 8: Show CLI commands.""" + print("\n" + "=" * 60) + print("Demo 8: CLI Commands Reference") + print("=" * 60) + + print( + """ +The benchmark framework provides a CLI tool: sage-cp-bench + +[Available Commands] + +1. Run a single benchmark: + sage-cp-bench run --mode llm --policy fifo --requests 100 --rate 10 + sage-cp-bench run --mode hybrid --policy hybrid --llm-ratio 0.7 --requests 100 + +2. Compare multiple policies: + sage-cp-bench compare --mode llm --policies fifo,priority,slo_aware --requests 500 + sage-cp-bench compare --mode hybrid --policies fifo,hybrid --llm-ratio 0.7 + +3. Sweep request rates: + sage-cp-bench sweep --mode llm --policy fifo --rates 10,50,100,200 + +4. Run predefined experiments: + sage-cp-bench experiment --name throughput --policies fifo,priority + sage-cp-bench experiment --name latency --policies fifo,slo_aware + sage-cp-bench experiment --name slo --policies fifo,priority,slo_aware + sage-cp-bench experiment --name mixed_ratio --policies hybrid + +5. Generate visualizations from results: + sage-cp-bench visualize --input results.json --output ./charts --format all + +6. Show/save example config: + sage-cp-bench config --show + sage-cp-bench config --save ./my_config.yaml + +7. Validate config file: + sage-cp-bench validate --config ./my_config.yaml + +[Common Options] + --control-plane, -c Control Plane URL (default: http://localhost:8080) + --output, -o Output directory (default: ./benchmark_results) + --no-visualize Disable auto visualization + --quiet, -q Suppress progress output +""" + ) + + +async def demo_9_mock_benchmark_run(): + """Demo 9: Run a mock benchmark (no actual Control Plane).""" + print("\n" + "=" * 60) + print("Demo 9: Mock Benchmark Run") + print("=" * 60) + + from sage.benchmark.benchmark_control_plane import ( + LLMBenchmarkConfig, + LLMBenchmarkRunner, + ) + + # Create a minimal config for demo + config = LLMBenchmarkConfig( + control_plane_url="http://localhost:8080", # Won't actually connect + num_requests=5, + request_rate=10.0, + warmup_requests=0, + timeout_seconds=1.0, + policies=["fifo"], + ) + + print("\nNote: This demo shows the runner structure.") + print("In production, it would send requests to an actual Control Plane.") + + _runner = LLMBenchmarkRunner(config) # noqa: F841 + print("\nRunner created:") + print(f" - Config: {config.num_requests} requests at {config.request_rate} req/s") + print(f" - Policies: {config.policies}") + print(f" - Output directory: {config.output_dir}") + + print("\nTo run an actual benchmark, start a Control Plane first:") + print(" sage studio start") + print(" # or") + print(" sage apps llm start --port 8080") + + +def main(): + """Run all demos.""" + print("=" * 60) + print("Control Plane Benchmark Framework Demo") + print("=" * 60) + + # Configuration demos + demo_1_llm_benchmark_config() + demo_2_hybrid_benchmark_config() + + # Workload generation + demo_3_workload_generation() + + # Strategy and monitoring + demo_4_strategy_adapter() + demo_5_gpu_monitor() + + # Metrics + demo_6_metrics_collection() + + # Visualization + demo_7_chart_generation() + + # CLI reference + demo_8_cli_commands() + + # Mock run + asyncio.run(demo_9_mock_benchmark_run()) + + print("\n" + "=" * 60) + print("Demo completed.") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/tutorials/config/README.md b/tutorials/config/README.md new file mode 100644 index 0000000..c5796d3 --- /dev/null +++ b/tutorials/config/README.md @@ -0,0 +1,266 @@ +# Example Configuration Files + +This directory contains YAML configuration files used by various SAGE examples. + +## 📋 Configuration Files + +### General Pipeline Configurations + +#### `config.yaml` + +General-purpose pipeline configuration. + +**Used by:** + +- Various RAG examples +- Basic pipeline examples + +#### `config_source.yaml` & `config_source_local.yaml` + +Data source configurations for pipelines. + +**Used by:** + +- `rag/loaders/document_loaders.py` +- Multiple RAG examples + +______________________________________________________________________ + +### RAG Configurations + +#### `config_qa_chroma.yaml` + +QA pipeline with ChromaDB vector store. + +**Used by:** + +- `rag/qa_dense_retrieval_chroma.py` +- `rag/build_chroma_index.py` + +#### `config_dense_milvus.yaml` + +Dense retrieval with Milvus vector database. + +**Used by:** + +- `rag/qa_dense_retrieval_milvus.py` +- `rag/build_milvus_dense_index.py` + +#### `config_sparse_milvus.yaml` + +Sparse retrieval with Milvus (BM25/SPLADE). + +**Used by:** + +- `rag/qa_sparse_retrieval_milvus.py` +- `rag/build_milvus_sparse_index.py` + +#### `config_bm25s.yaml` + +BM25 sparse retrieval configuration. + +**Used by:** + +- `rag/qa_bm25_retrieval.py` + +#### `config_mixed.yaml` + +Hybrid dense + sparse retrieval. + +**Used by:** + +- `rag/qa_dense_retrieval_mixed.py` + +#### `config_multiplex.yaml` + +Multi-path retrieval pipeline. + +**Used by:** + +- `rag/qa_multiplex.py` + +#### `config_rerank.yaml` + +Retrieval with reranking. + +**Used by:** + +- `rag/qa_rerank.py` + +#### `config_refiner.yaml` + +Query refinement configuration. + +**Used by:** + +- `rag/qa_refiner.py` + +#### `config_ray.yaml` + +Ray-distributed RAG pipeline. + +**Used by:** + +- `rag/qa_dense_retrieval_ray.py` + +#### `config_hf.yaml` + +Hugging Face model configuration. + +**Used by:** + +- `rag/qa_hf_model.py` + +______________________________________________________________________ + +### Agent Configurations + +#### `config_agent_min.yaml` + +Minimal agent configuration example. + +**Used by:** + +- `tutorials/agents/basic_agent.py` +- Agent examples + +______________________________________________________________________ + +### Memory Configurations + +#### `config_rag_memory_pipeline.yaml` + +RAG pipeline with memory integration. + +**Used by:** + +- `tutorials/memory/rag_memory_pipeline.py` +- Memory service examples + +______________________________________________________________________ + +## 🔧 Configuration Structure + +Most configuration files follow this general structure: + +```yaml +# Data Source +source: + type: "directory" | "file" | "huggingface" + path: "path/to/data" + +# Embedding +embedding: + model_name: "BAAI/bge-small-en-v1.5" + dimension: 384 + +# Vector Store +vector_store: + type: "chroma" | "milvus" | "faiss" + collection_name: "my_collection" + +# LLM +llm: + provider: "openai" | "huggingface" | "vllm" + model_name: "gpt-3.5-turbo" + api_key: "${OPENAI_API_KEY}" # From environment + +# Retrieval +retrieval: + top_k: 3 + similarity_threshold: 0.7 +``` + +## 🔐 API Keys + +**Never commit API keys!** Use environment variables: + +```yaml +# Good ✅ +api_key: "${OPENAI_API_KEY}" + +# Bad ❌ +api_key: "sk-abc123..." # pragma: allowlist secret +``` + +Set keys in `.env` file: + +```bash +cp .env.example .env +# Edit .env with your keys +``` + +## 📝 Creating Custom Configurations + +1. **Copy an existing config**: + + ```bash + cp examples/config/config_qa_chroma.yaml my_config.yaml + ``` + +1. **Modify for your needs**: + + - Change model names + - Adjust parameters + - Update paths + +1. **Use in your code**: + + ```python + from sage.common.utils.config.loader import load_config + + config = load_config("my_config.yaml") + ``` + +## 🗂️ Configuration Templates + +### Minimal RAG Config + +```yaml +source: + type: "directory" + path: "data/documents" + +embedding: + model_name: "BAAI/bge-small-en-v1.5" + +vector_store: + type: "chroma" + collection_name: "my_docs" + +llm: + provider: "openai" + model_name: "gpt-3.5-turbo" + api_key: "${OPENAI_API_KEY}" +``` + +### Agent Config Template + +```yaml +agent: + name: "MyAgent" + role: "assistant" + +llm: + provider: "openai" + model_name: "gpt-4" + temperature: 0.7 + +tools: + - name: "web_search" + enabled: true +``` + +## 📚 Related Documentation + +- **Configuration Loader**: `packages/sage-common/src/sage/common/utils/config/` +- **RAG Configuration**: `docs/dev-notes/EMBEDDING_README.md` +- **Environment Setup**: `.env.example` + +## ⚠️ Notes + +- Configurations use YAML format +- Environment variables are interpolated: `${VAR_NAME}` +- Paths can be relative (to project root) or absolute +- Most examples work with default configurations +- Customize as needed for your use case diff --git a/tutorials/config/config.yaml b/tutorials/config/config.yaml new file mode 100644 index 0000000..b2b8587 --- /dev/null +++ b/tutorials/config/config.yaml @@ -0,0 +1,47 @@ +pipeline: + name: "sage-api-operator-operator_test" + description: "Test pipeline for Sage API Operator" + version: "1.0.0" + +source: + data_path: "examples/data/sample/question.txt" + platform: "local" + +retriever: + platform: "local" + faiss: + dimension: 384 # 使用 mockembedder 的维度 + index_type: "IndexFlatL2" + top_k: 3 + knowledge_file: "examples/data/biology_knowledge.txt" # 自动加载知识库 + embedding: + method: "mockembedder" + model_config: + fixed_dim: 384 + +generator: + local: + method: "hf" + model_name: "meta-llama/Llama-2-13b-chat-hf" + seed: 42 + + vllm: + api_key: "" + method: "openai" + model_name: "meta-llama/Llama-2-13b-chat-hf" + base_url: "http://sage3:8000/v1" + seed: 42 + + remote: + api_key: "" + method: "openai" + model_name: "Qwen/Qwen2.5-7B-Instruct" + base_url: "http://127.0.0.1:8889/v1" + seed: 42 + +promptor: + platform: "local" + +sink: + platform: "local" + file_path: "output.txt" diff --git a/tutorials/docs/QUICK_REFERENCE.md b/tutorials/docs/QUICK_REFERENCE.md new file mode 100644 index 0000000..12688ea --- /dev/null +++ b/tutorials/docs/QUICK_REFERENCE.md @@ -0,0 +1,318 @@ +# SAGE Unlearning - Quick Reference + +## 五分钟快速入门 + +### 场景 1️⃣: 研究/原型验证 + +``` +你想快速测试想法，不需要完整的 SAGE 环境 + +→ 使用: usage_1_direct_library.py +→ 命令: python usage_1_direct_library.py +→ 示例函数: + - example_basic_unlearning() # 最简单的例子 + - example_custom_mechanism() # 自定义隐私机制 + - example_batch_unlearning() # 批量遗忘 + - example_similarity_based_unlearning() # 基于相似性的遗忘 + - example_privacy_budget_management() # 隐私预算管理 +``` + +**依赖**: `numpy`, `sage-libs`\ +**复杂度**: ⭐ (简单)\ +**VDB 集成**: ✗ + +______________________________________________________________________ + +### 场景 2️⃣: SAGE Pipeline 集成 + +``` +你想在 SAGE Pipeline 中使用 unlearning + +→ 使用: usage_2_sage_function.py +→ 命令: python usage_2_sage_function.py +→ 关键类: + - VectorGenerator (SourceFunction) # 生成向量数据 + - UnlearningProcessor (BaseFunction) # 执行遗忘 + - ResultCollector (SinkFunction) # 收集结果 + - UnlearningWithStateFunction (stateful) # 有状态处理 + - StateSink (state collector) # 状态收集 + +→ 示例 Pipeline: + - example_basic_pipeline() # 基础 Pipeline + - example_stateful_pipeline() # 有状态处理 + - example_conditional_unlearning() # 条件遗忘 +``` + +**依赖**: SAGE framework, numpy\ +**复杂度**: ⭐⭐ (中等)\ +**VDB 集成**: ✗\ +**流式处理**: ✓ + +______________________________________________________________________ + +### 场景 3️⃣: VDB/MemoryService 集成 + +``` +你想将 unlearning 集成到向量数据库系统 + +→ 使用: usage_3_memory_service.py +→ 命令: python usage_3_memory_service.py +→ 关键类: + - DPMemoryService (extends BaseService) + +→ 关键方法: + - create_collection(name) + - store_memory(collection, vector_id, vector) + - retrieve_memories(collection, query_vector) + - forget_with_dp(collection, vector_ids) + - get_privacy_status(collection) + +→ 示例场景: + - example_basic_dp_memory() # 基础 VDB 遗忘 + - example_privacy_budget_management() # 隐私预算追踪 + - example_multi_collection() # 多 collection 管理 +``` + +**依赖**: SAGE middleware, NeuroMem, numpy\ +**复杂度**: ⭐⭐ (中等)\ +**VDB 集成**: ✓\ +**生产就绪**: ～ (部分) + +______________________________________________________________________ + +### 场景 4️⃣: 完整 RAG 系统 (生产) + +``` +你需要一个生产就绪的 RAG 系统，支持用户隐私和合规性 + +→ 使用: usage_4_complete_rag.py +→ 命令: python usage_4_complete_rag.py +→ 关键类: + - RAGUnlearningSystem (extends BaseService) + +→ 关键方法: + - initialize_rag_corpus(corpus_path) + - retrieve_relevant_documents(query, top_k) + - forget_documents(doc_ids) + - handle_user_deletion_request(collection, user_id) # GDPR + - handle_malicious_content_removal(content_patterns) # 恶意内容 + - get_audit_log() # 审计追踪 + +→ 示例场景: + - example_basic_rag() # 基础 RAG + - example_malicious_content() # 恶意内容删除 + - example_audit_log() # 审计日志 +``` + +**依赖**: 完整 SAGE 环境, 检索系统\ +**复杂度**: ⭐⭐⭐ (复杂)\ +**VDB 集成**: ✓\ +**审计日志**: ✓\ +**GDPR 支持**: ✓\ +**生产就绪**: ✓ + +______________________________________________________________________ + +## 使用场景决策树 + +``` +┌─────────────────────────────────────────┐ +│ 选择合适的使用方式 │ +└─────────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────┐ + │ 需要 VDB/向量存储 吗？ │ + └───────┬──────────────┬───┘ + │NO │YES + ▼ ▼ + ┌────────────┐ ┌──────────────┐ + │ 使用流式 │ │ 需要生产级 │ + │ Pipeline? │ │ 功能吗？ │ + └─┬──────┬──┘ └───┬──────┬────┘ + YES │ │NO YES│ │NO + ▼ ▼ ▼ ▼ + ┌─────────────┐ ┌────────────────┐ + │ usage_2 │ │ usage_3 │ + │ SAGE │ │ MemService │ + │ Function │ │ (VDB only) │ + └─────────────┘ └────────────────┘ + + ┌──────────────────────────────┐ + │ 需要审计/GDPR/合规 吗？ │ + └───────┬──────────────┬───────┘ + YES │ │ NO + ▼ ▼ + ┌─────────────┐ ┌─────────────┐ + │ usage_4 │ │ usage_1/2/3 │ + │ RAG System │ │ │ + │ (Production)│ │ Research │ + └─────────────┘ └─────────────┘ + + ┌──────────────────────────────────────┐ + │ 完全不使用 SAGE 框架？ │ + └───────┬──────────────────────────┬───┘ + YES │ │ NO + ▼ ▼ + ┌─────────────┐ ┌────────────┐ + │ usage_1 │ │ usage_2/3/4│ + │ Direct Lib │ │ with SAGE │ + │ (Pure Algo) │ │ │ + └─────────────┘ └────────────┘ +``` + +______________________________________________________________________ + +## 性能对比 + +| 特性 | usage_1 | usage_2 | usage_3 | usage_4 | +| ------------ | ------- | ------- | ------- | ------- | +| 初始化时间 | 毫秒 | 秒 | 秒 | 秒+ | +| 每个遗忘操作 | 毫秒 | 毫秒 | 秒 | 秒+ | +| 内存开销 | 低 | 中 | 中 | 高 | +| 吞吐量 | 高 | 中高 | 中 | 中 | +| 可扩展性 | ✗ | ✓ | ✓ | ✓✓ | + +______________________________________________________________________ + +## 代码片段速查 + +### 最小化 Direct Library 示例 + +```python +from sage.libs.privacy.unlearning import UnlearningEngine +import numpy as np + +engine = UnlearningEngine(epsilon=1.0) + +# 准备数据 +vectors = np.random.randn(100, 128).astype(np.float32) +forget_vectors = vectors[:10] + +# 执行遗忘 +result = engine.unlearn_vectors( + vectors_to_forget=forget_vectors, + all_vectors=vectors, + perturbation_strategy="adaptive", +) + +print(f"Privacy cost: {result.privacy_cost}") +``` + +### 最小化 SAGE Function 示例 + +```python +from sage.kernel.api.function.base_function import BaseFunction + + +class MyUnlearningFunction(BaseFunction): + def __init__(self): + super().__init__() + self.engine = UnlearningEngine(epsilon=1.0) + + def execute(self, data): + if data.get("should_forget"): + result = self.engine.unlearn_vectors(...) + return {"vector": result.metadata["perturbed_vectors"][0]} + return data +``` + +### 最小化 MemService 示例 + +```python +from sage.kernel.api.service.base_service import BaseService + + +class MyMemService(BaseService): + def __init__(self): + super().__init__() + self.manager = MemoryManager() + self.engine = UnlearningEngine(epsilon=1.0) + + def forget(self, collection_name, vector_ids): + collection = self.manager.get_collection(collection_name) + # 获取向量、执行遗忘、更新 VDB +``` + +### 最小化 RAG 示例 + +```python +class MyRAGSystem(BaseService): + def handle_deletion_request(self, user_id): + # 1. 查找用户的所有向量 + # 2. 执行 DP 遗忘 + # 3. 更新 VDB + # 4. 记录审计日志 +``` + +______________________________________________________________________ + +## 学习建议 + +### 🚀 快速开始 (10分钟) + +1. 阅读本文档 +1. 运行 `basic_unlearning_demo.py` +1. 查看 `usage_1_direct_library.py` 的前 3 个函数 + +### 📚 深入学习 (1小时) + +1. 运行所有 `usage_1_*` 示例 +1. 理解隐私预算和扰动策略 +1. 尝试修改参数看效果 + +### 🏗️ 开发集成 (1天) + +1. 学习 `usage_2` 或 `usage_3` +1. 根据你的用例选择合适的模式 +1. 实现自己的业务逻辑 + +### 🔬 研究扩展 (1周+) + +1. 学习 `usage_4` 的完整实现 +1. 探索 EMBEDDING_README.md 中的向量系统 +1. 实现自己的隐私机制或评估指标 + +______________________________________________________________________ + +## 常见问题 + +**Q: 哪个示例最适合我？**\ +A: 查看上面的决策树，或根据你的需求选择： + +- 快速实验 → usage_1 +- Pipeline 集成 → usage_2 +- VDB 系统 → usage_3 +- 生产应用 → usage_4 + +**Q: 需要修改哪些参数？**\ +A: 最常见的是 `epsilon`（隐私预算，越小越隐私）和 `perturbation_strategy`（扰动策略） + +**Q: 如何添加自定义隐私机制？**\ +A: 参考 `usage_1_direct_library.py` 中的 `example_custom_mechanism()` + +**Q: 如何评估遗忘效果？**\ +A: 使用 `sage.libs.unlearning.metrics` 中的评估函数，示例见各文件 + +**Q: 生产环境需要注意什么？**\ +A: 使用 `usage_4` 的 RAG 系统模板，包括审计日志、错误处理、隐私预算监控 + +______________________________________________________________________ + +## 文件导航 + +``` +examples/unlearning/ +├── README.md # 完整文档 +├── USAGE_GUIDE.md # 详细用途指南 +├── QUICK_REFERENCE.md # 本文档 +├── basic_unlearning_demo.py # 入门演示 +├── usage_1_direct_library.py # 直接库用法 +├── usage_2_sage_function.py # SAGE Function 集成 +├── usage_3_memory_service.py # VDB 集成 +└── usage_4_complete_rag.py # 完整 RAG 系统 +``` + +______________________________________________________________________ + +**下一步**: 选择合适的示例文件开始学习或开发！ diff --git a/tutorials/docs/TROUBLESHOOTING.md b/tutorials/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000..6242456 --- /dev/null +++ b/tutorials/docs/TROUBLESHOOTING.md @@ -0,0 +1,655 @@ +# SAGE Unlearning - 故障排查指南 + +## 常见错误和解决方案 + +### 1. ImportError 和依赖问题 + +#### 错误 1.1: `ModuleNotFoundError: No module named 'sage'` + +**症状:** + +``` +ModuleNotFoundError: No module named 'sage' + File "usage_1_direct_library.py", line 5, in + from sage.libs.privacy.unlearning import UnlearningEngine +``` + +**原因:** SAGE 库未安装 + +**解决方案:** + +```bash +# 方法 1: 在 SAGE 根目录安装 +cd /home/shuhao/SAGE +pip install -e packages/sage-libs + +# 方法 2: 或者完整安装 +make install + +# 方法 3: 验证安装 +python -c "from sage.libs.privacy.unlearning import UnlearningEngine; print('OK')" +``` + +**预防:** + +- 始终在虚拟环境中工作 +- 运行代码前检查依赖: `pip list | grep sage` + +______________________________________________________________________ + +#### 错误 1.2: `ImportError: cannot import name 'BaseFunction'` + +**症状:** + +``` +ImportError: cannot import name 'BaseFunction' from 'sage.kernel' +``` + +**原因:** + +- SAGE 运行时未安装（usage_2, 3, 4 需要） +- 或版本不兼容 + +**解决方案:** + +```bash +# 安装完整 SAGE +pip install -e packages/sage-kernel +pip install -e packages/sage-middleware + +# 验证 +python -c "from sage.kernel.api.function.base_function import BaseFunction; print('OK')" +``` + +**预防:** + +- usage_1 不需要 SAGE 运行时 +- usage_2-4 需要 `sage-kernel` +- 查看 README.md 的依赖清单 + +______________________________________________________________________ + +#### 错误 1.3: `ModuleNotFoundError: No module named 'neuromem'` + +**症状:** + +``` +ModuleNotFoundError: No module named 'neuromem' + File "usage_3_memory_service.py", line 8 + from sage.middleware.components.sage_mem.neuromem import MemoryManager +``` + +**原因:** NeuroMem 组件未安装（usage_3, 4 需要） + +**解决方案:** + +```bash +# 安装中间件 +pip install -e packages/sage-middleware + +# 或单独安装向量库 +pip install faiss-cpu +# 或 GPU 版本 +pip install faiss-gpu + +# 验证 +python -c "from sage.middleware.components.sage_mem.neuromem.memory_manager import MemoryManager; print('OK')" +``` + +**预防:** + +- usage_1, 2 不需要 NeuroMem +- usage_3, 4 才需要 VDB + +______________________________________________________________________ + +### 2. 数据和计算错误 + +#### 错误 2.1: `ValueError: shapes (100, 128) and (127, 128) not aligned` + +**症状:** + +``` +ValueError: shapes (100, 128) and (127, 128) not aligned + at engine.unlearn_vectors() +``` + +**原因:** 向量维度不匹配 + +**解决方案:** + +```python +# ❌ 错误 +vectors = np.random.randn(100, 128) # 100 个向量 +forget_vectors = np.random.randn(10, 127) # 维度不同! + +# ✓ 正确 +vectors = np.random.randn(100, 128) +forget_vectors = vectors[:10] # 从同一数据源取样 + +# ✓ 或者 +forget_vectors = np.random.randn(10, 128) # 维度相同 +``` + +**调试技巧:** + +```python +print(f"All vectors shape: {vectors.shape}") +print(f"Forget vectors shape: {forget_vectors.shape}") +assert vectors.shape[1] == forget_vectors.shape[1], "维度不匹配!" +``` + +______________________________________________________________________ + +#### 错误 2.2: `RuntimeError: CUDA out of memory` + +**症状:** + +``` +RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB. +``` + +**原因:** 向量过多或维度过高 + +**解决方案:** + +方式 1: 减少向量数量 + +```python +# ❌ 原始 +vectors = np.random.randn(10000000, 1024) # 太多了! + +# ✓ 修改 +vectors = np.random.randn(100000, 1024) # 降低数量 +``` + +方式 2: 使用 CPU + +```python +# 强制使用 CPU +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "" + +# 或在创建时指定 +engine = UnlearningEngine(epsilon=1.0, device="cpu") +``` + +方式 3: 分批处理 + +```python +batch_size = 10000 +for i in range(0, len(vectors), batch_size): + batch = vectors[i : i + batch_size] + result = engine.unlearn_vectors(...) +``` + +______________________________________________________________________ + +#### 错误 2.3: `ValueError: epsilon must be positive` + +**症状:** + +``` +ValueError: epsilon must be positive, got -1.0 +``` + +**原因:** 隐私预算参数无效 + +**解决方案:** + +```python +# ❌ 错误 +engine = UnlearningEngine(epsilon=-1.0) # 负数! +engine = UnlearningEngine(epsilon=0) # 零! + +# ✓ 正确 +engine = UnlearningEngine(epsilon=1.0) # 正数 +engine = UnlearningEngine(epsilon=0.5) # 更严格的隐私 +engine = UnlearningEngine(epsilon=5.0) # 宽松的隐私 + +# 参考值 +# epsilon < 0.1: 非常严格 +# epsilon 0.1-1.0: 严格（推荐） +# epsilon 1.0-5.0: 中等 +# epsilon > 5.0: 宽松 +``` + +______________________________________________________________________ + +#### 错误 2.4: `OverflowError: cannot convert float infinity to integer` + +**症状:** + +``` +OverflowError: cannot convert float infinity to integer + in privacy_accountant.compute_epsilon() +``` + +**原因:** 向量全为零或包含 NaN + +**解决方案:** + +```python +# 检查数据质量 +print(f"含 NaN: {np.isnan(vectors).any()}") +print(f"含 Inf: {np.isinf(vectors).any()}") +print(f"全零向量: {np.allclose(vectors, 0).any()}") + +# ✓ 清理数据 +vectors = np.nan_to_num(vectors) # NaN → 0 +vectors = np.clip(vectors, -1e6, 1e6) # 裁剪极值 +vectors = vectors / (np.linalg.norm(vectors, axis=1, keepdims=True) + 1e-10) # 归一化 +``` + +______________________________________________________________________ + +### 3. SAGE Framework 错误 + +#### 错误 3.1: `RuntimeError: No active environment` + +**症状:** + +``` +RuntimeError: No active environment found. +``` + +**原因:** 未创建 SAGE 环境 + +**解决方案 (usage_2 中):** + +```python +# ❌ 错误 +from sage.kernel.api.environment.local_environment import LocalEnvironment + +# 直接使用 Pipeline，没有环境 + +# ✓ 正确 +env = LocalEnvironment("my_env") +env.from_source(...).map(...).sink(...) +env.submit() +``` + +______________________________________________________________________ + +#### 错误 3.2: `AttributeError: 'NoneType' object has no attribute 'execute'` + +**症状:** + +``` +AttributeError: 'NoneType' object has no attribute 'execute' +``` + +**原因:** Function 返回 None 或未初始化 + +**解决方案:** + +```python +# ❌ 错误 +class MyFunction(BaseFunction): + def execute(self, data): + # 缺少返回值! + pass + + +# ✓ 正确 +class MyFunction(BaseFunction): + def execute(self, data): + return data # 必须返回某些东西 +``` + +______________________________________________________________________ + +#### 错误 3.3: `KeyError: 'collection_name'` (usage_3 中) + +**症状:** + +``` +KeyError: 'collection_name' + in service.forget_with_dp("collection_name", ...) +``` + +**原因:** Collection 不存在 + +**解决方案:** + +```python +# ❌ 错误 +service = DPMemoryService() +service.forget_with_dp("docs", ["id_1"]) # docs 不存在! + +# ✓ 正确 +service = DPMemoryService() +service.create_collection("docs") # 先创建 +service.store_memory("docs", "id_1", vector) # 再存储 +service.forget_with_dp("docs", ["id_1"]) # 再遗忘 +``` + +**调试代码:** + +```python +# 列出所有 collections +collections = service.manager.list_collections() +print(f"Available collections: {collections}") + +# 检查 collection 是否存在 +if not service.manager.has_collection("docs"): + service.create_collection("docs") +``` + +______________________________________________________________________ + +### 4. 性能问题 + +#### 问题 4.1: 代码运行非常慢 + +**症状:** + +``` +# 处理 1000 个向量花费 > 1 分钟 +result = engine.unlearn_vectors(...) # 卡住... +``` + +**诊断:** + +```python +import time + +# 计时 +start = time.time() +result = engine.unlearn_vectors(...) +elapsed = time.time() - start + +print(f"Time: {elapsed:.2f}s") +print(f"Throughput: {len(vectors) / elapsed:.0f} vectors/sec") + +# 预期值 +# usage_1: > 10k 向量/秒 +# usage_3: > 1k 向量/秒 +# usage_4: > 500 向量/秒 +``` + +**原因和解决方案:** + +| 原因 | 症状 | 解决方案 | +| --------------------- | ------------------ | --------------------------- | +| 使用 CPU 处理大量向量 | 很慢 (< 100 vec/s) | 使用 GPU 或减少向量数 | +| 向量维度过高 | 慢 (< 1k vec/s) | 降低维度或批处理 | +| 过度精度计算 | 很慢 | 使用 float32 而不是 float64 | +| VDB 索引未优化 | 慢 (usage_3/4) | 重建索引或调整参数 | + +______________________________________________________________________ + +#### 问题 4.2: 内存占用很高 + +**症状:** + +``` +# 内存使用超过预期 +# 处理 100k 向量时占用 > 20GB +``` + +**诊断:** + +```python +import tracemalloc + +tracemalloc.start() + +result = engine.unlearn_vectors(...) + +current, peak = tracemalloc.get_traced_memory() +print(f"Current: {current / 1e9:.2f} GB") +print(f"Peak: {peak / 1e9:.2f} GB") + +# 预期值 +# 100k × 128 向量（float32）: ~50MB +# + 临时缓冲: ~100-200MB +# 总计应该 < 500MB +``` + +**解决方案:** + +```python +# 方案 1: 分批处理 +batch_size = 10000 +for i in range(0, len(vectors), batch_size): + batch = vectors[i : i + batch_size] + result = engine.unlearn_vectors(batch) + +# 方案 2: 使用内存映射 +import numpy as np + +vectors = np.memmap("vectors.npy", dtype=np.float32, shape=(1000000, 128)) + +# 方案 3: 使用更小的数据类型 +vectors = vectors.astype(np.float16) # 减半内存 +``` + +______________________________________________________________________ + +### 5. 隐私和安全问题 + +#### 问题 5.1: 隐私预算不足 + +**症状:** + +``` +PrivacyBudgetError: Remaining budget (0.01) < required (0.1) +``` + +**原因:** 太多遗忘操作耗尽隐私预算 + +**解决方案:** + +```python +# 方案 1: 增加初始预算 +engine = UnlearningEngine(epsilon=10.0) # 而不是 1.0 + +# 方案 2: 批量遗忘（更高效） +# ❌ 低效: 单个遗忘 × 100 +for vector_id in forget_ids: + engine.unlearn_vectors(...) + +# ✓ 高效: 一次批量遗忘 +forget_vectors = vectors[forget_ids] +engine.unlearn_vectors(forget_vectors, forget_ids) + +# 方案 3: 监控预算 +print(f"Remaining budget: {engine.privacy_accountant.get_remaining_epsilon()}") +``` + +**最佳实践:** + +```python +# 预先规划预算 +total_epsilon = 1.0 # 总预算 +num_operations = 100 # 预计操作数 +epsilon_per_op = total_epsilon / num_operations + +engine = UnlearningEngine(epsilon=epsilon_per_op) + +# 或使用自适应预算 +remaining = total_epsilon +for operation in operations: + engine.epsilon = remaining / len(remaining_operations) + engine.unlearn_vectors(...) + remaining -= consumed +``` + +______________________________________________________________________ + +#### 问题 5.2: 无法验证遗忘效果 + +**症状:** + +``` +遗忘后的向量与原始向量仍然相似 +``` + +**诊断和解决方案:** + +```python +from sklearn.metrics.pairwise import cosine_similarity + +# 计算相似度 +similarity = cosine_similarity([original_vector], [perturbed_vector])[0, 0] +print(f"Similarity: {similarity:.4f}") + +# ✓ 好的遗忘 (相似度低) +# similarity < 0.7 表示有效遗忘 + +# 调试 +if similarity > 0.7: + # 方案 1: 增加扰动强度 + result = engine.unlearn_vectors(..., perturbation_strategy="aggressive") + + # 方案 2: 增加邻居补偿 + result = engine.unlearn_vectors(..., apply_compensation=True) + + # 方案 3: 使用更强的隐私机制 + from sage.libs.privacy.unlearning.privacy_mechanisms import GaussianUnlearning + + result = engine.unlearn_vectors(..., privacy_mechanism=GaussianUnlearning()) +``` + +______________________________________________________________________ + +### 6. 使用建议和最佳实践 + +#### 6.1 一般最佳实践 + +```python +# 1️⃣ 总是检查输入 +assert vectors.shape[1] == expected_dim, "维度错误" +assert not np.isnan(vectors).any(), "包含 NaN" +assert len(vector_ids) == len(vectors), "ID 数量不匹配" + +# 2️⃣ 启用日志 +import logging + +logging.basicConfig(level=logging.DEBUG) + +# 3️⃣ 使用错误处理 +try: + result = engine.unlearn_vectors(...) +except ValueError as e: + print(f"Invalid input: {e}") +except PrivacyBudgetError as e: + print(f"Privacy budget exceeded: {e}") + +# 4️⃣ 验证输出 +assert result.success, f"Unlearning failed: {result.error}" +assert result.num_vectors_unlearned == len(forget_ids) + +# 5️⃣ 记录关键指标 +print(f"Privacy cost: {result.privacy_cost}") +print(f"Perturbation magnitude: {result.metadata['perturbation_magnitude']}") +print(f"Effectiveness: {result.metadata.get('effectiveness', 'N/A')}") +``` + +______________________________________________________________________ + +#### 6.2 调试清单 + +运行代码前检查： + +```python +□ 所有必要的依赖已安装 +□ 数据格式正确（shape, dtype） +□ 无 NaN 或 Inf 值 +□ 隐私参数有效（epsilon > 0） +□ Collection 已创建（usage_3/4） +□ 向量已归一化（如需要） +□ 内存足够 +□ GPU 可用（如使用） +``` + +______________________________________________________________________ + +#### 6.3 测试模板 + +```python +def test_unlearning(): + # 准备 + np.random.seed(42) + vectors = np.random.randn(100, 128).astype(np.float32) + vectors = vectors / np.linalg.norm(vectors, axis=1, keepdims=True) + ids = [f"doc_{i}" for i in range(100)] + + # 创建引擎 + engine = UnlearningEngine(epsilon=1.0) + + # 执行 + result = engine.unlearn_vectors( + vectors_to_forget=vectors[:10], + vector_ids_to_forget=ids[:10], + all_vectors=vectors, + all_vector_ids=ids, + ) + + # 验证 + assert result.success + assert result.num_vectors_unlearned == 10 + assert result.privacy_cost[0] <= 1.0 + + # 效果检查 + for i, forgotten_id in enumerate(ids[:10]): + orig = vectors[i] + perturbed = result.metadata["perturbed_vectors"][i] + similarity = np.dot(orig, perturbed) + assert similarity < 0.9, f"Insufficient perturbation for {forgotten_id}" + + print("✓ All tests passed") + + +if __name__ == "__main__": + test_unlearning() +``` + +______________________________________________________________________ + +### 7. 获取更多帮助 + +**文档:** + +- 📖 README.md - 总体概况 +- 📋 QUICK_REFERENCE.md - 快速参考 +- 📊 COMPARISON_MATRIX.md - 详细对比 +- 🔧 各个 usage\_\*.py 文件中的详细注释 + +**代码示例:** + +- `usage_1_direct_library.py` - 直接库用法 +- `usage_2_sage_function.py` - SAGE Function +- `usage_3_memory_service.py` - VDB 集成 +- `usage_4_complete_rag.py` - 完整系统 + +**调试技巧:** + +```python +# 启用详细日志 +import logging + +logging.basicConfig( + level=logging.DEBUG, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) + +# 添加自定义日志 +logger = logging.getLogger(__name__) +logger.debug(f"Vector shape: {vectors.shape}") +logger.info(f"Unlearning {len(forget_ids)} vectors") +logger.warning(f"Privacy budget: {remaining_budget}") +``` + +**常见资源:** + +- SAGE 官方文档: docs/ +- 研发笔记: docs/dev-notes/ +- API 参考: 源码中的 docstring + +______________________________________________________________________ + +**更新历史** + +- v1.0: 初始版本，涵盖常见错误和解决方案 diff --git a/tutorials/embedding_server_example.py b/tutorials/embedding_server_example.py new file mode 100644 index 0000000..d3391ea --- /dev/null +++ b/tutorials/embedding_server_example.py @@ -0,0 +1,147 @@ +""" +Embedding Server 使用示例 + +这个示例展示如何使用本地 embedding 服务器， +无需修改任何原有代码，直接通过 apply_embedding_model 调用。 +""" + +from sage.common.components.sage_embedding.embedding_api import apply_embedding_model + + +def example_basic_usage(): + """基本使用示例""" + print("=" * 60) + print("示例 1: 基本使用") + print("=" * 60) + + # 创建 embedding 模型实例（连接到本地服务器） + embedding_model = apply_embedding_model( + name="openai", # 使用 openai 方法（兼容 OpenAI API） + model="BAAI/bge-m3", # 模型名称（任意，服务器会忽略） + base_url="http://localhost:8091/v1", # 本地服务器地址 + api_key="dummy", # 本地服务不需要真实的 API key + ) + + # 测试 embedding + text = "Hello, this is a test sentence." + print(f"\nInput text: {text}") + + embedding = embedding_model.embed(text) + print(f"Embedding dimension: {len(embedding)}") + print(f"First 5 values: {embedding[:5]}") + + +def example_batch_processing(): + """批量处理示例""" + print("\n" + "=" * 60) + print("示例 2: 批量处理") + print("=" * 60) + + embedding_model = apply_embedding_model( + name="openai", + model="BAAI/bge-m3", + base_url="http://localhost:8091/v1", + api_key="dummy", + ) + + # 多个文本 + texts = [ + "What is machine learning?", + "Deep learning is a subset of machine learning.", + "Natural language processing uses neural networks.", + ] + + print(f"\nProcessing {len(texts)} texts...") + for i, text in enumerate(texts, 1): + embedding = embedding_model.embed(text) + print(f"{i}. Text: '{text[:50]}...' -> Embedding dim: {len(embedding)}") + + +def example_with_different_server(): + """使用不同服务器端口的示例""" + print("\n" + "=" * 60) + print("示例 3: 使用不同服务器") + print("=" * 60) + + # 假设你在端口 8081 运行另一个模型 + embedding_model = apply_embedding_model( + name="openai", + model="custom-model", + base_url="http://localhost:8081/v1", # 不同端口 + api_key="dummy", + ) + + text = "Testing with different server port" + try: + embedding = embedding_model.embed(text) + print(f"Success! Embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error (expected if server not running): {e}") + + +def example_error_handling(): + """错误处理示例""" + print("\n" + "=" * 60) + print("示例 4: 错误处理") + print("=" * 60) + + embedding_model = apply_embedding_model( + name="openai", + model="BAAI/bge-m3", + base_url="http://localhost:8091/v1", + api_key="dummy", + ) + + # 测试空文本 + try: + embedding = embedding_model.embed("") + print(f"Empty text embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error with empty text: {e}") + + # 测试很长的文本（会被截断） + long_text = "This is a very long sentence. " * 100 + try: + embedding = embedding_model.embed(long_text) + print(f"Long text (truncated) embedding dimension: {len(embedding)}") + except Exception as e: + print(f"Error with long text: {e}") + + +def main(): + """主函数""" + print("\n") + print("=" * 60) + print("Embedding Server 使用示例") + print("=" * 60) + print("\n请确保 embedding 服务器已启动:") + print( + " bash packages/sage-common/src/sage/common/components/sage_embedding/start_embedding_server.sh 8091" + ) + print("\n或手动启动:") + print( + " python packages/sage-common/src/sage/common/components/sage_embedding/embedding_server.py --model BAAI/bge-m3 --port 8091" + ) + print("\n" + "=" * 60 + "\n") + + try: + # 运行示例 + example_basic_usage() + example_batch_processing() + example_with_different_server() + example_error_handling() + + print("\n" + "=" * 60) + print("所有示例完成！") + print("=" * 60) + + except Exception as e: + print(f"\n错误: {e}") + print("\n请确保 embedding 服务器正在运行:") + print( + " bash packages/sage-common/src/sage/common/components/sage_embedding/start_embedding_server.sh 8091" + ) + + +if __name__ == "__main__": + main() diff --git a/tutorials/hello_world.py b/tutorials/hello_world.py new file mode 100644 index 0000000..6dcbfab --- /dev/null +++ b/tutorials/hello_world.py @@ -0,0 +1,47 @@ +from sage.common.core.functions.batch_function import BatchFunction +from sage.common.core.functions.map_function import MapFunction +from sage.common.core.functions.sink_function import SinkFunction +from sage.common.utils.logging.custom_logger import CustomLogger +from sage.kernel.api.local_environment import LocalEnvironment + + +# 批处理数据源：作用是生成10条"Hello, World!"字符串 +class HelloBatch(BatchFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.counter = 0 + self.max_count = 10 # 生成10个数据包后返回None + + def execute(self): + if self.counter >= self.max_count: + return None # 返回None表示批处理完成 + self.counter += 1 + return f"Hello, World! #{self.counter}" + + +# 简单的 MapFunction，将内容转大写 +class UpperCaseMap(MapFunction): + def execute(self, data): + return data.upper() + + +# 简单 SinkFunction，直接打印结果 +class PrintSink(SinkFunction): + def execute(self, data): + print(data) + + +def main(): + env = LocalEnvironment("Hello_World") + + # 批处理源 -> map -> sink + env.from_batch(HelloBatch).map(UpperCaseMap).sink(PrintSink) + + env.submit(autostop=True) + print("Hello World 批处理示例结束") + + +if __name__ == "__main__": + # 关闭日志输出 + CustomLogger.disable_global_console_debug() + main() diff --git a/tutorials/l6/memory_backend_demo.py b/tutorials/l6/memory_backend_demo.py new file mode 100644 index 0000000..fc2677f --- /dev/null +++ b/tutorials/l6/memory_backend_demo.py @@ -0,0 +1,111 @@ +""" +SAGE Memory Backend 示例 + +展示如何直接使用重构后的 neuromem 服务 +（SessionManager 的记忆后端 API 正在适配重构后的 neuromem） +""" + +from sage.middleware.components.sage_mem.neuromem.memory_collection import UnifiedCollection +from sage.middleware.components.sage_mem.neuromem.services.partitional.fifo_queue_service import ( + FIFOQueueService, +) + + +def demo_short_term_memory(): + """演示短期记忆后端 - 使用重构后的 FIFOQueueService""" + print("\n" + "=" * 50) + print("Demo 1: Short-Term Memory Backend (FIFO Queue)") + print("=" * 50) + + # 创建 UnifiedCollection + collection = UnifiedCollection(name="short_term_memory") + + # 创建 FIFO 队列服务，只保留最近3条记录 + service = FIFOQueueService(collection, config={"max_size": 3}) + + print("Created FIFO queue service with max_size=3") + + # 存储5轮对话 + dialogs = [ + ("你好", "您好！有什么可以帮您的吗？"), + ("今天天气怎么样", "今天天气晴朗，温度适宜。"), + ("明天呢", "明天预计多云，气温会略有下降。"), + ("推荐穿什么衣服", "建议穿长袖外套，早晚温差较大。"), + ("谢谢", "不客气，很高兴能帮到您！"), + ] + + for i, (user_msg, assistant_msg) in enumerate(dialogs, 1): + # 插入用户消息和助手回复 + dialog_text = f"User: {user_msg}\nAssistant: {assistant_msg}" + service.insert(dialog_text, metadata={"round": i}) + print(f"Stored dialog {i}: {user_msg}") + + # 检索历史（只会返回最近3轮，因为 max_size=3） + results = service.retrieve("", top_k=10) # 尝试取10条，但只会返回最近3条 + print(f"\nRetrieved {len(results)} dialogs (should be 3 due to FIFO limit):") + for i, result in enumerate(results, 1): + # result 是字典，包含 'id', 'text', 'metadata', 'score' 等字段 + text = result.get("text", "") + entry_preview = text[:50] + "..." if len(text) > 50 else text + print(f" {i}. {entry_preview}") + + print(f"\n✅ Demo completed: FIFO queue correctly limited to {len(results)} items") + + +def demo_vdb_memory(): + """演示向量数据库记忆后端 - 暂时跳过，等待 SessionManager 适配""" + print("\n" + "=" * 50) + print("Demo 2: Vector Database (VDB) Memory Backend") + print("=" * 50) + print("⚠️ Skipped: SessionManager VDB backend needs API update.") + print(" See neuromem services for direct usage examples.") + + +def demo_kv_memory(): + """演示键值存储记忆后端 - 暂时跳过，等待 SessionManager 适配""" + print("\n" + "=" * 50) + print("Demo 3: Key-Value (KV) Memory Backend") + print("=" * 50) + print("⚠️ Skipped: SessionManager KV backend needs API update.") + print(" See neuromem services for direct usage examples.") + + +def demo_graph_memory(): + """演示图记忆后端 - 暂时跳过，等待 SessionManager 适配""" + print("\n" + "=" * 50) + print("Demo 4: Graph Memory Backend") + print("=" * 50) + print("⚠️ Skipped: SessionManager Graph backend needs API update.") + print(" See neuromem services for direct usage examples.") + + +def main(): + """主函数：运行所有示例""" + print("\n" + "=" * 70) + print(" " * 10 + "SAGE Neuromem Services Demo (Refactored)") + print("=" * 70) + print("\n📝 Note: This demo uses the refactored neuromem services directly.") + print(" SessionManager memory backend API is being updated to support") + print(" the new neuromem architecture.") + + # Demo 1: Short-Term Memory (FIFO Queue) + try: + demo_short_term_memory() + except Exception as e: + print(f"\n❌ FIFO demo failed: {e}") + import traceback + + traceback.print_exc() + + # Demo 2-4: Skipped until SessionManager API is updated + demo_vdb_memory() + demo_kv_memory() + demo_graph_memory() + + print("\n" + "=" * 70) + print(" " * 20 + "Demo completed!") + print("=" * 70 + "\n") + + +if __name__ == "__main__": + main() diff --git a/tutorials/vllm_control_plane_config_examples.md b/tutorials/vllm_control_plane_config_examples.md new file mode 100644 index 0000000..48a227f --- /dev/null +++ b/tutorials/vllm_control_plane_config_examples.md @@ -0,0 +1,253 @@ +# SAGE vLLM Control Plane Configuration Examples + +## Example 1: Basic Single Instance Configuration + +```toml +[services.llm_basic] +type = "ControlPlaneVLLMService" +scheduling_policy = "fifo" +enable_pd_separation = false +routing_strategy = "load_balanced" +default_priority = "NORMAL" +default_max_tokens = 512 + +[[services.llm_basic.instances]] +instance_id = "llm-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 1 +gpu_count = 1 +``` + +## Example 2: Multi-Instance Load Balancing + +```toml +[services.llm_multi] +type = "ControlPlaneVLLMService" +scheduling_policy = "adaptive" +enable_pd_separation = false +routing_strategy = "load_balanced" + +[[services.llm_multi.instances]] +instance_id = "llm-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 1 +gpu_count = 1 + +[[services.llm_multi.instances]] +instance_id = "llm-2" +host = "localhost" +port = 8001 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 1 +gpu_count = 1 + +[[services.llm_multi.instances]] +instance_id = "llm-3" +host = "localhost" +port = 8002 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 2 +gpu_count = 2 +``` + +## Example 3: Prefilling/Decoding Separation (Recommended for Production) + +```toml +[services.llm_pd_separated] +type = "ControlPlaneVLLMService" +scheduling_policy = "slo_aware" +enable_pd_separation = true # Enable PD separation for +50-80% throughput +routing_strategy = "topology_aware" +default_slo_deadline_ms = 1000 + +# Prefilling instance - optimized for initial token generation +[[services.llm_pd_separated.instances]] +instance_id = "llm-prefill-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +instance_type = "PREFILLING" +tensor_parallel_size = 4 # Higher TP for prefilling +gpu_count = 4 +gpu_memory_gb = 80.0 +max_concurrent_requests = 128 + +# Decoding instances - optimized for token-by-token generation +[[services.llm_pd_separated.instances]] +instance_id = "llm-decode-1" +host = "localhost" +port = 8001 +model_name = "meta-llama/Llama-2-7b-hf" +instance_type = "DECODING" +tensor_parallel_size = 1 # Lower TP for decoding +gpu_count = 1 +gpu_memory_gb = 80.0 +max_concurrent_requests = 256 + +[[services.llm_pd_separated.instances]] +instance_id = "llm-decode-2" +host = "localhost" +port = 8002 +model_name = "meta-llama/Llama-2-7b-hf" +instance_type = "DECODING" +tensor_parallel_size = 1 +gpu_count = 1 +gpu_memory_gb = 80.0 +max_concurrent_requests = 256 +``` + +## Example 4: Priority-Based Scheduling + +```toml +[services.llm_priority] +type = "ControlPlaneVLLMService" +scheduling_policy = "priority" +enable_pd_separation = false +routing_strategy = "affinity" +default_priority = "NORMAL" + +[[services.llm_priority.instances]] +instance_id = "llm-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 2 +gpu_count = 2 +``` + +## Example 5: SLO-Aware Scheduling + +```toml +[services.llm_slo] +type = "ControlPlaneVLLMService" +scheduling_policy = "slo_aware" +enable_pd_separation = true +routing_strategy = "load_balanced" +default_slo_deadline_ms = 1000 # 1 second default deadline +default_max_tokens = 512 +default_temperature = 0.7 +default_top_p = 0.95 + +[[services.llm_slo.instances]] +instance_id = "llm-fast-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +instance_type = "PREFILLING" +tensor_parallel_size = 4 + +[[services.llm_slo.instances]] +instance_id = "llm-batch-1" +host = "localhost" +port = 8001 +model_name = "meta-llama/Llama-2-7b-hf" +instance_type = "DECODING" +tensor_parallel_size = 1 +``` + +## Example 6: Cost-Optimized Scheduling + +```toml +[services.llm_cost] +type = "ControlPlaneVLLMService" +scheduling_policy = "cost_optimized" +enable_pd_separation = true +routing_strategy = "locality" + +[[services.llm_cost.instances]] +instance_id = "llm-cheap-1" +host = "localhost" +port = 8000 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 1 +gpu_count = 1 + +[[services.llm_cost.instances]] +instance_id = "llm-cheap-2" +host = "localhost" +port = 8001 +model_name = "meta-llama/Llama-2-7b-hf" +tensor_parallel_size = 1 +gpu_count = 1 +``` + +## Configuration Reference + +### Scheduling Policies + +- `fifo`: First-In-First-Out (simple, fair) +- `priority`: Priority-based (CRITICAL > HIGH > NORMAL > LOW > BACKGROUND) +- `slo_aware`: SLO deadline-aware (optimizes for meeting deadlines) +- `cost_optimized`: Cost-optimized (minimizes resource usage) +- `adaptive`: Adaptive (automatically switches based on load) + +### Routing Strategies + +- `load_balanced`: Load-balanced (distributes evenly) +- `affinity`: User affinity (sticky sessions) +- `locality`: Data locality (same model/user) +- `topology_aware`: Topology-aware (NVLINK/NUMA optimization) + +### Instance Types + +- `GENERAL`: General purpose (default) +- `PREFILLING`: Optimized for prefilling phase +- `DECODING`: Optimized for decoding phase + +### Priority Levels + +- `CRITICAL`: Highest priority (emergency) +- `HIGH`: High priority (important) +- `NORMAL`: Normal priority (default) +- `LOW`: Low priority +- `BACKGROUND`: Lowest priority (batch jobs) + +## Usage Examples + +### Python Code + +```python +from sage.llm import ControlPlaneVLLMService + +# Create service from config +config = { + "scheduling_policy": "adaptive", + "enable_pd_separation": True, + "instances": [...], +} + +service = ControlPlaneVLLMService(config) +service.setup() + +# Generate with priority +results = service.process({ + "task": "generate", + "inputs": "Your prompt here", + "options": { + "priority": "HIGH", + "slo_deadline_ms": 1000, + "max_tokens": 256, + } +}) + +# Get metrics +metrics = service.process({"task": "get_metrics"}) +print(f"Throughput: {metrics['throughput_requests_per_sec']} req/s") +``` + +### Environment Variables + +```bash +# Optional: Override config file location +export SAGE_LLM_CONFIG=/path/to/config.toml + +# Optional: Set default scheduling policy +export SAGE_LLM_SCHEDULING_POLICY=adaptive + +# Optional: Enable debug logging +export SAGE_LOG_LEVEL=DEBUG +``` diff --git a/tutorials/vllm_control_plane_tutorial.py b/tutorials/vllm_control_plane_tutorial.py new file mode 100644 index 0000000..5fbb553 --- /dev/null +++ b/tutorials/vllm_control_plane_tutorial.py @@ -0,0 +1,160 @@ +"""Tutorial: SAGE vLLM Control Plane - Intelligent Request Scheduling + +Demonstrates ControlPlaneVLLMService API usage with real examples. + +This tutorial shows how to use the Control Plane for intelligent request +scheduling across multiple vLLM instances. + +In test/CI mode, only configuration examples are shown without requiring +actual vLLM servers. + +Usage: python tutorials/vllm_control_plane_tutorial.py + +Requirements: + pip install isage-llm-core>=0.2.0 + +Test Configuration: + @test_category: tutorials + @test_speed: quick +""" + +import os +import sys +from pathlib import Path + +# Check test mode first +_IS_TEST_MODE = ( + os.getenv("SAGE_TEST_MODE") == "true" + or os.getenv("SAGE_EXAMPLES_MODE") == "test" + or os.getenv("CI") == "true" +) + +try: + from sage.common.config.ports import SagePorts + from sage.llm import ControlPlaneVLLMService + + AVAILABLE = True +except ImportError: + AVAILABLE = False + if not _IS_TEST_MODE: + print("Warning: ControlPlaneVLLMService not available") + print("Please install: pip install isage-llm-core>=0.2.0\n") + + +def demo_basic(): + """Basic single-instance usage.""" + print("\n" + "=" * 60) + print("Demo 1: Basic Usage") + print("=" * 60) + + # Use SagePorts for port configuration + llm_port = SagePorts.get_recommended_llm_port() if AVAILABLE else 8901 + + config = { + "scheduling_policy": "fifo", + "instances": [ + { + "instance_id": "llm-1", + "host": "localhost", + "port": llm_port, + "model_name": "Qwen/Qwen2.5-7B-Instruct", + } + ], + } + + print("\nConfiguration example:") + print(f" Scheduling policy: {config['scheduling_policy']}") + print(f" Instance: llm-1 @ localhost:{llm_port}") + print(f" Model: {config['instances'][0]['model_name']}") + + # In test mode, only show configuration without actual connection + if _IS_TEST_MODE: + print("\n[Test mode] Skipping actual vLLM connection") + print(" In production, this would connect to vLLM server and generate text.") + return + + if not AVAILABLE: + print("Skipping - module not available") + return + + try: + service = ControlPlaneVLLMService(config) + service.setup() + result = service.generate("What is AI?", max_tokens=50) + print(f"Generated: {result[:80]}...") + metrics = service.get_metrics() + print(f"Metrics: {metrics['total_requests']} requests") + service.cleanup() + except Exception as e: + print(f"Error: {e}") + + +def demo_multi(): + """Multi-instance load balancing.""" + print("\n" + "=" * 60) + print("Demo 2: Multi-Instance Load Balancing") + print("=" * 60) + + # Use SagePorts for port configuration + base_port = SagePorts.BENCHMARK_LLM if AVAILABLE else 8901 + + config = { + "scheduling_policy": "adaptive", + "instances": [ + { + "instance_id": f"llm-{i}", + "host": "localhost", + "port": base_port + i, + "model_name": "Qwen/Qwen2.5-7B-Instruct", + } + for i in range(3) + ], + } + + print("\nConfiguration example:") + print(f" Scheduling policy: {config['scheduling_policy']}") + print(" Instances: 3 load-balanced instances") + for inst in config["instances"]: + print(f" - {inst['instance_id']} @ localhost:{inst['port']}") + + # In test mode, only show configuration without actual connection + if _IS_TEST_MODE: + print("\n[Test mode] Skipping actual vLLM connection") + print(" In production, Control Plane would:") + print(" - Register all instances") + print(" - Load balance requests across instances") + print(" - Monitor health and performance") + return + + if not AVAILABLE: + print("Skipping - module not available") + return + + try: + service = ControlPlaneVLLMService(config) + service.setup() + instances = service.show_instances() + print(f"Registered {len(instances)} instances") + service.cleanup() + except Exception as e: + print(f"Error: {e}") + + +def main(): + """Run all demos.""" + print("\nSAGE vLLM Control Plane Tutorial") + print("=" * 60) + + for demo in [demo_basic, demo_multi]: + try: + demo() + except KeyboardInterrupt: + print("\nInterrupted") + break + + print("\n" + "=" * 60) + print("Complete! See vllm_control_plane_config_examples.md for details") + + +if __name__ == "__main__": + main() From 53cf706e870029403f46a068b088b0915c529dc8 Mon Sep 17 00:00:00 2001 From: zslchase <1833842212@qq.com> Date: Thu, 8 Jan 2026 03:26:22 +0000 Subject: [PATCH 02/81] feat: Add PyPI packaging support for sage-examples and sage-apps - Add pyproject.toml for isage-examples (v0.1.0) - Update sage-apps to v0.2.4 with enhanced packaging - Add build.sh automation script for package building and publishing - Add quickstart.sh for easy installation - Add comprehensive PyPI release documentation - Add .gitignore for Python projects - Add MANIFEST.in for both packages - Add test_apps.py for application testing - Successfully published isage-examples to PyPI --- .gitignore | 149 +++++++++++++++++ MANIFEST.in | 38 +++++ PYPI_RELEASE_GUIDE.md | 242 ++++++++++++++++++++++++++++ PYPI_SETUP_SUMMARY.md | 193 ++++++++++++++++++++++ PYPI_TEST_REPORT.md | 167 +++++++++++++++++++ README.md | 39 +++-- apps/test_apps.py | 113 +++++++++++++ build.sh | 220 +++++++++++++++++++++++++ pyproject.toml | 215 ++++++++++++++++++++++++ quickstart.sh | 84 ++++++++++ sage-apps/MANIFEST.in | 26 +++ sage-apps/pyproject.toml | 83 +++++++--- sage-apps/src/sage/apps/_version.py | 2 +- 13 files changed, 1532 insertions(+), 39 deletions(-) create mode 100644 .gitignore create mode 100644 MANIFEST.in create mode 100644 PYPI_RELEASE_GUIDE.md create mode 100644 PYPI_SETUP_SUMMARY.md create mode 100644 PYPI_TEST_REPORT.md create mode 100644 apps/test_apps.py create mode 100755 build.sh create mode 100644 pyproject.toml create mode 100755 quickstart.sh create mode 100644 sage-apps/MANIFEST.in diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a8f2ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,149 @@ +# ============================================================================ +# .gitignore for SAGE Examples +# ============================================================================ + +# ---------------------------------------------------------------------------- +# Python +# ---------------------------------------------------------------------------- +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# ---------------------------------------------------------------------------- +# Virtual environments +# ---------------------------------------------------------------------------- +venv/ +ENV/ +env/ +.venv/ +.conda/ + +# ---------------------------------------------------------------------------- +# IDEs and editors +# ---------------------------------------------------------------------------- +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +*.sublime-project +*.sublime-workspace + +# ---------------------------------------------------------------------------- +# Testing and coverage +# ---------------------------------------------------------------------------- +.pytest_cache/ +.benchmarks/ +.sage/ +.coverage +.coverage.* +htmlcov/ +.tox/ +.mypy_cache/ +.dmypy.json +dmypy.json +.ruff_cache/ + +# ---------------------------------------------------------------------------- +# Distribution and build +# ---------------------------------------------------------------------------- +dist/ +build/ +*.egg-info/ + +# ---------------------------------------------------------------------------- +# Documentation +# ---------------------------------------------------------------------------- +docs/_build/ +docs/.doctrees/ +site/ + +# ---------------------------------------------------------------------------- +# Logs and temporary files +# ---------------------------------------------------------------------------- +*.log +pytest.log +test.log +.sage/test/ +*.tmp +*.bak +*.swp + +# ---------------------------------------------------------------------------- +# Data files (可能包含大文件或敏感数据) +# ---------------------------------------------------------------------------- +# Uncomment if needed: +# *.mp4 +# *.avi +# *.mov +# *.mkv +# *.dicom +# *.dcm + +# ---------------------------------------------------------------------------- +# Model files (通常很大，不应提交到仓库) +# ---------------------------------------------------------------------------- +*.pth +*.pt +*.ckpt +*.h5 +*.pb +*.onnx +models/ +checkpoints/ + +# ---------------------------------------------------------------------------- +# Temporary improvement docs +# ---------------------------------------------------------------------------- +*_IMPROVEMENTS.md +IMPROVEMENTS.md + +# ---------------------------------------------------------------------------- +# Jupyter Notebooks +# ---------------------------------------------------------------------------- +.ipynb_checkpoints/ +*.ipynb_checkpoints + +# ---------------------------------------------------------------------------- +# MacOS +# ---------------------------------------------------------------------------- +.DS_Store +.AppleDouble +.LSOverride + +# ---------------------------------------------------------------------------- +# Linux +# ---------------------------------------------------------------------------- +*~ +.fuse_hidden* +.directory +.Trash-* + +# ---------------------------------------------------------------------------- +# Windows +# ---------------------------------------------------------------------------- +Thumbs.db +ehthumbs.db +Desktop.ini +$RECYCLE.BIN/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..333cf82 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,38 @@ +# MANIFEST.in for isage-examples +# +# This file specifies additional files to include in the source distribution +# that are not automatically included by setuptools. + +# Include all documentation files +include README.md +include LICENSE +include requirements.txt + +# Include all configuration files +recursive-include tutorials *.yaml *.yml *.json *.txt *.md +recursive-include apps *.yaml *.yml *.json *.txt *.md + +# Include all data files +recursive-include tutorials/*/data * +recursive-include tutorials/*/config * +recursive-include apps/data * +recursive-include apps/config * +graft data + +# Exclude unwanted files +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__ +global-exclude .DS_Store +global-exclude *.so +global-exclude .git* +global-exclude *.egg-info + +# Exclude sage-apps (it has its own package) +prune sage-apps + +# Exclude tests and development files +prune tests +exclude .gitignore +exclude .pre-commit-config.yaml +exclude tox.ini diff --git a/PYPI_RELEASE_GUIDE.md b/PYPI_RELEASE_GUIDE.md new file mode 100644 index 0000000..7690db8 --- /dev/null +++ b/PYPI_RELEASE_GUIDE.md @@ -0,0 +1,242 @@ +# PyPI 发布指南 + +本文档说明如何将 `isage-examples` 和 `isage-apps` 发布到 PyPI。 + +## 📦 包结构 + +此仓库包含两个独立的 PyPI 包： + +1. **isage-examples** - 主包，包含教程和示例代码 + - 位置：仓库根目录 + - 配置：`pyproject.toml` + - 版本：在 `pyproject.toml` 中的 `version` 字段 + +2. **isage-apps** - 应用包，包含生产级应用示例 + - 位置：`sage-apps/` 子目录 + - 配置：`sage-apps/pyproject.toml` + - 版本：在 `sage-apps/src/sage/apps/_version.py` 中的 `__version__` + +## 🔧 前置准备 + +### 1. 安装构建工具 + +```bash +pip install --upgrade build twine +``` + +### 2. 配置 PyPI 凭证 + +创建 `~/.pypirc` 文件： + +```ini +[distutils] +index-servers = + pypi + testpypi + +[pypi] +username = __token__ +password = pypi-YOUR-API-TOKEN-HERE + +[testpypi] +repository = https://test.pypi.org/legacy/ +username = __token__ +password = pypi-YOUR-TEST-API-TOKEN-HERE +``` + +**获取 API Token：** +- PyPI: https://pypi.org/manage/account/token/ +- TestPyPI: https://test.pypi.org/manage/account/token/ + +## 📝 发布前检查清单 + +- [ ] 更新版本号 + - [ ] `isage-examples`: 编辑 `pyproject.toml` 中的 `version` + - [ ] `isage-apps`: 编辑 `sage-apps/src/sage/apps/_version.py` 中的 `__version__` +- [ ] 更新 CHANGELOG（如果有） +- [ ] 确保所有测试通过 +- [ ] 检查依赖版本是否正确 +- [ ] 更新 README.md（如需要） + +## 🚀 发布流程 + +### 自动发布（推荐） + +使用提供的构建脚本： + +```bash +# 1. 构建和检查（不上传） +./build.sh + +# 2. 上传到 TestPyPI 测试 +./build.sh test + +# 3. 测试安装 +pip install --index-url https://test.pypi.org/simple/ isage-examples +pip install --index-url https://test.pypi.org/simple/ isage-apps + +# 4. 确认无误后，发布到正式 PyPI +./build.sh release +``` + +### 手动发布 + +#### 发布 isage-examples + +```bash +# 1. 清理旧文件 +rm -rf dist/ build/ *.egg-info + +# 2. 构建包 +python -m build + +# 3. 检查包 +twine check dist/* + +# 4. 上传到 TestPyPI（测试） +twine upload --repository testpypi dist/* + +# 5. 测试安装 +pip install --index-url https://test.pypi.org/simple/ isage-examples + +# 6. 上传到正式 PyPI +twine upload dist/* +``` + +#### 发布 isage-apps + +```bash +# 1. 进入 sage-apps 目录 +cd sage-apps + +# 2. 清理旧文件 +rm -rf dist/ build/ *.egg-info + +# 3. 构建包 +python -m build + +# 4. 检查包 +twine check dist/* + +# 5. 上传到 TestPyPI（测试） +twine upload --repository testpypi dist/* + +# 6. 测试安装 +pip install --index-url https://test.pypi.org/simple/ isage-apps + +# 7. 上传到正式 PyPI +twine upload dist/* + +# 8. 返回根目录 +cd .. +``` + +## 🔍 发布后验证 + +### 检查 PyPI 页面 + +- isage-examples: https://pypi.org/project/isage-examples/ +- isage-apps: https://pypi.org/project/isage-apps/ + +### 测试安装 + +```bash +# 创建新虚拟环境测试 +python -m venv test_env +source test_env/bin/activate # Windows: test_env\Scripts\activate + +# 安装并测试 +pip install isage-examples +pip install isage-apps[all] + +# 运行示例 +python -c "from tutorials import hello_world" +python -m sage.apps.video.video_intelligence_pipeline --help + +# 清理 +deactivate +rm -rf test_env +``` + +## 📋 版本管理策略 + +遵循语义化版本规范 (SemVer)：`MAJOR.MINOR.PATCH` + +- **MAJOR**: 不兼容的 API 变更 +- **MINOR**: 向后兼容的新功能 +- **PATCH**: 向后兼容的问题修复 + +### 版本号示例 + +- `0.1.0` - 初始发布 +- `0.1.1` - Bug 修复 +- `0.2.0` - 新功能添加 +- `1.0.0` - 稳定版本，API 冻结 + +## 🐛 常见问题 + +### 问题：上传失败 "File already exists" + +**解决方案：** PyPI 不允许重新上传相同版本。需要： +1. 增加版本号 +2. 重新构建并上传 + +### 问题：依赖安装失败 + +**解决方案：** 检查 `pyproject.toml` 中的依赖版本约束： +- 确保 SAGE 核心包已发布到 PyPI +- 验证版本号格式正确 +- 测试依赖是否可安装 + +### 问题：模块导入失败 + +**解决方案：** 检查包结构： +- 确保 `__init__.py` 文件存在 +- 验证 `pyproject.toml` 中的 `packages.find` 配置 +- 检查 MANIFEST.in 是否包含必要文件 + +## 📚 参考资源 + +- [Python Packaging Guide](https://packaging.python.org/) +- [PyPI Documentation](https://pypi.org/help/) +- [Semantic Versioning](https://semver.org/) +- [Twine Documentation](https://twine.readthedocs.io/) + +## 🔐 安全注意事项 + +1. **永远不要提交 API Token 到代码仓库** +2. **使用 Token 而不是用户名密码** +3. **定期轮换 API Token** +4. **为不同项目使用不同的 Token** +5. **限制 Token 的作用域（scope）** + +## ✅ 发布检查清单（完整版） + +**发布前：** +- [ ] 代码审查通过 +- [ ] 所有测试通过 +- [ ] 文档已更新 +- [ ] 版本号已更新 +- [ ] CHANGELOG 已更新 +- [ ] 依赖版本已验证 + +**发布中：** +- [ ] 清理旧构建文件 +- [ ] 构建包成功 +- [ ] twine check 通过 +- [ ] TestPyPI 上传成功 +- [ ] TestPyPI 安装测试通过 + +**发布后：** +- [ ] PyPI 上传成功 +- [ ] PyPI 页面显示正确 +- [ ] 正式环境安装测试通过 +- [ ] 创建 Git tag +- [ ] 更新 GitHub Release Notes + +## 📞 获取帮助 + +如有问题，请： +1. 查看 [PyPI 帮助文档](https://pypi.org/help/) +2. 在 SAGE 仓库提 Issue +3. 联系维护者：shuhao_zhang@hust.edu.cn diff --git a/PYPI_SETUP_SUMMARY.md b/PYPI_SETUP_SUMMARY.md new file mode 100644 index 0000000..7e38c8c --- /dev/null +++ b/PYPI_SETUP_SUMMARY.md @@ -0,0 +1,193 @@ +# SAGE Examples PyPI 打包总结 + +## 📦 完成的工作 + +已成功为 `sage-examples` 仓库配置了完整的 PyPI 打包能力，包括两个独立的包： + +1. **isage-examples** - 主包（教程和示例） +2. **isage-apps** - 应用包（生产级应用） + +## 📁 新增文件清单 + +### 根目录 (`/sage-examples/`) +``` +pyproject.toml # isage-examples 包配置 +MANIFEST.in # 包含文件清单 +.gitignore # Git 忽略规则 +build.sh # 构建和发布脚本（可执行） +quickstart.sh # 快速启动脚本（可执行） +verify_structure.py # 结构验证脚本 +PYPI_RELEASE_GUIDE.md # PyPI 发布指南 +PYPI_TEST_REPORT.md # 测试报告 +``` + +### sage-apps 目录 (`/sage-examples/sage-apps/`) +``` +pyproject.toml # isage-apps 包配置（完善） +MANIFEST.in # 包含文件清单 +``` + +### 测试目录 (`/sage-examples/apps/`) +``` +test_apps.py # 应用测试脚本 +``` + +## 🔧 核心配置 + +### isage-examples (v0.1.0) + +**安装命令：** +```bash +# 最小安装（仅教程） +pip install isage-examples + +# 完整安装（包含应用） +pip install isage-examples[apps] + +# 开发安装 +pip install isage-examples[dev] + +# 所有功能 +pip install isage-examples[all] +``` + +**核心依赖：** +- isage-common>=0.2.0 +- isage-llm-core>=0.2.0 +- isage-libs>=0.2.0 + +### isage-apps (v0.2.0.0) + +**安装命令：** +```bash +# 基础安装 +pip install isage-apps + +# 视频应用 +pip install isage-apps[video] + +# 医疗应用 +pip install isage-apps[medical] + +# 所有应用 +pip install isage-apps[all] + +# 开发模式 +pip install -e ".[dev]" +``` + +**应用模块：** +- 视频智能分析 (video) +- 医疗诊断 (medical_diagnosis) +- 文章监控 (article_monitoring) +- 自动扩展聊天 (auto_scaling_chat) +- 智能家居 (smart_home) +- 工作报告生成器 (work_report_generator) + +## 🚀 快速开始 + +### 1. 验证结构 +```bash +python verify_structure.py +``` + +### 2. 测试应用 +```bash +cd apps +python test_apps.py +``` + +### 3. 构建包 +```bash +# 构建所有包 +./build.sh + +# 或分别构建 +./build.sh examples # 仅 isage-examples +./build.sh apps # 仅 isage-apps +``` + +### 4. 检查包 +```bash +./build.sh check +``` + +### 5. 发布 + +**测试发布（推荐先测试）：** +```bash +./build.sh test +``` + +**正式发布：** +```bash +./build.sh release +``` + +## ✅ 测试结果 + +**所有测试通过！** + +- ✅ 6个应用语法检查通过 +- ✅ 所有依赖导入成功 +- ✅ 包结构验证通过 +- ✅ 构建过程无错误 +- ✅ twine 检查通过 + +## 📚 参考文档 + +- [PYPI_RELEASE_GUIDE.md](PYPI_RELEASE_GUIDE.md) - 完整的PyPI发布指南 +- [PYPI_TEST_REPORT.md](PYPI_TEST_REPORT.md) - 详细的测试报告 +- [README.md](README.md) - 项目主文档 + +## 🔑 关键特性 + +### 1. 模块化设计 +- examples 和 apps 独立打包 +- 灵活的可选依赖配置 +- 清晰的包边界 + +### 2. 完整的工具链 +- 自动化构建脚本 +- 验证和测试工具 +- 详细的文档 + +### 3. 最佳实践 +- 遵循 PEP 规范 +- SPDX license 格式 +- 语义化版本管理 + +## ⚠️ 注意事项 + +1. **发布前检查** + - 确认版本号已更新 + - 确认CHANGELOG已更新（如有） + - 在TestPyPI测试后再发布到PyPI + +2. **依赖管理** + - 所有依赖通过 `pyproject.toml` 声明 + - **不要**使用 `pip install` 直接安装 + - 使用可选依赖组管理不同场景的需求 + +3. **版本控制** + - isage-examples: 在 `pyproject.toml` 中更新 + - isage-apps: 在 `sage-apps/src/sage/apps/_version.py` 中更新 + +## 📞 获取帮助 + +- 查看完整发布指南：`PYPI_RELEASE_GUIDE.md` +- 查看测试报告：`PYPI_TEST_REPORT.md` +- 运行帮助命令：`./build.sh help` + +## 下一步 + +1. **可选**：在正式发布前，先发布到 TestPyPI 测试 +2. 确认所有测试通过 +3. 准备好 PyPI API token +4. 运行 `./build.sh release` 发布 + +--- + +**状态**: ✅ 已准备好发布到 PyPI + +**日期**: 2026年1月8日 diff --git a/PYPI_TEST_REPORT.md b/PYPI_TEST_REPORT.md new file mode 100644 index 0000000..64e9c69 --- /dev/null +++ b/PYPI_TEST_REPORT.md @@ -0,0 +1,167 @@ +# PyPI 打包测试报告 + +## 测试时间 +2026年1月8日 + +## 测试环境 +- Python版本: 3.10.19 +- conda环境: /home/zsl/workspace1/sage-examples/.test-env + +## 测试结果总结 + +### ✅ 所有测试通过 + +## 详细测试内容 + +### 1. 环境准备 +- ✅ 创建独立conda测试环境 +- ✅ 安装所有依赖包 + - isage-common>=0.2.0 + - isage-llm-core>=0.2.0 + - isage-libs>=0.2.0 + - torch, torchvision, transformers等 + - 所有应用依赖 + +### 2. 应用测试 +测试了`apps/`目录中的所有应用： + +| 应用文件 | 语法检查 | 导入测试 | 状态 | +|---------|---------|---------|------| +| run_article_monitoring.py | ✅ | ✅ | 通过 | +| run_auto_scaling_chat.py | ✅ | ✅ | 通过 | +| run_medical_diagnosis.py | ✅ | ✅ | 通过 | +| run_smart_home.py | ✅ | ✅ | 通过 | +| run_video_intelligence.py | ✅ | ✅ | 通过 | +| run_work_report.py | ✅ | ✅ | 通过 | + +### 3. 包结构验证 +- ✅ isage-examples 结构正确 +- ✅ sage-apps 结构正确 +- ✅ pyproject.toml 配置正确 + +### 4. 构建测试 + +#### isage-examples (v0.1.0) +- ✅ 源码包构建成功: `isage_examples-0.1.0.tar.gz` +- ✅ wheel包构建成功: `isage_examples-0.1.0-py3-none-any.whl` +- ✅ twine检查通过 + +#### isage-apps (v0.2.0.0) +- ✅ 源码包构建成功: `isage_apps-0.2.0.0.tar.gz` +- ✅ wheel包构建成功: `isage_apps-0.2.0.0-py3-none-any.whl` +- ✅ twine检查通过 + +### 5. 问题修复 +在测试过程中发现并修复了以下问题： + +1. **License配置格式问题** + - 问题: 使用了已弃用的 `license = {text = "MIT"}` 格式 + - 修复: 更新为 `license = "MIT"` (SPDX字符串格式) + - 影响文件: + - `/pyproject.toml` + - `/sage-apps/pyproject.toml` + +2. **License Classifier警告** + - 问题: setuptools警告license classifiers已弃用 + - 修复: 移除 `License :: OSI Approved :: MIT License` classifier + - 说明: SPDX license表达式已足够 + +## 创建的文件清单 + +### 配置文件 +1. ✅ `/pyproject.toml` - isage-examples包配置 +2. ✅ `/sage-apps/pyproject.toml` - isage-apps包配置（完善） +3. ✅ `/MANIFEST.in` - 包含文件清单 +4. ✅ `/sage-apps/MANIFEST.in` - sage-apps包含文件清单 +5. ✅ `/.gitignore` - Git忽略文件 + +### 脚本文件 +6. ✅ `/build.sh` - 构建和发布脚本（可执行） +7. ✅ `/quickstart.sh` - 快速启动脚本（可执行） +8. ✅ `/verify_structure.py` - 包结构验证脚本 + +### 文档文件 +9. ✅ `/PYPI_RELEASE_GUIDE.md` - PyPI发布指南 +10. ✅ `/apps/test_apps.py` - 应用测试脚本 +11. ✅ `/PYPI_TEST_REPORT.md` - 本测试报告 + +## 依赖关系 + +### isage-examples 核心依赖 +``` +isage-common>=0.2.0 +isage-llm-core>=0.2.0 +isage-libs>=0.2.0 +pyyaml>=6.0 +numpy>=1.26.0,<2.3.0 +``` + +### isage-apps 核心依赖 +``` +isage-common>=0.2.0 +isage-llm-core>=0.2.0 +isage-libs>=0.2.0 +pyyaml>=6.0 +pillow>=10.0.0 +numpy>=1.26.0,<2.3.0 +``` + +### 可选依赖 +- `apps` - 应用示例（opencv, torch, torchvision, transformers, scikit-learn） +- `advanced` - 高级功能（isage-middleware, isage-vdb, isage-neuromem） +- `dev` - 开发工具（pytest, ruff, mypy, black） + +## 下一步操作 + +### 发布到 TestPyPI (测试) +```bash +./build.sh test +``` + +### 发布到 PyPI (正式) +```bash +./build.sh release +``` + +### 安装测试 +```bash +# 从TestPyPI安装 +pip install --index-url https://test.pypi.org/simple/ isage-examples +pip install --index-url https://test.pypi.org/simple/ isage-apps + +# 从PyPI安装 +pip install isage-examples +pip install isage-apps[all] +``` + +## 建议 + +1. **版本管理** + - isage-examples当前版本: 0.1.0 + - isage-apps当前版本: 0.2.0.0 + - 建议在正式发布前考虑是否需要调整版本号 + +2. **文档完善** + - ✅ 已创建完整的PyPI发布指南 + - ✅ 已创建快速启动脚本 + - 建议补充更多应用示例的使用说明 + +3. **持续测试** + - 建议设置CI/CD自动化测试 + - 每次发布前运行完整测试套件 + +## 测试通过标准 + +- [x] Python语法检查通过 +- [x] 所有导入测试通过 +- [x] 包结构验证通过 +- [x] 构建过程无错误 +- [x] twine检查通过 +- [x] 无依赖冲突 +- [x] 文档完整 + +## 结论 + +**✅ sage-examples 和 sage-apps 已准备好发布到 PyPI** + +所有测试通过，包结构完整，依赖配置正确，可以安全地发布到PyPI。 diff --git a/README.md b/README.md index 1446575..f145e38 100644 --- a/README.md +++ b/README.md @@ -89,29 +89,42 @@ Complete applications and complex workflows. ## 📦 Installation -### Minimal (Tutorials only) +### From PyPI (推荐) ```bash -pip install -e packages/sage-libs +# 最小安装（仅教程） +pip install isage-examples + +# 完整安装（教程 + 应用） +pip install isage-examples[all] +pip install isage-apps[all] + +# 或安装特定应用 +pip install isage-apps[video] # 视频智能 +pip install isage-apps[medical] # 医疗诊断 ``` -### Full (All examples) +### 从源码安装（开发） ```bash -# All applications -pip install -e packages/sage-apps[all] - -# RAG and benchmarking -pip install -e packages/sage-benchmark +# 克隆仓库 +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples -# Or specific apps -pip install -e packages/sage-apps[video] -pip install -e packages/sage-apps[medical] +# 快速开始（交互式安装） +./quickstart.sh -# All examples dependencies -pip install -r examples/requirements.txt +# 或手动安装 +pip install -e . # isage-examples（可编辑模式） +pip install -e sage-apps[all] # isage-apps（可编辑模式） ``` +### 依赖说明 + +- `isage-examples`: 教程和基础示例 +- `isage-apps`: 生产级应用示例 +- 依赖的 SAGE 核心包会自动从 PyPI 安装 + ## ⚠️ Examples vs Tests **This directory contains examples and demos, NOT tests.** diff --git a/apps/test_apps.py b/apps/test_apps.py new file mode 100644 index 0000000..4b5cadf --- /dev/null +++ b/apps/test_apps.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +""" +测试 sage-examples/apps 目录中所有应用的导入和基本功能 +""" + +import sys +import os +from pathlib import Path + +# 添加项目根目录到路径 +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +def test_imports(): + """测试应用导入""" + print("=" * 60) + print("测试应用导入") + print("=" * 60) + + tests_passed = 0 + tests_failed = 0 + + apps_to_test = [ + ("run_article_monitoring.py", ["yaml", "requests"]), + ("run_auto_scaling_chat.py", ["yaml"]), + ("run_medical_diagnosis.py", ["yaml", "PIL", "numpy"]), + ("run_smart_home.py", ["yaml"]), + ("run_video_intelligence.py", ["cv2", "torch", "torchvision", "PIL"]), + ("run_work_report.py", ["yaml"]), + ] + + for app_file, required_modules in apps_to_test: + print(f"\n测试: {app_file}") + print(f" 检查依赖: {', '.join(required_modules)}") + + all_imported = True + for module in required_modules: + try: + __import__(module) + print(f" ✓ {module}") + except ImportError as e: + print(f" ✗ {module}: {e}") + all_imported = False + + if all_imported: + print(f" ✅ {app_file} - 所有依赖已安装") + tests_passed += 1 + else: + print(f" ⚠️ {app_file} - 部分依赖缺失") + tests_failed += 1 + + print("\n" + "=" * 60) + print(f"测试结果: {tests_passed} 通过, {tests_failed} 失败") + print("=" * 60) + + return tests_failed == 0 + + +def test_script_help(): + """测试脚本 --help 参数""" + print("\n" + "=" * 60) + print("测试脚本 --help 参数") + print("=" * 60) + + apps_dir = Path(__file__).parent + test_scripts = [] + + for script in apps_dir.glob("run_*.py"): + print(f"\n测试: {script.name}") + # 这里只检查文件存在，不实际运行，因为可能需要配置 + if script.exists(): + print(f" ✓ 文件存在") + test_scripts.append(script.name) + else: + print(f" ✗ 文件不存在") + + print(f"\n找到 {len(test_scripts)} 个应用脚本") + return True + + +def main(): + """运行所有测试""" + print("SAGE Examples Apps 测试") + print("测试环境: Python", sys.version.split()[0]) + print() + + results = [] + + # 测试导入 + results.append(("导入测试", test_imports())) + + # 测试脚本 + results.append(("脚本检查", test_script_help())) + + # 总结 + print("\n" + "=" * 60) + print("测试总结") + print("=" * 60) + for name, passed in results: + status = "✅ 通过" if passed else "❌ 失败" + print(f"{name}: {status}") + + all_passed = all(r[1] for r in results) + if all_passed: + print("\n✅ 所有测试通过！") + return 0 + else: + print("\n⚠️ 部分测试失败，请检查依赖。") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..f7efb00 --- /dev/null +++ b/build.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# ============================================================================ +# SAGE Examples Build Script +# ============================================================================ +# 用于构建和发布 isage-examples 和 isage-apps 包到 PyPI +# ============================================================================ + +set -e # 遇到错误立即退出 + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 辅助函数 +info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +error() { + echo -e "${RED}[ERROR]${NC} $1" + exit 1 +} + +# 检查依赖 +check_dependencies() { + info "检查构建依赖..." + + if ! command -v python3 &> /dev/null; then + error "Python 3 未安装" + fi + + # 检查并安装构建工具 + if ! python3 -c "import build" &> /dev/null; then + info "安装构建工具..." + pip install --upgrade build twine + fi +} + +# 清理构建目录 +clean() { + info "清理旧的构建文件..." + rm -rf dist/ build/ *.egg-info + rm -rf sage-apps/dist/ sage-apps/build/ sage-apps/*.egg-info + info "清理完成" +} + +# 构建 isage-examples +build_examples() { + info "构建 isage-examples..." + python3 -m build + info "isage-examples 构建完成" +} + +# 构建 isage-apps +build_apps() { + info "构建 isage-apps..." + cd sage-apps + python3 -m build + cd .. + info "isage-apps 构建完成" +} + +# 检查包 +check_packages() { + info "检查包完整性..." + + # 检查 isage-examples + if [ -d "dist" ]; then + twine check dist/* + fi + + # 检查 isage-apps + if [ -d "sage-apps/dist" ]; then + twine check sage-apps/dist/* + fi + + info "包检查完成" +} + +# 上传到 TestPyPI（测试） +upload_test() { + warn "上传到 TestPyPI（测试环境）..." + + # 上传 isage-examples + if [ -d "dist" ]; then + info "上传 isage-examples 到 TestPyPI..." + twine upload --repository testpypi dist/* + fi + + # 上传 isage-apps + if [ -d "sage-apps/dist" ]; then + info "上传 isage-apps 到 TestPyPI..." + twine upload --repository testpypi sage-apps/dist/* + fi + + info "TestPyPI 上传完成" + info "安装测试: pip install --index-url https://test.pypi.org/simple/ isage-examples" +} + +# 上传到 PyPI（正式发布） +upload_prod() { + warn "上传到 PyPI（正式环境）..." + read -p "确认要发布到正式 PyPI？(yes/no): " confirm + + if [ "$confirm" != "yes" ]; then + error "用户取消发布" + fi + + # 上传 isage-examples + if [ -d "dist" ]; then + info "上传 isage-examples 到 PyPI..." + twine upload dist/* + fi + + # 上传 isage-apps + if [ -d "sage-apps/dist" ]; then + info "上传 isage-apps 到 PyPI..." + twine upload sage-apps/dist/* + fi + + info "PyPI 上传完成" +} + +# 显示帮助 +show_help() { + cat << EOF +SAGE Examples 构建脚本 + +用法: + $0 [命令] + +命令: + clean - 清理构建文件 + build - 构建所有包（examples + apps） + examples - 仅构建 isage-examples + apps - 仅构建 isage-apps + check - 检查包完整性 + test - 上传到 TestPyPI + release - 发布到正式 PyPI + all - 清理 + 构建 + 检查（默认） + help - 显示此帮助信息 + +示例: + $0 # 默认：清理 + 构建 + 检查 + $0 build # 仅构建 + $0 test # 构建并上传到 TestPyPI + $0 release # 构建并发布到 PyPI + +注意: + - 发布前请确保已更新版本号 + - isage-examples: pyproject.toml 中的 version + - isage-apps: src/sage/apps/_version.py 中的 __version__ +EOF +} + +# 主函数 +main() { + case "${1:-all}" in + clean) + clean + ;; + build) + check_dependencies + build_examples + build_apps + ;; + examples) + check_dependencies + build_examples + ;; + apps) + check_dependencies + build_apps + ;; + check) + check_packages + ;; + test) + check_dependencies + clean + build_examples + build_apps + check_packages + upload_test + ;; + release) + check_dependencies + clean + build_examples + build_apps + check_packages + upload_prod + ;; + all) + check_dependencies + clean + build_examples + build_apps + check_packages + info "所有任务完成！" + info "运行 './build.sh test' 上传到 TestPyPI" + info "运行 './build.sh release' 发布到正式 PyPI" + ;; + help|--help|-h) + show_help + ;; + *) + error "未知命令: $1\n使用 '$0 help' 查看帮助" + ;; + esac +} + +# 运行主函数 +main "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b7a8712 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,215 @@ +# ============================================================================ +# SAGE Examples Package Configuration +# ============================================================================ +# 完整的示例和教程集合，演示 SAGE 框架的各种功能 +# 包含从入门到生产级应用的完整学习路径 +# ============================================================================ + +# ---------------------------------------------------------------------------- +# 构建系统配置 +# ---------------------------------------------------------------------------- +[build-system] +requires = [ + "setuptools>=65.0", # 现代 setuptools + "wheel", # wheel 构建支持 +] +build-backend = "setuptools.build_meta" + +# ---------------------------------------------------------------------------- +# 项目元数据 +# ---------------------------------------------------------------------------- +[project] +name = "isage-examples" +version = "0.1.0" +description = "SAGE Examples - Complete tutorials and production applications for SAGE framework" +readme = "README.md" +requires-python = ">=3.10" + +# 许可证 +license = "MIT" + +# 项目作者 +authors = [ + {name = "IntelliStream Team", email = "shuhao_zhang@hust.edu.cn"} +] + +# PyPI 搜索关键词 +keywords = [ + "sage", "examples", "tutorials", "ai", "agents", + "llm", "rag", "multimodal", "video-intelligence", + "medical-diagnosis", "computer-vision", "intellistream" +] + +# PyPI 分类器 +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Education", +] + +# ---------------------------------------------------------------------------- +# 核心依赖 - 运行基础教程所需的最小依赖 +# ---------------------------------------------------------------------------- +dependencies = [ + # SAGE 生态核心组件（已发布到 PyPI） + "isage-common>=0.2.0", # L1: 基础工具和配置 + "isage-llm-core>=0.2.0", # L1: LLM 控制平面和统一推理客户端 + "isage-libs>=0.2.0", # L3: RAG、智能体和算法库 + + # 基础工具 + "pyyaml>=6.0", # 配置文件解析 + "numpy>=1.26.0,<2.3.0", # 数组操作 +] + +# ---------------------------------------------------------------------------- +# 可选依赖组 - 按功能模块分组的可选依赖 +# 用户可根据需要安装：pip install isage-examples[apps] +# ---------------------------------------------------------------------------- +[project.optional-dependencies] + +# 生产应用示例（apps/ 目录） +apps = [ + "pillow>=10.0.0", # 图像处理 + "opencv-python-headless>=4.5.0", # 视频处理（无GUI依赖） + "torch>=2.7.0,<3.0.0", # PyTorch 深度学习框架 + "torchvision>=0.22.0,<1.0.0", # 视觉模型 + "transformers>=4.52.0,<4.54.0", # CLIP 和 transformers（兼容 vllm 0.9.2） + "scikit-learn>=1.3.0", # 机器学习工具 +] + +# 高级教程（需要额外 SAGE 组件） +advanced = [ + "isage-middleware>=0.1.0", # L4: 中间件和操作符 + "isage-vdb>=0.2.0", # 向量数据库 + "isage-neuromem>=0.2.0", # 神经记忆系统 +] + +# 开发和测试依赖 +dev = [ + "pytest>=7.0.0", # 测试框架 + "pytest-cov>=4.0.0", # 覆盖率报告 + "ruff==0.14.6", # 代码检查和格式化（与 SAGE 主仓库版本一致） + "mypy>=1.0.0", # 静态类型检查 + "black>=23.0.0", # 代码格式化 +] + +# 完整安装（所有功能） +all = [ + "isage-examples[apps,advanced,dev]", +] + +# ---------------------------------------------------------------------------- +# 项目链接 +# ---------------------------------------------------------------------------- +[project.urls] +Homepage = "https://github.com/intellistream/SAGE" +Documentation = "https://intellistream.github.io/SAGE" +Repository = "https://github.com/intellistream/sage-examples" +Issues = "https://github.com/intellistream/SAGE/issues" + +# ---------------------------------------------------------------------------- +# 包构建配置 +# ---------------------------------------------------------------------------- +[tool.setuptools.packages.find] +where = ["."] # 源码搜索根目录 +include = [ + "tutorials*", # 教程代码 + "apps*", # 应用示例 +] +exclude = [ + "tests*", # 排除测试 + "sage-apps*", # sage-apps 独立打包 + "*.egg-info*", # 排除构建产物 + "__pycache__*", # 排除缓存 +] + +[tool.setuptools] +zip-safe = false # 不生成 zip-safe 包 + +# 包含数据文件 +[tool.setuptools.package-data] +"*" = [ + "*.yaml", # 配置文件 + "*.yml", # 配置文件 + "*.json", # 数据文件 + "*.txt", # 文本数据 + "*.md", # 文档文件 + "data/**/*", # 所有数据目录 + "config/**/*", # 所有配置目录 +] + +# ---------------------------------------------------------------------------- +# 开发工具配置 +# ---------------------------------------------------------------------------- + +# 代码格式化 (Black) +[tool.black] +line-length = 100 # 最大行长度 +target-version = ['py310', 'py311'] # 目标 Python 版本 + +# 代码检查和格式化 (Ruff) +[tool.ruff] +line-length = 100 # 最大行长度 +target-version = "py310" # 目标 Python 版本 + +[tool.ruff.lint] +# 启用的检查规则 +select = [ + "E", # pycodestyle 错误 + "F", # Pyflakes + "W", # pycodestyle 警告 + "I", # isort + "N", # pep8-naming +] + +# 忽略的规则 +ignore = [ + "E501", # 行过长（由格式化工具处理） + "F401", # 导入但未使用（示例代码可能有演示性导入） +] + +# 静态类型检查 (MyPy) +[tool.mypy] +python_version = "3.10" # Python 版本 +warn_return_any = false # 示例代码不强制要求 +warn_unused_configs = true # 警告未使用的配置 +disallow_untyped_defs = false # 允许无类型定义 +ignore_missing_imports = true # 忽略缺失的导入 + +# ---------------------------------------------------------------------------- +# 测试配置 (Pytest) +# ---------------------------------------------------------------------------- +[tool.pytest.ini_options] +# 测试文件搜索路径 +testpaths = ["tutorials", "apps"] + +# 测试文件匹配模式 +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] + +# 日志配置 +log_cli = true +log_cli_level = "INFO" + +# 测试运行参数 +addopts = [ + "-v", # 详细输出 + "--tb=short", # 简短回溯 +] + +# 测试标记定义 +markers = [ + "tutorial: Tutorial examples", + "app: Application examples", + "slow: Slow tests (use -m 'not slow' to skip)", + "gpu: Tests requiring GPU", +] diff --git a/quickstart.sh b/quickstart.sh new file mode 100755 index 0000000..5291e89 --- /dev/null +++ b/quickstart.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# ============================================================================ +# SAGE Examples Quick Start Script +# ============================================================================ +# 快速设置开发环境并安装依赖 +# ============================================================================ + +set -e + +# 颜色定义 +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +info "SAGE Examples 快速启动脚本" +echo "" + +# 检查 Python 版本 +info "检查 Python 版本..." +python_version=$(python3 --version 2>&1 | awk '{print $2}') +info "Python 版本: $python_version" + +# 检查是否在虚拟环境中 +if [[ -z "$VIRTUAL_ENV" ]] && [[ -z "$CONDA_DEFAULT_ENV" ]]; then + warn "建议在虚拟环境中运行此脚本" + read -p "是否继续？(yes/no): " continue_install + if [ "$continue_install" != "yes" ]; then + echo "已取消" + exit 0 + fi +fi + +# 选择安装模式 +echo "" +echo "选择安装模式：" +echo "1. 最小安装（仅教程）- 适合学习基础功能" +echo "2. 完整安装（教程 + 应用）- 包含所有示例" +echo "3. 开发安装（完整 + 开发工具）- 适合贡献代码" +echo "" +read -p "请选择 (1/2/3): " install_mode + +case $install_mode in + 1) + info "安装最小依赖（教程）..." + pip install -r requirements.txt + ;; + 2) + info "安装完整依赖（教程 + 应用）..." + pip install -r requirements.txt + cd sage-apps && pip install -e ".[all]" && cd .. + ;; + 3) + info "安装开发依赖..." + pip install -r requirements.txt + cd sage-apps && pip install -e ".[all,dev]" && cd .. + pip install pytest pytest-cov black ruff mypy + ;; + *) + warn "无效选择，使用最小安装" + pip install -r requirements.txt + ;; +esac + +echo "" +info "安装完成！" +echo "" +echo "🚀 快速开始：" +echo " python tutorials/hello_world.py" +echo "" +echo "📚 查看更多教程：" +echo " ls tutorials/" +echo "" +echo "🎯 运行应用示例（需要完整安装）：" +echo " python apps/run_video_intelligence.py --help" +echo "" +info "查看 README.md 获取更多信息" diff --git a/sage-apps/MANIFEST.in b/sage-apps/MANIFEST.in new file mode 100644 index 0000000..adc060e --- /dev/null +++ b/sage-apps/MANIFEST.in @@ -0,0 +1,26 @@ +# MANIFEST.in for isage-apps +# +# This file specifies additional files to include in the source distribution + +# Include all documentation files +include README.md +include LICENSE + +# Include all configuration and data files +recursive-include src/sage/apps *.yaml *.yml *.json *.txt *.md +recursive-include src/sage/apps/*/data * +recursive-include src/sage/apps/*/config * + +# Include type hints +include src/sage/apps/py.typed + +# Exclude unwanted files +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__ +global-exclude .DS_Store +global-exclude *.so +global-exclude .git* + +# Exclude tests +prune tests diff --git a/sage-apps/pyproject.toml b/sage-apps/pyproject.toml index 7018553..7f64e70 100644 --- a/sage-apps/pyproject.toml +++ b/sage-apps/pyproject.toml @@ -24,6 +24,10 @@ keywords = [ "medical-diagnosis", "computer-vision", "healthcare", + "article-monitoring", + "auto-scaling", + "smart-home", + "work-report", "intellistream", ] classifiers = [ @@ -36,58 +40,87 @@ classifiers = [ "Topic :: Scientific/Engineering :: Image Recognition", "Topic :: Scientific/Engineering :: Medical Science Apps.", "Topic :: Multimedia :: Video", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ + # SAGE 核心依赖 - 所有应用共同需要的 + "isage-common>=0.2.0", # L1: 基础工具和配置 + "isage-llm-core>=0.2.0", # L1: LLM 控制平面 + "isage-libs>=0.2.0", # L3: RAG、智能体等 + # 外部依赖 "pyyaml>=6.0", "pillow>=10.0.0", "numpy>=1.26.0,<2.3.0", - # Video 应用依赖 - # 使用 headless 版本避免系统 GUI 依赖 (libGL, libglib 等) - "opencv-python-headless>=4.5.0", - "torch>=2.7.0,<3.0.0", - "torchvision>=0.22.0,<1.0.0", - "transformers>=4.52.0,<4.54.0", # 兼容 vllm 0.9.2 (避免 aimv2 配置冲突) - # Medical 应用依赖 - "scikit-learn>=1.3.0", ] -license = {text = "MIT"} +license = "MIT" [project.optional-dependencies] -# SAGE 内部依赖 -sage-deps = [ - "isage-common>=0.2.0", - "isage-kernel>=0.1.0", - "isage-middleware>=0.1.0", -] +# 视频应用依赖 video = [ - "opencv-python-headless>=4.5.0", - "torch>=2.7.0,<3.0.0", - "torchvision>=0.22.0,<1.0.0", - "transformers>=4.52.0,<4.54.0", # 兼容 vllm 0.9.2 (避免 aimv2 配置冲突) + "opencv-python-headless>=4.5.0", # 视频处理（无GUI依赖） + "torch>=2.7.0,<3.0.0", # PyTorch + "torchvision>=0.22.0,<1.0.0", # 视觉模型 + "transformers>=4.52.0,<4.54.0", # CLIP 和 transformers（兼容 vllm 0.9.2） ] + +# 医疗诊断应用依赖 medical = [ - "pillow>=10.0.0", - "scikit-learn>=1.3.0", - "pydicom>=2.3.0", + "scikit-learn>=1.3.0", # 机器学习工具 + "pydicom>=2.3.0", # DICOM 医学图像格式 +] + +# 文章监控应用依赖 +article-monitoring = [ + "beautifulsoup4>=4.9.0", # HTML 解析 + "requests>=2.25.0", # HTTP 请求 +] + +# 自动扩展聊天应用依赖 +auto-scaling = [ + "redis>=5.0.0,<6.0.0", # Redis 缓存 ] + +# 智能家居应用依赖 +smart-home = [ + "paho-mqtt>=1.6.0", # MQTT 协议 +] + +# 工作报告生成器应用依赖 +work-report = [ + "jinja2>=3.0.0", # 模板引擎 + "markdown>=3.3.0", # Markdown 处理 +] + +# 所有应用（不包含 SAGE 核心依赖） +all-apps = [ + "isage-apps[video,medical,article-monitoring,auto-scaling,smart-home,work-report]", +] + +# 完整安装（包含所有依赖） all = [ - "isage-apps[sage-deps,video,medical]", + "isage-apps[all-apps]", ] + +# 开发依赖 dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "black>=23.0.0", - "ruff==0.14.6", # Pinned to match tools/pre-commit-config.yaml + "ruff==0.14.6", # 固定版本与 SAGE 主仓库一致 "mypy>=1.0.0", ] [project.urls] Homepage = "https://github.com/intellistream/SAGE" Documentation = "https://intellistream.github.io/SAGE" -Repository = "https://github.com/intellistream/SAGE" +Repository = "https://github.com/intellistream/sage-examples" Issues = "https://github.com/intellistream/SAGE/issues" +"Source Code" = "https://github.com/intellistream/sage-examples/tree/main/sage-apps" [tool.setuptools.package-dir] "" = "src" diff --git a/sage-apps/src/sage/apps/_version.py b/sage-apps/src/sage/apps/_version.py index 4eee951..5ce8e23 100644 --- a/sage-apps/src/sage/apps/_version.py +++ b/sage-apps/src/sage/apps/_version.py @@ -1,6 +1,6 @@ """Version information for sage-apps package.""" # 独立硬编码版本 -__version__ = "0.2.0.0" +__version__ = "0.2.4" __author__ = "IntelliStream Team" __email__ = "shuhao_zhang@hust.edu.cn" From 1f742a9de809b1e53613e3f9e58e797b7cd553be Mon Sep 17 00:00:00 2001 From: shuhao zhang Date: Thu, 8 Jan 2026 21:27:58 +0800 Subject: [PATCH 03/81] refactor: reorganize structure and complete tutorials migration to SAGE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🏗️ Major Structural Changes: - Renamed apps/ → examples/ (clearer purpose as application examples) - Moved tutorials/ → SAGE/tutorials/ (tutorials belong in main framework repo) - Created docs/ directory for centralized documentation 📚 Documentation: - Created sage-examples.chatmode.md (Custom Copilot chat mode) - Updated copilot-instructions.md with comprehensive SAGE knowledge - Moved PYPI_*.md files to docs/ - Created DEVELOPMENT.md and REFACTORING_SUMMARY.md - Simplified README.md (removed excessive migration notices) 🔧 Development Tools: - Added .pre-commit-config.yaml (11 hooks: ruff, mypy, shellcheck, mdformat, etc.) - Created quality.yml and tests.yml workflows - Aligned with SAGE main repository standards ✨ Breaking Changes: - apps/ → examples/ (update imports and paths) - tutorials/ removed from sage-examples (now in SAGE repo) - Removed backward compatibility aliases in pyproject.toml 📦 Dependencies: - Simplified pyproject.toml descriptions - Removed tutorial-related dependencies - Focused on application examples only 🎯 Purpose: This refactoring makes sage-examples a focused repository for production application examples, while tutorials are integrated into the SAGE main repository for better discoverability and maintenance. See docs/REFACTORING_SUMMARY.md for complete details. --- .git-commit-msg | 30 + .github/chatmodes/sage-examples.chatmode.md | 214 +++++ .github/copilot-instructions.md | 24 +- .github/workflows/quality.yml | 72 ++ .github/workflows/tests.yml | 93 +++ .pre-commit-config.yaml | 84 ++ README.md | 334 ++++---- README.md.old | 251 ++++++ apps/README.md | 136 --- docs/COPILOT_CONFIGURATION.md | 181 ++++ docs/DEVELOPMENT.md | 380 +++++++++ .../PYPI_RELEASE_GUIDE.md | 0 .../PYPI_SETUP_SUMMARY.md | 0 .../PYPI_TEST_REPORT.md | 0 docs/REFACTORING_SUMMARY.md | 287 +++++++ examples/README.md | 370 +++++++++ {apps => examples}/__init__.py | 0 {apps => examples}/demo_feature_extraction.py | 0 {apps => examples}/run_article_monitoring.py | 0 {apps => examples}/run_auto_scaling_chat.py | 0 {apps => examples}/run_medical_diagnosis.py | 0 {apps => examples}/run_smart_home.py | 0 {apps => examples}/run_video_intelligence.py | 0 {apps => examples}/run_work_report.py | 0 {apps => examples}/test_apps.py | 0 migrate_tutorials_to_sage.sh | 138 ++++ pyproject.toml | 31 +- tutorials/L1-common/README.md | 92 --- .../L1-common/embedding_server_example.py | 224 ----- tutorials/L1-common/hello_world.py | 47 -- .../unified_inference_client_example.py | 281 ------- tutorials/L2-platform/README.md | 48 -- tutorials/L2-platform/environment/README.md | 28 - .../L2-platform/environment/remote_env.py | 290 ------- tutorials/L3-kernel/README.md | 85 -- .../checkpoint_recovery_test.py | 199 ----- .../fault_tolerance/fault_tolerance.py | 227 ----- .../fault_tolerance_examples.py | 412 --------- .../advanced/hello_future_stream_example.py | 193 ----- .../L3-kernel/advanced/hello_future_world.py | 118 --- .../hello_realistic_service_example.py | 506 ------------ .../advanced/parallelism_local_validation.py | 283 ------- .../advanced/parallelism_remote_validation.py | 326 -------- .../advanced/pipeline_as_service/README.md | 65 -- .../async_client_pipeline_as_service.py | 178 ---- .../hello_pipeline_as_service.py | 315 ------- .../multi_client_pipeline_as_service.py | 156 ---- .../pipeline_as_service/pipeline_bridge.py | 66 -- .../qa_pipeline_as_service.py | 638 -------------- .../advanced/simple_parallelism_validation.py | 309 ------- .../batch/hello_batch_operator_examples.py | 274 ------ .../batch/hello_batch_vs_source_comparison.py | 208 ----- .../L3-kernel/batch/hello_local_batch.py | 293 ------- .../L3-kernel/batch/hello_remote_batch.py | 372 --------- tutorials/L3-kernel/cpu_node_demo.py | 589 ------------- .../functions/hello_comap_function_example.py | 220 ----- .../functions/hello_comap_lambda_example.py | 351 -------- .../hello_wordcount_batch_example.py | 69 -- .../hello_wordcount_lambda_example.py | 104 --- tutorials/L3-kernel/keyed_state_example.py | 403 --------- .../L3-kernel/operators/hello_comap_world.py | 65 -- .../L3-kernel/operators/hello_filter_world.py | 54 -- .../operators/hello_flatmap_world.py | 52 -- .../L3-kernel/operators/hello_join_world.py | 104 --- .../operators/hello_three_input_comap.py | 107 --- .../stream/hello_connected_stream_example.py | 102 --- .../L3-kernel/stream/hello_onebyone_world.py | 47 -- .../L3-kernel/stream/hello_streaming_world.py | 53 -- .../stream/hello_wordcount_source_example.py | 77 -- tutorials/L3-libs/README.md | 87 -- tutorials/L3-libs/agent_sft_demo.py | 179 ---- tutorials/L3-libs/agents/README.md | 34 - tutorials/L3-libs/agents/__init__.py | 1 - tutorials/L3-libs/agents/arxiv_search_tool.py | 239 ------ tutorials/L3-libs/agents/basic_agent.py | 205 ----- tutorials/L3-libs/agents/config/README.md | 24 - .../agents/config/config_agent_min.yaml | 93 --- tutorials/L3-libs/agents/data/README.md | 27 - .../L3-libs/agents/data/agent_queries.jsonl | 4 - .../agents/data/agent_queries_test.jsonl | 1 - tutorials/L3-libs/agents/demo_arxiv_search.py | 179 ---- tutorials/L3-libs/agents/tool_use_pipeline.py | 676 --------------- tutorials/L3-libs/agents/workflow_demo.py | 260 ------ tutorials/L3-libs/amms_example.py | 116 --- tutorials/L3-libs/embeddings/README.md | 26 - .../L3-libs/embeddings/cross_modal_search.py | 101 --- .../L3-libs/embeddings/embedding_demo.py | 260 ------ .../embeddings/embedding_service_demo.py | 448 ---------- .../pipeline_builder_embedding_demo.py | 232 ------ tutorials/L3-libs/embeddings/quickstart.py | 103 --- tutorials/L3-libs/llm/demo_new_templates.py | 183 ---- .../L3-libs/llm/pipeline_builder_llm_demo.py | 260 ------ .../L3-libs/llm/templates_to_llm_demo.py | 240 ------ tutorials/L3-libs/rag/README.md | 37 - tutorials/L3-libs/rag/adaptive_rag_v2.py | 676 --------------- .../L3-libs/rag/advanced_rag_topology.py | 779 ------------------ tutorials/L3-libs/rag/qa_local_llm.py | 143 ---- tutorials/L3-libs/rag/qa_no_retrieval.py | 135 --- tutorials/L3-libs/rag/simple_rag.py | 305 ------- .../L3-libs/rag/usage_1_direct_library.py | 279 ------- .../L3-libs/rag/usage_2_sage_function.py | 226 ----- .../L3-libs/rag/usage_3_memory_service.py | 472 ----------- tutorials/L3-libs/rag/usage_4_complete_rag.py | 582 ------------- tutorials/L3-libs/unlearning/README.md | 71 -- tutorials/L3-libs/unlearning/__init__.py | 6 - .../unlearning/basic_unlearning_demo.py | 143 ---- .../unlearning/machine_unlearning_examples.py | 366 -------- tutorials/L4-middleware/README.md | 65 -- .../L4-middleware/hello_service_world.py | 51 -- tutorials/L4-middleware/rag/examples.py | 270 ------ tutorials/L4-middleware/sage_db/README.md | 42 - .../sage_db/hello_sage_db_app.py | 69 -- .../sage_db/hello_sage_db_service.py | 70 -- .../L4-middleware/sage_db/workflow_demo.py | 349 -------- tutorials/L4-middleware/sage_flow/README.md | 38 - .../sage_flow/hello_sage_flow_app.py | 97 --- .../sage_flow/hello_sage_flow_service.py | 105 --- tutorials/L4-middleware/sage_tsdb/README.md | 262 ------ tutorials/L4-middleware/sage_tsdb/__init__.py | 9 - .../sage_tsdb/advanced_dag_example.py | 265 ------ .../sage_tsdb/basic_dag_example.py | 211 ----- .../sage_tsdb/stream_join_dag_example.py | 202 ----- tutorials/L5-apps/README.md | 34 - tutorials/L6-interface/README.md | 174 ---- tutorials/QUICK_START.md | 320 ------- tutorials/README.md | 348 -------- tutorials/__init__.py | 9 - tutorials/benchmark_control_plane_demo.py | 484 ----------- tutorials/config/README.md | 266 ------ tutorials/config/config.yaml | 47 -- tutorials/docs/QUICK_REFERENCE.md | 318 ------- tutorials/docs/TROUBLESHOOTING.md | 655 --------------- tutorials/embedding_server_example.py | 147 ---- tutorials/hello_world.py | 47 -- tutorials/l6/memory_backend_demo.py | 111 --- .../vllm_control_plane_config_examples.md | 253 ------ tutorials/vllm_control_plane_tutorial.py | 160 ---- 137 files changed, 2270 insertions(+), 22661 deletions(-) create mode 100644 .git-commit-msg create mode 100644 .github/chatmodes/sage-examples.chatmode.md create mode 100644 .github/workflows/quality.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .pre-commit-config.yaml create mode 100644 README.md.old delete mode 100644 apps/README.md create mode 100644 docs/COPILOT_CONFIGURATION.md create mode 100644 docs/DEVELOPMENT.md rename PYPI_RELEASE_GUIDE.md => docs/PYPI_RELEASE_GUIDE.md (100%) rename PYPI_SETUP_SUMMARY.md => docs/PYPI_SETUP_SUMMARY.md (100%) rename PYPI_TEST_REPORT.md => docs/PYPI_TEST_REPORT.md (100%) create mode 100644 docs/REFACTORING_SUMMARY.md create mode 100644 examples/README.md rename {apps => examples}/__init__.py (100%) rename {apps => examples}/demo_feature_extraction.py (100%) rename {apps => examples}/run_article_monitoring.py (100%) rename {apps => examples}/run_auto_scaling_chat.py (100%) rename {apps => examples}/run_medical_diagnosis.py (100%) rename {apps => examples}/run_smart_home.py (100%) rename {apps => examples}/run_video_intelligence.py (100%) rename {apps => examples}/run_work_report.py (100%) rename {apps => examples}/test_apps.py (100%) create mode 100755 migrate_tutorials_to_sage.sh delete mode 100644 tutorials/L1-common/README.md delete mode 100644 tutorials/L1-common/embedding_server_example.py delete mode 100644 tutorials/L1-common/hello_world.py delete mode 100644 tutorials/L1-common/unified_inference_client_example.py delete mode 100644 tutorials/L2-platform/README.md delete mode 100644 tutorials/L2-platform/environment/README.md delete mode 100644 tutorials/L2-platform/environment/remote_env.py delete mode 100644 tutorials/L3-kernel/README.md delete mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/checkpoint_recovery_test.py delete mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance.py delete mode 100644 tutorials/L3-kernel/advanced/fault_tolerance/fault_tolerance_examples.py delete mode 100644 tutorials/L3-kernel/advanced/hello_future_stream_example.py delete mode 100644 tutorials/L3-kernel/advanced/hello_future_world.py delete mode 100644 tutorials/L3-kernel/advanced/hello_realistic_service_example.py delete mode 100644 tutorials/L3-kernel/advanced/parallelism_local_validation.py delete mode 100644 tutorials/L3-kernel/advanced/parallelism_remote_validation.py delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/README.md delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/async_client_pipeline_as_service.py delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/hello_pipeline_as_service.py delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/multi_client_pipeline_as_service.py delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/pipeline_bridge.py delete mode 100644 tutorials/L3-kernel/advanced/pipeline_as_service/qa_pipeline_as_service.py delete mode 100644 tutorials/L3-kernel/advanced/simple_parallelism_validation.py delete mode 100644 tutorials/L3-kernel/batch/hello_batch_operator_examples.py delete mode 100644 tutorials/L3-kernel/batch/hello_batch_vs_source_comparison.py delete mode 100644 tutorials/L3-kernel/batch/hello_local_batch.py delete mode 100644 tutorials/L3-kernel/batch/hello_remote_batch.py delete mode 100644 tutorials/L3-kernel/cpu_node_demo.py delete mode 100644 tutorials/L3-kernel/functions/hello_comap_function_example.py delete mode 100644 tutorials/L3-kernel/functions/hello_comap_lambda_example.py delete mode 100644 tutorials/L3-kernel/functions/hello_wordcount_batch_example.py delete mode 100644 tutorials/L3-kernel/functions/hello_wordcount_lambda_example.py delete mode 100644 tutorials/L3-kernel/keyed_state_example.py delete mode 100644 tutorials/L3-kernel/operators/hello_comap_world.py delete mode 100644 tutorials/L3-kernel/operators/hello_filter_world.py delete mode 100644 tutorials/L3-kernel/operators/hello_flatmap_world.py delete mode 100644 tutorials/L3-kernel/operators/hello_join_world.py delete mode 100644 tutorials/L3-kernel/operators/hello_three_input_comap.py delete mode 100644 tutorials/L3-kernel/stream/hello_connected_stream_example.py delete mode 100644 tutorials/L3-kernel/stream/hello_onebyone_world.py delete mode 100644 tutorials/L3-kernel/stream/hello_streaming_world.py delete mode 100644 tutorials/L3-kernel/stream/hello_wordcount_source_example.py delete mode 100644 tutorials/L3-libs/README.md delete mode 100644 tutorials/L3-libs/agent_sft_demo.py delete mode 100644 tutorials/L3-libs/agents/README.md delete mode 100644 tutorials/L3-libs/agents/__init__.py delete mode 100644 tutorials/L3-libs/agents/arxiv_search_tool.py delete mode 100644 tutorials/L3-libs/agents/basic_agent.py delete mode 100644 tutorials/L3-libs/agents/config/README.md delete mode 100644 tutorials/L3-libs/agents/config/config_agent_min.yaml delete mode 100644 tutorials/L3-libs/agents/data/README.md delete mode 100644 tutorials/L3-libs/agents/data/agent_queries.jsonl delete mode 100644 tutorials/L3-libs/agents/data/agent_queries_test.jsonl delete mode 100644 tutorials/L3-libs/agents/demo_arxiv_search.py delete mode 100644 tutorials/L3-libs/agents/tool_use_pipeline.py delete mode 100644 tutorials/L3-libs/agents/workflow_demo.py delete mode 100644 tutorials/L3-libs/amms_example.py delete mode 100644 tutorials/L3-libs/embeddings/README.md delete mode 100644 tutorials/L3-libs/embeddings/cross_modal_search.py delete mode 100644 tutorials/L3-libs/embeddings/embedding_demo.py delete mode 100644 tutorials/L3-libs/embeddings/embedding_service_demo.py delete mode 100644 tutorials/L3-libs/embeddings/pipeline_builder_embedding_demo.py delete mode 100644 tutorials/L3-libs/embeddings/quickstart.py delete mode 100644 tutorials/L3-libs/llm/demo_new_templates.py delete mode 100644 tutorials/L3-libs/llm/pipeline_builder_llm_demo.py delete mode 100644 tutorials/L3-libs/llm/templates_to_llm_demo.py delete mode 100644 tutorials/L3-libs/rag/README.md delete mode 100644 tutorials/L3-libs/rag/adaptive_rag_v2.py delete mode 100644 tutorials/L3-libs/rag/advanced_rag_topology.py delete mode 100644 tutorials/L3-libs/rag/qa_local_llm.py delete mode 100644 tutorials/L3-libs/rag/qa_no_retrieval.py delete mode 100644 tutorials/L3-libs/rag/simple_rag.py delete mode 100644 tutorials/L3-libs/rag/usage_1_direct_library.py delete mode 100644 tutorials/L3-libs/rag/usage_2_sage_function.py delete mode 100644 tutorials/L3-libs/rag/usage_3_memory_service.py delete mode 100644 tutorials/L3-libs/rag/usage_4_complete_rag.py delete mode 100644 tutorials/L3-libs/unlearning/README.md delete mode 100644 tutorials/L3-libs/unlearning/__init__.py delete mode 100644 tutorials/L3-libs/unlearning/basic_unlearning_demo.py delete mode 100644 tutorials/L3-libs/unlearning/machine_unlearning_examples.py delete mode 100644 tutorials/L4-middleware/README.md delete mode 100644 tutorials/L4-middleware/hello_service_world.py delete mode 100644 tutorials/L4-middleware/rag/examples.py delete mode 100644 tutorials/L4-middleware/sage_db/README.md delete mode 100644 tutorials/L4-middleware/sage_db/hello_sage_db_app.py delete mode 100644 tutorials/L4-middleware/sage_db/hello_sage_db_service.py delete mode 100644 tutorials/L4-middleware/sage_db/workflow_demo.py delete mode 100644 tutorials/L4-middleware/sage_flow/README.md delete mode 100644 tutorials/L4-middleware/sage_flow/hello_sage_flow_app.py delete mode 100644 tutorials/L4-middleware/sage_flow/hello_sage_flow_service.py delete mode 100644 tutorials/L4-middleware/sage_tsdb/README.md delete mode 100644 tutorials/L4-middleware/sage_tsdb/__init__.py delete mode 100644 tutorials/L4-middleware/sage_tsdb/advanced_dag_example.py delete mode 100644 tutorials/L4-middleware/sage_tsdb/basic_dag_example.py delete mode 100644 tutorials/L4-middleware/sage_tsdb/stream_join_dag_example.py delete mode 100644 tutorials/L5-apps/README.md delete mode 100644 tutorials/L6-interface/README.md delete mode 100644 tutorials/QUICK_START.md delete mode 100644 tutorials/README.md delete mode 100644 tutorials/__init__.py delete mode 100644 tutorials/benchmark_control_plane_demo.py delete mode 100644 tutorials/config/README.md delete mode 100644 tutorials/config/config.yaml delete mode 100644 tutorials/docs/QUICK_REFERENCE.md delete mode 100644 tutorials/docs/TROUBLESHOOTING.md delete mode 100644 tutorials/embedding_server_example.py delete mode 100644 tutorials/hello_world.py delete mode 100644 tutorials/l6/memory_backend_demo.py delete mode 100644 tutorials/vllm_control_plane_config_examples.md delete mode 100644 tutorials/vllm_control_plane_tutorial.py diff --git a/.git-commit-msg b/.git-commit-msg new file mode 100644 index 0000000..62f36c1 --- /dev/null +++ b/.git-commit-msg @@ -0,0 +1,30 @@ +refactor: reorganize structure and prepare for tutorials migration + +## Directory Restructuring + +- Rename `apps/` → `examples/` for clarity +- Create `docs/` directory for documentation +- All files preserve git history via `git mv` + +## Tutorials Migration + +Tutorials will be moved to SAGE main repository for better synchronization. + +**Changes**: +- Update README to point to SAGE/tutorials +- Remove tutorials from pyproject.toml +- Update CI workflows (no tutorial tests) +- Created migration script: `migrate_tutorials_to_sage.sh` + +## Code Quality Infrastructure + +- Pre-commit hooks (ruff, mypy, shellcheck, etc.) +- CI/CD workflows (quality + tests) +- Comprehensive development documentation + +## Breaking Changes + +- Path changes: `apps/` → `examples/` +- Tutorials location: Moving to SAGE main repo + +See docs/REFACTORING_SUMMARY.md for details. diff --git a/.github/chatmodes/sage-examples.chatmode.md b/.github/chatmodes/sage-examples.chatmode.md new file mode 100644 index 0000000..2f61212 --- /dev/null +++ b/.github/chatmodes/sage-examples.chatmode.md @@ -0,0 +1,214 @@ +```chatmode--- + +---description: 'Description of the custom chat mode.' + +description: 'SAGE Examples Assistant - Expert guide for SAGE tutorials and applications with layered architecture expertise'tools: ['runCommands', 'runTasks', 'edit', 'runNotebooks', 'search', 'new', 'extensions', 'usages', 'vscodeAPI', 'problems', 'changes', 'testFailure', 'openSimpleBrowser', 'fetch', 'githubRepo'] + +tools: ['runCommands', 'runTasks', 'edit', 'runNotebooks', 'search', 'new', 'extensions', 'usages', 'vscodeAPI', 'problems', 'changes', 'testFailure', 'openSimpleBrowser', 'fetch', 'githubRepo']--- + +---Define the purpose of this chat mode and how AI should behave: response style, available tools, focus areas, and any mode-specific instructions or constraints. + +# SAGE Examples Assistant + +You are an expert assistant for the **sage-examples** repository, specialized in helping users learn and build with the SAGE framework through tutorials and production applications. + +## 🎯 Primary Mission + +Guide users through SAGE's **6-layer architecture** (L1-L6) with clear, actionable examples that demonstrate best practices for AI application development. + +## 🏗️ SAGE Architecture Knowledge + +**CRITICAL**: SAGE uses a strict 6-layer architecture with **unidirectional dependencies** (top → bottom only): + +``` +L6: Interface → sage-cli, sage-studio, sage-tools (CLI + Web UI) +L5: Apps → sage-apps, sage-benchmark (Applications) +L4: Middleware → sage-middleware (Domain operators) +L3: Core → sage-kernel (Execution) + sage-libs (RAG/Agents/Algorithms) +L2: Platform → sage-platform (Queues, Storage, Services) +L1: Foundation → sage-common, sage-llm-core (Config, Logging, LLM Control Plane) +``` + +**Dependency Rules**: +- ✅ Can depend on lower layers (L6→L5→L4→L3→L2→L1) +- ❌ NEVER depend on upper layers or create circular dependencies +- 📦 PyPI packages use `isage-*` prefix (e.g., `isage-libs`, `isage-common`) + +## 📚 Repository Structure Expertise + +### Tutorials (`tutorials/`) +Organized by architectural layers for structured learning: +- `L1-common/` - Foundation basics (config, logging, unified client) +- `L2-platform/` - Platform services (scheduler, storage) +- `L3-kernel/` - Execution engine (batch, stream, operators) +- `L3-libs/` - Algorithms (RAG, Agents, Embeddings, LLM) +- `L4-middleware/` - Domain operators (vector DB, time-series DB) +- `L5-apps/` - Application examples +- `L6-interface/` - CLI and UI examples + +### Applications (`examples/`) +Production-ready reference implementations: +- Video Intelligence (CLIP + MobileNetV3) +- Medical Diagnosis (Multi-agent systems) +- Smart Home (IoT automation) +- Article Monitoring (NLP pipeline) +- Auto Scaling Chat (Dynamic scaling) + +### sage-apps Package (`sage-apps/`) +Standalone application package with independent development cycle. + +## 🔍 Documentation-First Approach + +**ALWAYS check documentation BEFORE making assumptions**: + +1. **Start with READMEs**: `README.md`, `tutorials/README.md`, `examples/README.md` +2. **Quick references**: `tutorials/QUICK_START.md`, `tutorials/docs/QUICK_REFERENCE.md` +3. **Layer-specific guides**: `tutorials/L{1-6}-*/README.md` +4. **Troubleshooting**: `tutorials/docs/TROUBLESHOOTING.md` + +**Use `grep_search` or `semantic_search` to find docs before guessing!** + +## 🚨 Critical Development Principles + +### 1. ❌ NO Manual pip install - ALWAYS Use pyproject.toml + +**FORBIDDEN**: +```bash +pip install transformers # ❌ Manual install +pip install torch==2.7.0 # ❌ Manual version +``` + +**REQUIRED**: +```toml +# Declare in pyproject.toml or requirements.txt +dependencies = [ + "isage-libs>=0.2.0", + "transformers>=4.52.0", +] +``` +```bash +# Then install package +pip install -e . +``` + +**Why**: Reproducibility, version control, dependency tracking, single source of truth. + +### 2. ❌ NO Fallback Logic - Project-Wide Rule + +**FORBIDDEN** (anywhere in code): +```python +try: + from ._version import __version__ +except ImportError: + __version__ = "unknown" # ❌ Hides missing file + +config = os.getenv("API_KEY") or "default" # ❌ Silent fallback +``` + +**REQUIRED** (fail-fast with clear errors): +```python +from ._version import __version__ # Let it raise ImportError +config = os.environ["API_KEY"] # Let it raise KeyError + +# Or provide helpful error: +if not os.path.exists("config.yaml"): + raise FileNotFoundError( + "config.yaml not found. Create from config.yaml.template" + ) +``` + +**Why**: Early detection, clear debugging, production-safe behavior. + +### 3. 📦 SAGE Dependencies + +**Core packages** (installed via PyPI): +- `isage-common` - Foundation (L1) +- `isage-llm-core` - LLM control plane (L1) +- `isage-libs` - RAG/Agents/Algorithms (L3) +- `isage-middleware` - Domain operators (L4) +- `isage-apps` - Applications (L5) + +**Optional packages**: +- `isage-vdb` - Vector database +- `isage-benchmark` - Benchmarking tools +- `isage-neuromem` - Neural memory systems + +## 🎓 Guiding Users Through Learning Paths + +### For Beginners (🟢) +1. Start with `tutorials/hello_world.py` (30 seconds) +2. Progress through `L1-common/` examples (15-30 min) +3. Move to `L3-libs/` for RAG/Agents (1-2 hours) + +### For Application Developers (🟡) +1. Quick review of L1 basics +2. Focus on `L3-libs/` (RAG, Agents) +3. Study `examples/` reference implementations +4. Build custom applications + +### For Platform Engineers (🔴) +1. Systematic study L1→L6 +2. Understand execution engine (L3-kernel) +3. Middleware integration (L4) +4. Full application lifecycle (L5-L6) + +## 💡 Response Style + +- **Clear and structured**: Use headings, code blocks, and step-by-step guidance +- **Layer-aware**: Always mention which layer a component belongs to +- **Dependency-conscious**: Check architectural constraints before suggesting code +- **Example-driven**: Reference existing examples from tutorials or apps +- **Documentation-backed**: Quote or link to relevant READMEs + +## 🔧 Common Tasks + +### Adding New Examples +1. Identify correct layer (L1-L6) +2. Check layer README for conventions +3. Update layer README with new example +4. Add dependencies to `pyproject.toml` (NOT manual pip install!) + +### Debugging Import Errors +1. Check if SAGE packages are installed (`pip list | grep isage`) +2. Verify Python version (>=3.10) +3. Check `requirements.txt` or `pyproject.toml` dependencies +4. Suggest `pip install -e .` or `pip install -r requirements.txt` + +### Environment Setup Issues +1. Check `.env.template` exists +2. Guide user to copy and configure `.env` +3. Verify API keys are set (OPENAI_API_KEY, HF_TOKEN, etc.) +4. Check SAGE Gateway connection (localhost:8889) + +## 🚫 What NOT to Do + +- ❌ Don't assume architecture without checking docs +- ❌ Don't suggest manual pip install commands +- ❌ Don't add try-except fallbacks that hide errors +- ❌ Don't create cross-layer dependencies that violate architecture +- ❌ Don't modify code without understanding layer context +- ❌ Don't guess configuration - check `.env.template` and docs + +## ✅ What TO Do + +- ✅ Always read relevant READMEs first +- ✅ Suggest declarative dependency management (pyproject.toml) +- ✅ Use fail-fast error handling +- ✅ Respect layer boundaries and dependencies +- ✅ Reference existing examples as templates +- ✅ Provide complete, runnable code snippets +- ✅ Explain architectural reasoning behind suggestions + +## 🎯 Success Metrics + +You succeed when users: +- Understand SAGE's layered architecture +- Can navigate tutorials independently +- Write code that follows SAGE patterns +- Build applications respecting layer boundaries +- Debug issues using documentation and examples + +--- + +**Remember**: You're not just helping with code - you're teaching SAGE's design philosophy and best practices. Make every interaction a learning opportunity! +``` diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 44ff243..3b8cf5c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -142,7 +142,7 @@ L1: sage-common, sage-llm-core ``` sage-examples/ -├── apps/ # 🎯 生产应用示例 +├── apps/ # 🎯 生产应用示例（运行入口） │ ├── run_video_intelligence.py │ ├── run_medical_diagnosis.py │ ├── run_smart_home.py @@ -183,7 +183,7 @@ sage-examples/ - Root overview: `README.md` (示例总览、快速开始) - Tutorials overview: `tutorials/README.md` (教程结构、学习路径) -- Apps overview: `apps/README.md` (应用列表、使用方法) +- Examples overview: `examples/README.md` (应用列表、使用方法) - Quick start: `tutorials/QUICK_START.md` (5分钟快速入门) **在处理特定层级/主题时，Copilot 应该额外阅读：** @@ -204,7 +204,7 @@ sage-examples/ - **常见文档位置：** - 教程文档：`tutorials/docs/` - 配置示例：`tutorials/config/` - - 应用说明：`apps/README.md` + - 应用说明：`examples/README.md` - 快速入门：`tutorials/QUICK_START.md` **规则**: 不要猜测架构决策或策略。阅读文档。文档就是为此而存在的。 @@ -221,7 +221,7 @@ sage-examples/ - 循序渐进的学习路径 - 配置文件示例 -2. **Applications (apps/)**: 生产级应用示例 +2. **Examples (examples/)**: 应用示例运行入口 - 视频智能分析 - 医疗诊断系统 - 智能家居 @@ -264,7 +264,7 @@ sage-examples/ │ (intellistream/sage-examples) │ ├──────────────────────────────────────────────┤ │ • 教程代码 (tutorials/) │ -│ • 应用示例 (apps/) │ +│ • 应用示例入口 (examples/) │ │ • 独立 sage-apps 包 (sage-apps/) │ │ • 依赖 PyPI 上的 isage-* 包 │ └──────────────────────────────────────────────┘ @@ -372,19 +372,19 @@ cd apps && python run_smart_home.py ```bash # 视频智能分析 -python apps/run_video_intelligence.py --video path/to/video.mp4 +python examples/run_video_intelligence.py --video path/to/video.mp4 # 医疗诊断 -python apps/run_medical_diagnosis.py +python examples/run_medical_diagnosis.py # 智能家居 -python apps/run_smart_home.py +python examples/run_smart_home.py # 文章监控 -python apps/run_article_monitoring.py +python examples/run_article_monitoring.py # 自动扩展聊天 -python apps/run_auto_scaling_chat.py +python examples/run_auto_scaling_chat.py ``` ## Common Issues @@ -495,7 +495,7 @@ pre-commit run --all-files 1. **选择正确的位置**: - 教程 → `tutorials/L{1-6}-*/` - - 应用 → `apps/` + - 应用示例 → `examples/` - 配置 → `tutorials/config/` 2. **遵循命名规范**: @@ -570,6 +570,6 @@ if __name__ == "__main__": - Root: `README.md` - Tutorials: `tutorials/README.md`, `tutorials/QUICK_START.md` - Layer guides: `tutorials/L{1-6}-*/README.md` - - Apps: `apps/README.md` + - Examples: `examples/README.md` **Remember**: Don't guess. Read the docs. They exist for this reason. diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..2aff1a0 --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,72 @@ +name: Code Quality Checks + +on: + push: + branches: [main, main-dev] + pull_request: + branches: [main, main-dev] + workflow_dispatch: + +jobs: + pre-commit: + name: Pre-commit Hooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pre-commit + run: | + pip install pre-commit + + - name: Run pre-commit + run: | + pre-commit run --all-files --show-diff-on-failure + + ruff: + name: Ruff Linting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install ruff + run: | + pip install ruff==0.14.6 + + - name: Run ruff check + run: | + ruff check . + + - name: Run ruff format check + run: | + ruff format --check . + + mypy: + name: Type Checking (mypy) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install mypy types-PyYAML types-requests types-setuptools + pip install -r requirements.txt + + - name: Run mypy + run: | + mypy . --config-file=pyproject.toml --no-error-summary + continue-on-error: true # Type checking errors shouldn't block CI diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..ebe908f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,93 @@ +name: Tests + +on: + push: + branches: [main, main-dev] + pull_request: + branches: [main, main-dev] + workflow_dispatch: + +jobs: + test-examples: + name: Test Examples + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.11'] + env: + # Skip tests requiring network/external services in CI + SAGE_EXAMPLES_MODE: test + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-cov + + - name: Install examples dependencies + run: | + pip install -e ".[examples]" + + - name: Run example tests + run: | + pytest examples/test_apps.py -v --tb=short + continue-on-error: true + + test-sage-apps-package: + name: Test sage-apps Package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install sage-apps package + run: | + cd sage-apps + pip install -e ".[dev]" + + - name: Run sage-apps tests + run: | + cd sage-apps + pytest tests/ -v --tb=short --cov=src/sage/apps --cov-report=term-missing + continue-on-error: true + + test-coverage: + name: Coverage Report + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + pip install pytest pytest-cov + + - name: Run tests with coverage + run: | + pytest tutorials/ apps/test_apps.py --cov=. --cov-report=xml --cov-report=term-missing + continue-on-error: true + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + continue-on-error: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9f96846 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,84 @@ +# Pre-commit hooks configuration for SAGE Examples +# Installation: +# pip install pre-commit +# pre-commit install +# +# Usage: +# pre-commit run --all-files +# git commit --no-verify # Skip hooks temporarily (not recommended) + +default_language_version: + python: python3.11 + +repos: + # General file checks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - id: end-of-file-fixer + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: check-toml + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: check-merge-conflict + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + - id: detect-private-key + + # Python: Ruff linter and formatter (replaces black, isort, flake8) + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.6 # Match SAGE main repository version + hooks: + - id: ruff + args: [--fix] + types_or: [python, pyi] + - id: ruff-format + types_or: [python, pyi] + + # Python: mypy type checking + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.14.0 + hooks: + - id: mypy + args: [--config-file=pyproject.toml, --no-error-summary] + additional_dependencies: + - types-PyYAML + - types-requests + - types-setuptools + + # Shell scripts + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: [--severity=warning] + + # YAML formatting + - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.15.0 + hooks: + - id: pretty-format-yaml + args: [--autofix, --indent, '2'] + + # Markdown formatting + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.21 + hooks: + - id: mdformat + args: [--wrap, '100'] + additional_dependencies: + - mdformat-gfm + - mdformat-black + + # Secret detection + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: package.lock.json diff --git a/README.md b/README.md index f145e38..84229ab 100644 --- a/README.md +++ b/README.md @@ -1,251 +1,213 @@ -# SAGE Examples Collection +# SAGE Examples -Complete examples and tutorials demonstrating SAGE's capabilities. +[![CI Status](https://github.com/intellistream/sage-examples/workflows/Tests/badge.svg)](https://github.com/intellistream/sage-examples/actions) +[![Code Quality](https://github.com/intellistream/sage-examples/workflows/Quality/badge.svg)](https://github.com/intellistream/sage-examples/actions) +[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -## 🚀 Quick Start +Production-ready application examples for the SAGE framework. + +## 🎯 What is This? + +**sage-examples** showcases production application examples for [SAGE](https://github.com/intellistream/SAGE): + +- **🎯 Examples**: Complete, runnable application demonstrations +- **📦 sage-apps Package**: Installable application library (published to PyPI as `isage-apps`) + +> **� Looking for tutorials?** Visit [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) for learning materials. -**New to SAGE?** Start with tutorials: +## 🚀 Quick Start ```bash -# Your first SAGE program (30 seconds) -python examples/tutorials/hello_world.py +# Clone this repository +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples -# Learn embeddings (2 minutes) -python examples/tutorials/embedding_demo.py +# Install dependencies +pip install -r requirements.txt -# Build an agent (5 minutes) -python examples/tutorials/agents/basic_agent.py +# Run an application example +python examples/run_video_intelligence.py ``` -## 📁 Directory Structure +> **New to SAGE?** Start with [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) first. + +## 📁 Repository Structure ``` -examples/ -├── tutorials/ # 📚 Learning tutorials (START HERE!) -│ ├── hello_world.py -│ ├── agents/ # Agent tutorials -│ │ ├── config/ # Agent configurations -│ │ └── data/ # Agent test data -│ ├── multimodal/ # Text+Image+Video -│ ├── memory/ # Memory systems -│ │ ├── config/ # Memory configurations -│ │ └── data/ # Memory test data -│ ├── service/ # Service integration -│ ├── scheduler/ # Task scheduling -│ ├── sage_db/ # Vector database -│ ├── config/ # General configurations -│ ├── data/ # General data utilities +sage-examples/ +├── examples/ # 🎯 Production application examples +│ ├── run_video_intelligence.py +│ ├── run_medical_diagnosis.py +│ ├── run_smart_home.py │ └── ... │ -├── apps/ # 🎯 Production applications -│ ├── run_video_intelligence.py -│ └── run_medical_diagnosis.py +├── sage-apps/ # 📦 Installable application package +│ ├── src/sage/apps/ # Application implementations +│ └── tests/ # Package tests │ -└── memory/ # � Advanced memory examples (DEPRECATED - use tutorials/memory) +├── docs/ # 📖 Project documentation +├── data/ # Shared data files +└── pyproject.toml # Project configuration ``` -**Note**: RAG examples and benchmarks have been moved to `packages/sage-benchmark/`. See -`packages/sage-benchmark/README.md` for details. - -## 📚 Examples by Level +## 📚 Learning vs Examples -### 🟢 Beginner (< 30 minutes) +| Your Goal | Repository | +|-----------|------------| +| **Learn SAGE basics** | [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) | +| **See production examples** | [sage-examples](https://github.com/intellistream/sage-examples) (this repo) | +| **Install applications** | `pip install isage-apps` | -Simple, focused tutorials to learn SAGE basics. +## 🎯 Application Examples -**Location**: `examples/tutorials/` +Complete, runnable applications demonstrating real-world use cases: -- **Hello World**: Your first pipeline -- **Embeddings**: Text embeddings basics -- **Basic Agent**: Create an AI agent -- **Simple RAG**: Question answering -- **Service Basics**: Embedding service, pipeline service +| Application | Description | Script | +|-------------|-------------|--------| +| 🎬 **Video Intelligence** | Multi-model video analysis | `examples/run_video_intelligence.py` | +| 🏥 **Medical Diagnosis** | AI medical image analysis | `examples/run_medical_diagnosis.py` | +| 🏠 **Smart Home** | IoT automation system | `examples/run_smart_home.py` | +| 📰 **Article Monitoring** | News monitoring pipeline | `examples/run_article_monitoring.py` | +| 💬 **Auto-scaling Chat** | Dynamic scaling chat | `examples/run_auto_scaling_chat.py` | -**Run**: See `examples/tutorials/README.md` +See `examples/README.md` for details. -### 🟡 Intermediate (30 min - 2 hours) +## 📦 Installation -Production-ready patterns and integrations. +```bash +# Clone repository +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples -**Location**: `packages/sage-benchmark/`, `examples/memory/` +# Install dependencies +pip install -r requirements.txt -- **Advanced RAG**: Dense/sparse retrieval, reranking, multimodal fusion (see `sage-benchmark`) -- **RAG Benchmarking**: Performance evaluation and metrics (see `sage-benchmark/benchmark_rag`) -- **Memory Systems**: RAG with memory, persistence patterns -- **Vector Databases**: Milvus, ChromaDB, FAISS integration (see `sage-benchmark`) -- **Distributed RAG**: Ray-based parallel processing (see `sage-benchmark`) +# Or install as package +pip install isage-examples[examples] -**See**: `packages/sage-benchmark/README.md` for RAG and benchmark examples +# Or install applications directly +pip install isage-apps[video] # Video intelligence +pip install isage-apps[medical] # Medical diagnosis +pip install isage-apps[all] # All applications +``` -### 🔴 Advanced (2+ hours) +## 🏗️ SAGE Architecture Overview -Complete applications and complex workflows. +SAGE uses a strict 6-layer architecture with unidirectional dependencies: -**Location**: `examples/apps/` +``` +┌─────────────────────────────────────────────┐ +│ L6: Interface │ CLI, Web UI, Tools +├─────────────────────────────────────────────┤ +│ L5: Applications │ Production Apps +├─────────────────────────────────────────────┤ +│ L4: Middleware │ Domain Operators +├─────────────────────────────────────────────┤ +│ L3: Core │ Execution + Algorithms +│ ├─ Kernel (Batch/Stream Engine) │ +│ └─ Libs (RAG/Agents/Algorithms) │ +├─────────────────────────────────────────────┤ +│ L2: Platform │ Scheduler, Storage +├─────────────────────────────────────────────┤ +│ L1: Foundation │ Config, Logging, LLM +└─────────────────────────────────────────────┘ +``` -- **Video Intelligence**: Multi-model video analysis -- **Medical Diagnosis**: AI medical imaging -- **Multi-agent Systems**: Coordinated AI agents +**Dependency Rule**: Upper layers can depend on lower layers (L6→L5→...→L1), but never the reverse. -## 📦 Installation +## 🛠️ Development -### From PyPI (推荐) +### Setup ```bash -# 最小安装（仅教程） -pip install isage-examples +# Install development dependencies +pip install -e ".[dev]" -# 完整安装（教程 + 应用） -pip install isage-examples[all] -pip install isage-apps[all] +# Install pre-commit hooks +pre-commit install -# 或安装特定应用 -pip install isage-apps[video] # 视频智能 -pip install isage-apps[medical] # 医疗诊断 +# Run all checks +pre-commit run --all-files ``` -### 从源码安装（开发） +### Code Quality ```bash -# 克隆仓库 -git clone https://github.com/intellistream/sage-examples.git -cd sage-examples - -# 快速开始（交互式安装） -./quickstart.sh - -# 或手动安装 -pip install -e . # isage-examples（可编辑模式） -pip install -e sage-apps[all] # isage-apps（可编辑模式） -``` - -### 依赖说明 - -- `isage-examples`: 教程和基础示例 -- `isage-apps`: 生产级应用示例 -- 依赖的 SAGE 核心包会自动从 PyPI 安装 +# Format code +ruff format . -## ⚠️ Examples vs Tests +# Lint code +ruff check . -**This directory contains examples and demos, NOT tests.** +# Auto-fix issues +ruff check --fix . -- **Examples** (`examples/`): How to use SAGE features -- **Unit Tests** (`packages/*/tests/`): Verify code correctness -- **Integration Tests** (`tools/tests/`): Test example execution +# Type checking +mypy . +``` -## 🎯 Learning Paths +### Testing -### Path 1: RAG Developer +```bash +# Run all tests +pytest -1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/rag_simple.py` - Learn - basics -1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_milvus.py` - \- Add vector search -1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_rerank.py` - Improve - results -1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` - \- Add multimodal -1. `tutorials/memory/rag_memory_pipeline.py` - Add memory +# Run specific tests +pytest tutorials/ +pytest examples/test_apps.py +pytest sage-apps/tests/ -See `packages/sage-benchmark/README.md` for complete RAG documentation. +# With coverage +pytest --cov=. --cov-report=html +``` -**Note**: All RAG examples have been moved from `examples/rag/` to `packages/sage-benchmark/`. +See `docs/DEVELOPMENT.md` for complete development guide. -### Path 2: Agent Builder +## 📖 Documentation -1. `tutorials/agents/basic_agent.py` - Agent basics -1. `tutorials/agents/workflow_demo.py` - Workflows -1. `tutorials/agents/arxiv_search_tool.py` - Custom tools -1. `apps/run_medical_diagnosis.py` - Multi-agent app +- **Examples Guide**: `examples/README.md` - Application examples +- **Development Guide**: `docs/DEVELOPMENT.md` - Contributing +- **SAGE Tutorials**: [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) - Learn SAGE +- **SAGE Docs**: https://intellistream.github.io/SAGE -### Path 3: Service Developer +## 🤝 Contributing -1. `tutorials/service/embedding_service_demo.py` - Service basics -1. `tutorials/service/pipeline_as_service/` - Pipeline services -1. `tutorials/service/sage_db/` - Vector database service -1. `tutorials/service/sage_flow/` - Stream processing service +We welcome contributions! Please see: -## 💡 Tips +1. **Development Guide**: `docs/DEVELOPMENT.md` +2. **Code of Conduct**: Follow respectful collaboration +3. **Issue Tracker**: https://github.com/intellistream/sage-examples/issues -**Running examples:** +### Adding Examples -```bash -# Always run from project root -cd /path/to/SAGE -python examples/tutorials/hello_world.py -``` +1. **Tutorials**: Add to [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) +2. **Applications**: + - Implementation → `sage-apps/src/sage/apps/your_app/` + - Entry script → `examples/run_your_app.py` +3. **Tests**: Add tests and ensure they pass +4. **Dependencies**: Update `pyproject.toml` -**API Keys:** +## 🔗 Related Repositories -```bash -# Copy and configure -cp .env.example .env -# Edit .env with your keys -``` +- **SAGE Main**: https://github.com/intellistream/SAGE +- **SAGE Benchmark**: https://github.com/intellistream/sage-benchmark +- **PyPI Packages**: https://pypi.org/search/?q=isage -**Configurations:** +## 📄 License -```bash -# Tutorials have their own config directories -ls examples/tutorials/agents/config/*.yaml -ls examples/tutorials/memory/config/*.yaml +MIT License - see [LICENSE](LICENSE) for details. -# RAG configs are in sage-benchmark -ls packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/*.yaml +## 🙋 Getting Help -# See respective README files for details -cat packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/README.md -``` +- **GitHub Issues**: Bug reports and feature requests +- **GitHub Discussions**: Questions and community support +- **GitHub Copilot**: Use "SAGE Examples Assistant" chat mode -**Troubleshooting:** +## 🌟 Star History -- Missing dependencies? Check `requirements.txt` in each category -- Import errors? Make sure you installed SAGE: `pip install -e packages/sage-libs` -- Need data? Check the `data/` subdirectory in each example category -- Need config? Check the `config/` subdirectory in each example category -- Need help? See `docs/COMMUNITY.md` +If you find this project helpful, please consider giving it a ⭐️! -## 📖 Documentation +--- -- **Tutorials README**: `examples/tutorials/README.md` -- **Apps README**: `examples/apps/README.md` -- **RAG & Benchmarks**: `packages/sage-benchmark/README.md` -- **Main Docs**: `docs/` and `docs-public/` -- **API Docs**: Docstrings in `packages/sage-libs/` - -## 🔍 Quick Reference - -### By Feature - -- **Agents**: `tutorials/agents/` -- **RAG**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/` -- **Multimodal**: `tutorials/multimodal/`, - `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` -- **Memory**: `tutorials/memory/` -- **Services**: `tutorials/service/` -- **Streaming**: `tutorials/stream_mode/` -- **Distributed**: - `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_ray.py` -- **Benchmarking**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/` - -### By Technology - -- **ChromaDB**: - `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_qa_chroma.yaml` -- **Milvus**: - `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_*_milvus.yaml` -- **Ray**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_ray.yaml` -- **OpenAI**: Most RAG examples in sage-benchmark -- **Hugging Face**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_hf_model.py` -- **Local LLMs**: Various examples in sage-benchmark - -### By Use Case - -- **Question Answering**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_*.py` -- **Document Search**: - `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/build_*_index.py` -- **Image Search**: `tutorials/multimodal/` -- **Video Analysis**: `apps/run_video_intelligence.py` -- **Medical AI**: `apps/run_medical_diagnosis.py` -- **Web Services**: `tutorials/service/pipeline_as_service/` -- **RAG Benchmarking**: `sage-benchmark/src/sage/benchmark/benchmark_rag/` +**Made with ❤️ by the IntelliStream Team** diff --git a/README.md.old b/README.md.old new file mode 100644 index 0000000..f145e38 --- /dev/null +++ b/README.md.old @@ -0,0 +1,251 @@ +# SAGE Examples Collection + +Complete examples and tutorials demonstrating SAGE's capabilities. + +## 🚀 Quick Start + +**New to SAGE?** Start with tutorials: + +```bash +# Your first SAGE program (30 seconds) +python examples/tutorials/hello_world.py + +# Learn embeddings (2 minutes) +python examples/tutorials/embedding_demo.py + +# Build an agent (5 minutes) +python examples/tutorials/agents/basic_agent.py +``` + +## 📁 Directory Structure + +``` +examples/ +├── tutorials/ # 📚 Learning tutorials (START HERE!) +│ ├── hello_world.py +│ ├── agents/ # Agent tutorials +│ │ ├── config/ # Agent configurations +│ │ └── data/ # Agent test data +│ ├── multimodal/ # Text+Image+Video +│ ├── memory/ # Memory systems +│ │ ├── config/ # Memory configurations +│ │ └── data/ # Memory test data +│ ├── service/ # Service integration +│ ├── scheduler/ # Task scheduling +│ ├── sage_db/ # Vector database +│ ├── config/ # General configurations +│ ├── data/ # General data utilities +│ └── ... +│ +├── apps/ # 🎯 Production applications +│ ├── run_video_intelligence.py +│ └── run_medical_diagnosis.py +│ +└── memory/ # � Advanced memory examples (DEPRECATED - use tutorials/memory) +``` + +**Note**: RAG examples and benchmarks have been moved to `packages/sage-benchmark/`. See +`packages/sage-benchmark/README.md` for details. + +## 📚 Examples by Level + +### 🟢 Beginner (< 30 minutes) + +Simple, focused tutorials to learn SAGE basics. + +**Location**: `examples/tutorials/` + +- **Hello World**: Your first pipeline +- **Embeddings**: Text embeddings basics +- **Basic Agent**: Create an AI agent +- **Simple RAG**: Question answering +- **Service Basics**: Embedding service, pipeline service + +**Run**: See `examples/tutorials/README.md` + +### 🟡 Intermediate (30 min - 2 hours) + +Production-ready patterns and integrations. + +**Location**: `packages/sage-benchmark/`, `examples/memory/` + +- **Advanced RAG**: Dense/sparse retrieval, reranking, multimodal fusion (see `sage-benchmark`) +- **RAG Benchmarking**: Performance evaluation and metrics (see `sage-benchmark/benchmark_rag`) +- **Memory Systems**: RAG with memory, persistence patterns +- **Vector Databases**: Milvus, ChromaDB, FAISS integration (see `sage-benchmark`) +- **Distributed RAG**: Ray-based parallel processing (see `sage-benchmark`) + +**See**: `packages/sage-benchmark/README.md` for RAG and benchmark examples + +### 🔴 Advanced (2+ hours) + +Complete applications and complex workflows. + +**Location**: `examples/apps/` + +- **Video Intelligence**: Multi-model video analysis +- **Medical Diagnosis**: AI medical imaging +- **Multi-agent Systems**: Coordinated AI agents + +## 📦 Installation + +### From PyPI (推荐) + +```bash +# 最小安装（仅教程） +pip install isage-examples + +# 完整安装（教程 + 应用） +pip install isage-examples[all] +pip install isage-apps[all] + +# 或安装特定应用 +pip install isage-apps[video] # 视频智能 +pip install isage-apps[medical] # 医疗诊断 +``` + +### 从源码安装（开发） + +```bash +# 克隆仓库 +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples + +# 快速开始（交互式安装） +./quickstart.sh + +# 或手动安装 +pip install -e . # isage-examples（可编辑模式） +pip install -e sage-apps[all] # isage-apps（可编辑模式） +``` + +### 依赖说明 + +- `isage-examples`: 教程和基础示例 +- `isage-apps`: 生产级应用示例 +- 依赖的 SAGE 核心包会自动从 PyPI 安装 + +## ⚠️ Examples vs Tests + +**This directory contains examples and demos, NOT tests.** + +- **Examples** (`examples/`): How to use SAGE features +- **Unit Tests** (`packages/*/tests/`): Verify code correctness +- **Integration Tests** (`tools/tests/`): Test example execution + +## 🎯 Learning Paths + +### Path 1: RAG Developer + +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/rag_simple.py` - Learn + basics +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_milvus.py` + \- Add vector search +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_rerank.py` - Improve + results +1. `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` + \- Add multimodal +1. `tutorials/memory/rag_memory_pipeline.py` - Add memory + +See `packages/sage-benchmark/README.md` for complete RAG documentation. + +**Note**: All RAG examples have been moved from `examples/rag/` to `packages/sage-benchmark/`. + +### Path 2: Agent Builder + +1. `tutorials/agents/basic_agent.py` - Agent basics +1. `tutorials/agents/workflow_demo.py` - Workflows +1. `tutorials/agents/arxiv_search_tool.py` - Custom tools +1. `apps/run_medical_diagnosis.py` - Multi-agent app + +### Path 3: Service Developer + +1. `tutorials/service/embedding_service_demo.py` - Service basics +1. `tutorials/service/pipeline_as_service/` - Pipeline services +1. `tutorials/service/sage_db/` - Vector database service +1. `tutorials/service/sage_flow/` - Stream processing service + +## 💡 Tips + +**Running examples:** + +```bash +# Always run from project root +cd /path/to/SAGE +python examples/tutorials/hello_world.py +``` + +**API Keys:** + +```bash +# Copy and configure +cp .env.example .env +# Edit .env with your keys +``` + +**Configurations:** + +```bash +# Tutorials have their own config directories +ls examples/tutorials/agents/config/*.yaml +ls examples/tutorials/memory/config/*.yaml + +# RAG configs are in sage-benchmark +ls packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/*.yaml + +# See respective README files for details +cat packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/README.md +``` + +**Troubleshooting:** + +- Missing dependencies? Check `requirements.txt` in each category +- Import errors? Make sure you installed SAGE: `pip install -e packages/sage-libs` +- Need data? Check the `data/` subdirectory in each example category +- Need config? Check the `config/` subdirectory in each example category +- Need help? See `docs/COMMUNITY.md` + +## 📖 Documentation + +- **Tutorials README**: `examples/tutorials/README.md` +- **Apps README**: `examples/apps/README.md` +- **RAG & Benchmarks**: `packages/sage-benchmark/README.md` +- **Main Docs**: `docs/` and `docs-public/` +- **API Docs**: Docstrings in `packages/sage-libs/` + +## 🔍 Quick Reference + +### By Feature + +- **Agents**: `tutorials/agents/` +- **RAG**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/` +- **Multimodal**: `tutorials/multimodal/`, + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_multimodal_fusion.py` +- **Memory**: `tutorials/memory/` +- **Services**: `tutorials/service/` +- **Streaming**: `tutorials/stream_mode/` +- **Distributed**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_dense_retrieval_ray.py` +- **Benchmarking**: `packages/sage-benchmark/src/sage/benchmark/benchmark_rag/` + +### By Technology + +- **ChromaDB**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_qa_chroma.yaml` +- **Milvus**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_*_milvus.yaml` +- **Ray**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/config/config_ray.yaml` +- **OpenAI**: Most RAG examples in sage-benchmark +- **Hugging Face**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_hf_model.py` +- **Local LLMs**: Various examples in sage-benchmark + +### By Use Case + +- **Question Answering**: `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/qa_*.py` +- **Document Search**: + `sage-benchmark/src/sage/benchmark/benchmark_rag/implementations/build_*_index.py` +- **Image Search**: `tutorials/multimodal/` +- **Video Analysis**: `apps/run_video_intelligence.py` +- **Medical AI**: `apps/run_medical_diagnosis.py` +- **Web Services**: `tutorials/service/pipeline_as_service/` +- **RAG Benchmarking**: `sage-benchmark/src/sage/benchmark/benchmark_rag/` diff --git a/apps/README.md b/apps/README.md deleted file mode 100644 index 3535b3e..0000000 --- a/apps/README.md +++ /dev/null @@ -1,136 +0,0 @@ -# SAGE Applications Examples - -This directory contains example scripts for running applications from the `sage-apps` package. - -## Overview - -The `sage-apps` package provides production-ready AI applications built on the SAGE framework. These -example scripts demonstrate how to use them. - -## Available Applications - -### 1. Video Intelligence Pipeline - -Multi-model video analysis combining CLIP and MobileNetV3 for comprehensive video understanding. - -**Run:** - -```bash -python examples/apps/run_video_intelligence.py --video path/to/video.mp4 -``` - -**Features:** - -- Scene understanding with CLIP -- Action recognition with MobileNetV3 -- Frame-by-frame analysis -- Comprehensive video insights - -### 2. Medical Diagnosis System - -AI-assisted medical imaging analysis using multi-agent systems. - -**Run:** - -```bash -python examples/apps/run_medical_diagnosis.py -``` - -**Features:** - -- Multi-agent diagnostic workflow -- Medical image analysis -- Knowledge base integration -- Diagnostic report generation - -### 3. Article Monitoring - -Automated article/news monitoring and analysis pipeline. - -**Run:** - -```bash -python examples/apps/run_article_monitoring.py -``` - -**Features:** - -- Article content extraction -- Sentiment analysis -- Topic classification -- Automated summarization - -### 4. Auto Scaling Chat - -Scalable chat application with automatic resource scaling. - -**Run:** - -```bash -python examples/apps/run_auto_scaling_chat.py -``` - -**Features:** - -- Dynamic scaling based on load -- Multi-turn conversation support -- LLM integration -- Resource-aware execution - -### 5. Smart Home - -IoT-enabled smart home automation using SAGE pipelines. - -**Run:** - -```bash -python examples/apps/run_smart_home.py -``` - -**Features:** - -- Sensor data processing -- Device control automation -- Event-driven workflows -- Real-time monitoring - -## Installation - -### Install All Applications - -```bash -pip install -e packages/sage-apps[all] -``` - -### Install Specific Applications - -```bash -# Video Intelligence only -pip install -e packages/sage-apps[video] - -# Medical Diagnosis only -pip install -e packages/sage-apps[medical] -``` - -## Configuration - -Each application can be configured using YAML files located in `examples/config/`: - -- Video Intelligence: `config_video_intelligence.yaml` -- Medical Diagnosis: Custom configurations in the medical diagnosis data directory - -## Development - -For developing new applications or modifying existing ones, see: - -- `packages/sage-apps/README.md` - Package overview -- `packages/sage-apps/MIGRATION.md` - Migration guide -- `packages/sage-apps/PACKAGE_CREATION_SUMMARY.md` - Implementation details - -## Support - -For questions or issues: - -1. Check the application-specific documentation in `packages/sage-apps/src/sage/apps//` -1. Review the main SAGE documentation -1. Open an issue on GitHub diff --git a/docs/COPILOT_CONFIGURATION.md b/docs/COPILOT_CONFIGURATION.md new file mode 100644 index 0000000..34f4668 --- /dev/null +++ b/docs/COPILOT_CONFIGURATION.md @@ -0,0 +1,181 @@ +# SAGE Examples - Copilot Configuration Summary + +## 📋 Overview + +This document summarizes the Copilot configuration for the **sage-examples** repository, including both the chatmode and the instructions files. + +## 📁 Files Created/Updated + +### 1. `.github/chatmodes/sage-examples.chatmode.md` ✅ UPDATED + +**Purpose**: Defines a custom GitHub Copilot chat mode specifically for the sage-examples repository. + +**Key Features**: +- 🎯 **Primary Mission**: Guide users through SAGE's 6-layer architecture (L1-L6) +- 🏗️ **Architecture Knowledge**: Comprehensive understanding of SAGE's layered design +- 📚 **Repository Structure Expertise**: Deep knowledge of tutorials, apps, and sage-apps package +- 🔍 **Documentation-First Approach**: Always check docs before making assumptions +- 🚨 **Critical Development Principles**: No manual pip install, no fallback logic, proper dependency management + +**Content Structure**: +``` +1. Primary Mission +2. SAGE Architecture Knowledge (L1-L6) +3. Repository Structure Expertise +4. Documentation-First Approach +5. Critical Development Principles + - NO Manual pip install + - NO Fallback Logic + - SAGE Dependencies +6. Guiding Users Through Learning Paths +7. Response Style Guidelines +8. Common Tasks +9. What NOT to Do / What TO Do +10. Success Metrics +``` + +### 2. `.github/copilot-instructions.md` ✅ ALREADY EXISTS + +**Purpose**: Provides comprehensive instructions for GitHub Copilot when working in this repository. + +**Key Sections**: +- Overview of sage-examples +- Critical Principles (NO manual pip install, NO fallback logic) +- SAGE Dependencies and Architecture +- Directory Structure +- Documentation-First Approach +- SAGE Examples Scope +- Installation Instructions +- Environment Setup +- Running Examples +- Common Issues and Solutions +- Testing and Code Style +- Contributing Guidelines +- Resources + +## 🎯 How They Work Together + +### copilot-instructions.md (General Copilot) +- **Scope**: All Copilot interactions in this repository +- **Usage**: Automatically loaded by GitHub Copilot +- **Focus**: Development guidelines, architecture, and best practices + +### sage-examples.chatmode.md (Chat Mode) +- **Scope**: Custom chat mode for focused assistance +- **Usage**: Activated when user selects "SAGE Examples Assistant" mode +- **Focus**: Interactive guidance and learning support + +## 🔑 Key Principles Enforced + +### 1. ❌ NO Manual pip install +```bash +# FORBIDDEN +pip install transformers + +# REQUIRED +# Add to pyproject.toml, then: +pip install -e . +``` + +### 2. ❌ NO Fallback Logic +```python +# FORBIDDEN +try: + config = load_config() +except: + config = {} # Silent fallback + +# REQUIRED +config = load_config() # Fail fast with clear error +``` + +### 3. 📚 Documentation-First +Before any non-trivial work: +1. Read `README.md` +2. Check `tutorials/README.md` and `tutorials/QUICK_START.md` +3. Review layer-specific READMEs (`L1-common/README.md`, etc.) +4. Search docs using `grep_search` or `semantic_search` + +### 4. 🏗️ Respect Architecture +``` +L6: Interface → CLI, Web UI, Tools +L5: Apps → sage-apps, sage-benchmark +L4: Middleware → sage-middleware +L3: Core → sage-kernel + sage-libs +L2: Platform → sage-platform +L1: Foundation → sage-common, sage-llm-core +``` + +**Rule**: Only depend downward (L6→L5→L4→L3→L2→L1) + +## 🎓 Learning Paths Defined + +### 🟢 Beginners (< 30 min) +- Start: `tutorials/hello_world.py` +- Progress: L1-common examples +- Advance: L3-libs (RAG, Agents) + +### 🟡 Application Developers (30 min - 2 hours) +- Quick review: L1 basics +- Focus: L3-libs (RAG, Agents) +- Study: `apps/` reference implementations + +### 🔴 Platform Engineers (2+ hours) +- Systematic: L1→L6 all layers +- Deep dive: L3-kernel execution engine +- Integration: L4 middleware +- Lifecycle: L5-L6 applications + +## 🛠️ Common Tasks Support + +Both configurations provide guidance for: + +1. **Adding New Examples** + - Identify correct layer + - Follow naming conventions + - Update layer README + - Add dependencies properly + +2. **Debugging Import Errors** + - Check SAGE packages installation + - Verify Python version (>=3.10) + - Review dependencies + - Suggest proper installation + +3. **Environment Setup** + - Guide `.env` configuration + - Verify API keys + - Check service connections + +4. **Running Examples** + - Quick start commands + - Category-based navigation + - Application execution + +## 📊 Success Metrics + +Users succeed when they: +- ✅ Understand SAGE's layered architecture +- ✅ Can navigate tutorials independently +- ✅ Write code following SAGE patterns +- ✅ Build applications respecting layer boundaries +- ✅ Debug issues using documentation + +## 🔗 Resources Referenced + +- **SAGE Main Repo**: https://github.com/intellistream/SAGE +- **SAGE Docs**: https://sage.intellistream.com +- **PyPI Packages**: https://pypi.org/search/?q=isage +- **Benchmark Repo**: https://github.com/intellistream/sage-benchmark +- **Issue Tracker**: https://github.com/intellistream/sage-examples/issues + +## 🚀 Next Steps for Users + +1. **Developers**: Start with `tutorials/QUICK_START.md` +2. **Contributors**: Review `.github/copilot-instructions.md` +3. **Chat Users**: Activate "SAGE Examples Assistant" chat mode +4. **Everyone**: Follow the learning paths based on your role + +--- + +**Note**: Both configurations emphasize **documentation-first** approach and **fail-fast** philosophy to ensure high-quality, maintainable code that respects SAGE's architectural principles. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..cc5e520 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,380 @@ +# SAGE Examples - 开发指南 + +本文档说明 sage-examples 项目的开发流程、代码质量标准和 CI/CD 配置。 + +## 📁 项目结构 + +``` +sage-examples/ +├── .github/ +│ ├── workflows/ # CI/CD workflows +│ │ ├── quality.yml # 代码质量检查 +│ │ └── tests.yml # 测试运行 +│ ├── chatmodes/ # GitHub Copilot 聊天模式 +│ ├── agents/ # GitHub Copilot agents +│ └── copilot-instructions.md # Copilot 指令 +│ +├── docs/ # 📚 项目文档 +│ ├── COPILOT_CONFIGURATION.md +│ ├── PYPI_RELEASE_GUIDE.md +│ ├── PYPI_SETUP_SUMMARY.md +│ └── PYPI_TEST_REPORT.md +│ +├── tutorials/ # 📚 教程代码（L1-L6 分层） +├── examples/ # 🎯 应用示例（运行入口） +├── sage-apps/ # 独立 sage-apps 包 +├── data/ # 共享数据 +│ +├── .pre-commit-config.yaml # Pre-commit 配置 +├── pyproject.toml # 项目配置 +├── requirements.txt # Python 依赖 +└── README.md # 项目说明 +``` + +## 🔧 开发环境设置 + +### 1. 克隆仓库 + +```bash +git clone https://github.com/intellistream/sage-examples.git +cd sage-examples +``` + +### 2. 安装依赖 + +```bash +# 核心依赖（教程） +pip install -r requirements.txt + +# 或安装完整依赖（包括应用） +pip install -e ".[full]" + +# sage-apps 包开发 +cd sage-apps +pip install -e ".[dev]" +cd .. +``` + +### 3. 安装 Pre-commit Hooks + +```bash +# 安装 pre-commit +pip install pre-commit + +# 安装 hooks +pre-commit install + +# 手动运行所有检查 +pre-commit run --all-files +``` + +## ✅ 代码质量标准 + +### Pre-commit Hooks + +项目使用 pre-commit 自动化代码质量检查，每次提交时自动运行： + +#### 检查项目 + +1. **通用检查** + - 去除尾随空格 + - 确保文件以换行符结尾 + - 检查 YAML/JSON/TOML 语法 + - 检查大文件（>1MB） + - 检测合并冲突 + - 统一行尾符（LF） + - 检测私钥泄露 + +2. **Python 代码质量** + - **Ruff**: 快速 lint 和格式化（替代 black, isort, flake8） + - **mypy**: 类型检查 + +3. **Shell 脚本** + - **shellcheck**: Shell 脚本静态分析 + +4. **文档** + - **mdformat**: Markdown 格式化 + - **pretty-format-yaml**: YAML 格式化 + +5. **安全** + - **detect-secrets**: 敏感信息检测 + +### 手动运行检查 + +```bash +# 运行所有 pre-commit 检查 +pre-commit run --all-files + +# 运行特定检查 +pre-commit run ruff --all-files +pre-commit run mypy --all-files +pre-commit run shellcheck --all-files + +# 跳过 hooks（不推荐） +git commit --no-verify +``` + +### 代码格式化 + +```bash +# 使用 ruff 格式化 +ruff format . + +# 检查格式（不修改） +ruff format --check . + +# 自动修复 lint 问题 +ruff check --fix . +``` + +### 类型检查 + +```bash +# 运行 mypy +mypy . --config-file=pyproject.toml + +# 检查特定目录 +mypy tutorials/ examples/ +``` + +## 🧪 测试 + +### 运行测试 + +```bash +# 运行所有测试 +pytest + +# 运行特定测试 +pytest tutorials/ +pytest examples/test_apps.py +pytest sage-apps/tests/ + +# 带覆盖率 +pytest --cov=. --cov-report=html + +# 并行运行（快速） +pytest -n auto +``` + +### 测试组织 + +``` +tests/ # 如果有通用测试 +tutorials/ # 教程示例（可作为测试运行） +examples/test_apps.py # 应用测试 +sage-apps/tests/ # sage-apps 包测试 +``` + +## 🚀 CI/CD + +### GitHub Actions Workflows + +项目包含两个主要 workflow： + +#### 1. 代码质量检查 (`.github/workflows/quality.yml`) + +**触发条件**: +- Push 到 `main` 或 `main-dev` +- Pull Request +- 手动触发 + +**检查项**: +- Pre-commit hooks（所有检查） +- Ruff linting +- Ruff formatting +- mypy 类型检查 + +#### 2. 测试 (`.github/workflows/tests.yml`) + +**触发条件**: +- Push 到 `main` 或 `main-dev` +- Pull Request +- 手动触发 + +**测试矩阵**: +- Python 版本: 3.10, 3.11, 3.12 +- 测试套件: + - 教程测试 + - 应用测试 + - sage-apps 包测试 + - 覆盖率报告 + +### CI 行为 + +- ✅ **Pre-commit**: 必须通过所有 hooks +- ⚠️ **mypy**: 类型检查错误不阻塞 CI（continue-on-error） +- ⚠️ **Tests**: 测试失败记录但不阻塞（continue-on-error） + +### 本地模拟 CI + +```bash +# 运行与 CI 相同的 pre-commit 检查 +pre-commit run --all-files + +# 运行与 CI 相同的测试 +pytest tutorials/ examples/test_apps.py -v --tb=short +``` + +## 📝 贡献指南 + +### 提交前检查清单 + +- [ ] 代码通过 `ruff format .` 格式化 +- [ ] 代码通过 `ruff check .` 检查 +- [ ] 通过 `pre-commit run --all-files` +- [ ] 新功能有测试覆盖 +- [ ] 更新相关文档 +- [ ] Commit message 遵循规范（feat/fix/docs/chore） + +### Commit Message 规范 + +``` +(): + + + +