diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index 15ca806f41a..84ba43c9666 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -2,6 +2,7 @@ dirs: - . excludedFiles: - ./python/CHANGELOG.md + - "**/SKILL.md" ignorePatterns: - pattern: "/github/" - pattern: "./actions" @@ -26,7 +27,7 @@ ignorePatterns: - pattern: "https:\/\/dotnet.microsoft.com" - pattern: "https://github.com/Rel1cx/eslint-react" # excludedDirs: - # Folders which include links to localhost, since it's not ignored with regular expressions +# Folders which include links to localhost, since it's not ignored with regular expressions baseUrl: https://github.com/microsoft/agent-framework/ aliveStatusCodes: - 200 diff --git a/.github/actions/sample-validation-save-playbooks/action.yml b/.github/actions/sample-validation-save-playbooks/action.yml new file mode 100644 index 00000000000..5d1ffb15afb --- /dev/null +++ b/.github/actions/sample-validation-save-playbooks/action.yml @@ -0,0 +1,19 @@ +name: Save Sample Playbooks +description: > + Save the cached sample-validation playbooks. Split out from + sample-validation-setup (which only restores) so the save runs even when the + validation step fails. Combining restore+save via actions/cache would skip the + save on a failing job (post-if: success()), so freshly authored playbooks for + samples that failed validation would never persist. Invoke this with + 'if: not-cancelled' after the validation step in each job. + +runs: + using: "composite" + steps: + - name: Save sample playbooks cache + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the restore path/key in sample-validation-setup/action.yml and the + # sample_validation --playbooks-dir default (samples/sample_validation/playbooks). + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} diff --git a/.github/actions/sample-validation-setup/action.yml b/.github/actions/sample-validation-setup/action.yml index c9d2d2d6ac6..9bb1b90f555 100644 --- a/.github/actions/sample-validation-setup/action.yml +++ b/.github/actions/sample-validation-setup/action.yml @@ -36,15 +36,31 @@ runs: shell: bash run: copilot --version && copilot -p "What can you do in one sentence?" + - name: Set up python and install the project + uses: ./.github/actions/python-setup + with: + python-version: ${{ inputs.python-version }} + os: ${{ inputs.os }} + + - name: Restore sample playbooks + # Restore-only. The matching save is a separate step in each job that runs with + # `if: ${{ !cancelled() }}` (see .github/actions/sample-validation-save-playbooks). + # A combined actions/cache would skip its post-job save on a failing job + # (post-if: success()), so playbooks authored for samples that failed validation + # would never persist. Keyed per job so each validate-* job keeps its own playbooks. + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + # Must match the sample_validation --playbooks-dir default, which resolves to + # samples/sample_validation/playbooks (see python/scripts/sample_validation/__main__.py). + # If a job overrides --playbooks-dir, update this path to match. + path: python/samples/sample_validation/playbooks/ + key: sample-playbooks-${{ github.job }}-${{ github.run_id }} + restore-keys: | + sample-playbooks-${{ github.job }}- + - name: Azure CLI Login uses: azure/login@a457da9ea143d694b1b9c7c869ebb04ebe844ef5 # v2 with: client-id: ${{ inputs.azure-client-id }} tenant-id: ${{ inputs.azure-tenant-id }} subscription-id: ${{ inputs.azure-subscription-id }} - - - name: Set up python and install the project - uses: ./.github/actions/python-setup - with: - python-version: ${{ inputs.python-version }} - os: ${{ inputs.os }} diff --git a/.github/workflows/python-sample-validation.yml b/.github/workflows/python-sample-validation.yml index 8c0ab1b7cf3..ed8e05e11a2 100644 --- a/.github/workflows/python-sample-validation.yml +++ b/.github/workflows/python-sample-validation.yml @@ -8,8 +8,11 @@ on: env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache + GITHUB_TOKEN: ${{ github.token }} + GITHUB_COPILOT_MODEL: auto permissions: + copilot-requests: write contents: read id-token: write @@ -20,8 +23,8 @@ jobs: environment: integration env: # Required configuration for get-started samples - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -45,6 +48,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 01-get-started --save-report --report-name 01-get-started + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -54,12 +61,13 @@ jobs: validate-02-agents: name: Validate 02-agents + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: # Foundry configuration - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -67,12 +75,12 @@ jobs: AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME || vars.AZUREOPENAI__EMBEDDINGDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # GitHub MCP GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # Observability ENABLE_INSTRUMENTATION: "true" defaults: @@ -105,7 +113,11 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers --save-report --report-name 02-agents + cd scripts && uv run python -m sample_validation --subdir 02-agents --exclude providers harness tools --save-report --report-name 02-agents + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -114,15 +126,105 @@ jobs: name: validation-report-02-agents path: python/samples/sample_validation/reports/ + validate-02-agents-harness: + name: Validate 02-agents/harness + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Optional: enables the Foundry memory path in harness samples + FOUNDRY_EMBEDDING_MODEL: ${{ vars.FOUNDRY_EMBEDDING_MODEL || '' }} + FOUNDRY_MEMORY_STORE: ${{ vars.FOUNDRY_MEMORY_STORE || '' }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + echo "FOUNDRY_EMBEDDING_MODEL=$FOUNDRY_EMBEDDING_MODEL" >> .env + echo "FOUNDRY_MEMORY_STORE=$FOUNDRY_MEMORY_STORE" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/harness --save-report --report-name 02-agents-harness + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-harness + path: python/samples/sample_validation/reports/ + + validate-02-agents-tools: + name: Validate 02-agents/tools + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Create .env for samples + run: | + echo "FOUNDRY_PROJECT_ENDPOINT=$FOUNDRY_PROJECT_ENDPOINT" >> .env + echo "FOUNDRY_MODEL=$FOUNDRY_MODEL" >> .env + + - name: Run sample validation + run: | + cd scripts && uv run python -m sample_validation --subdir 02-agents/tools --save-report --report-name 02-agents-tools + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-02-agents-tools + path: python/samples/sample_validation/reports/ + validate-02-agents-openai: name: Validate 02-agents/providers/openai + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -148,6 +250,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/openai --save-report --report-name 02-agents-openai + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -157,6 +263,7 @@ jobs: validate-02-agents-azure: name: Validate 02-agents/providers/azure + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -187,6 +294,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/azure --save-report --report-name 02-agents-azure + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -196,6 +307,7 @@ jobs: validate-02-agents-anthropic: name: Validate 02-agents/providers/anthropic + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: @@ -224,6 +336,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/anthropic --save-report --report-name 02-agents-anthropic + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -233,15 +349,9 @@ jobs: validate-02-agents-github-copilot: name: Validate 02-agents/providers/github_copilot + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration - permissions: - copilot-requests: write - contents: read - id-token: write - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_COPILOT_MODEL: claude-opus-4.6 defaults: run: working-directory: python @@ -260,6 +370,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/github_copilot --save-report --report-name 02-agents-github-copilot + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -292,6 +406,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/amazon --save-report --report-name 02-agents-amazon + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -324,6 +442,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/ollama --save-report --report-name 02-agents-ollama + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -333,12 +455,12 @@ jobs: validate-02-agents-foundry: name: Validate 02-agents/providers/foundry - if: false # Temporarily disabled - provider folder also contains the local Foundry sample + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_AGENT_NAME || '' }} FOUNDRY_AGENT_VERSION: ${{ vars.FOUNDRY_AGENT_VERSION || '' }} defaults: @@ -366,6 +488,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/foundry --save-report --report-name 02-agents-foundry + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -408,6 +534,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/copilotstudio --save-report --report-name 02-agents-copilotstudio + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -437,6 +567,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 02-agents/providers/custom --save-report --report-name 02-agents-custom + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -446,11 +580,12 @@ jobs: validate-03-workflows: name: Validate 03-workflows + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} defaults: run: working-directory: python @@ -474,6 +609,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 03-workflows --save-report --report-name 03-workflows + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -481,14 +620,61 @@ jobs: name: validation-report-03-workflows path: python/samples/sample_validation/reports/ - validate-04-hosting: - name: Validate 04-hosting - if: false # Temporarily disabled because of sample complexity + validate-04-hosting-foundry-hosted-agents: + name: Validate 04-hosting (foundry-hosted-agents) runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # Foundry hosted agent configuration + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.FOUNDRY_MODEL }} + FOUNDRY_PROJECT_ID: ${{ vars.FOUNDRY_PROJECT_ID }} + AZURE_CONTAINER_REGISTRY_ENDPOINT: ${{ vars.AZURE_CONTAINER_REGISTRY_ENDPOINT }} + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + TOOLBOX_ENDPOINT: ${{ vars.TOOLBOX_ENDPOINT }} + FOUNDRY_AGENT_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_NAME }} + MEMORY_STORE_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_MEMORY_STORE }} + AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }} + AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }} + defaults: + run: + working-directory: python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Setup environment + uses: ./.github/actions/sample-validation-setup + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + os: ${{ runner.os }} + + - name: Run sample validation + # Maximum parallel workers is set to 1 because all samples use the same port + run: | + cd scripts && uv run python -m sample_validation --subdir 04-hosting/foundry-hosted-agents --save-report --report-name 04-hosting-foundry-hosted-agents --max-parallel-workers 1 + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + + - name: Upload validation report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: validation-report-04-hosting-foundry-hosted-agents + path: python/samples/sample_validation/reports/ + + validate-04-hosting-other: + name: Validate 04-hosting (other) + if: false # Temporarily disabled - to free up Copilot quota for other jobs + runs-on: ubuntu-latest + environment: integration + env: + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # A2A configuration A2A_AGENT_HOST: http://localhost:5001/ defaults: @@ -507,13 +693,17 @@ jobs: - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir 04-hosting --save-report --report-name 04-hosting + cd scripts && uv run python -m sample_validation --subdir 04-hosting --exclude foundry-hosted-agents --save-report --report-name 04-hosting-other + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: - name: validation-report-04-hosting + name: validation-report-04-hosting-other path: python/samples/sample_validation/reports/ validate-05-end-to-end: @@ -522,8 +712,8 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} @@ -552,6 +742,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir 05-end-to-end --save-report --report-name 05-end-to-end + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -564,16 +758,16 @@ jobs: runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # OpenAI configuration - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} defaults: run: working-directory: python @@ -598,9 +792,16 @@ jobs: echo "OPENAI_CHAT_COMPLETION_MODEL=$OPENAI_CHAT_COMPLETION_MODEL" >> .env echo "OPENAI_CHAT_MODEL=$OPENAI_CHAT_MODEL" >> .env + - name: Pre-install AutoGen dependencies for migration samples + run: uv pip install "autogen-agentchat" "autogen-ext[openai]" + - name: Run sample validation run: | - cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration + cd scripts && uv run python -m sample_validation --subdir autogen-migration --save-report --report-name autogen-migration --agent-timeout 600 + + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -611,23 +812,25 @@ jobs: validate-semantic-kernel-migration: name: Validate semantic-kernel-migration + if: false # Temporarily disabled - to free up Copilot quota for other jobs runs-on: ubuntu-latest environment: integration env: - FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT || vars.AZURE_AI_PROJECT_ENDPOINT }} - FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} + FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} + FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} # Azure OpenAI configuration for AF AZURE_OPENAI_ENDPOINT: ${{ vars.AZUREOPENAI__ENDPOINT }} AZURE_OPENAI_MODEL: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME || vars.AZUREOPENAI__RESPONSESDEPLOYMENTNAME }} # Azure OpenAI configuration for SK AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }} # OpenAI key - OPENAI_API_KEY: ${{ secrets.OPENAI__APIKEY }} - OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI__CHATMODELID }} - OPENAI_CHAT_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} - OPENAI_MODEL: ${{ vars.OPENAI__RESPONSESMODELID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_CHAT_COMPLETION_MODEL: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_CHAT_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} + OPENAI_MODEL: ${{ vars.OPENAI_REASONING_MODEL_NAME }} # OpenAI configuration for SK - OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI__CHATMODELID }} + OPENAI_CHAT_MODEL_ID: ${{ vars.OPENAI_CHAT_MODEL_NAME }} + OPENAI_RESPONSES_MODEL_ID: ${{ vars.OPENAI__RESPONSESMODELID }} # Copilot Studio COPILOTSTUDIOAGENT__ENVIRONMENTID: ${{ secrets.COPILOTSTUDIOAGENT__ENVIRONMENTID }} COPILOTSTUDIOAGENT__SCHEMANAME: ${{ secrets.COPILOTSTUDIOAGENT__SCHEMANAME }} @@ -665,6 +868,10 @@ jobs: run: | cd scripts && uv run python -m sample_validation --subdir semantic-kernel-migration --save-report --report-name semantic-kernel-migration + - name: Save sample playbooks + if: ${{ !cancelled() }} + uses: ./.github/actions/sample-validation-save-playbooks + - name: Upload validation report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() @@ -679,6 +886,8 @@ jobs: needs: - validate-01-get-started - validate-02-agents + - validate-02-agents-harness + - validate-02-agents-tools - validate-02-agents-openai - validate-02-agents-azure - validate-02-agents-anthropic @@ -689,7 +898,8 @@ jobs: - validate-02-agents-copilotstudio - validate-02-agents-custom - validate-03-workflows - - validate-04-hosting + - validate-04-hosting-foundry-hosted-agents + - validate-04-hosting-other - validate-05-end-to-end - validate-autogen-migration - validate-semantic-kernel-migration diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index c5406001bd1..3c04126ce71 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -880,6 +880,8 @@ class MagenticOrchestrator(BaseGroupChatOrchestrator): 5. The outer loop handles replanning and reenters the inner loop. """ + MANAGER_NAME: ClassVar[str] = "magentic_orchestrator" + def __init__( self, manager: MagenticManagerBase, @@ -896,7 +898,7 @@ def __init__( Keyword Args: require_plan_signoff: If True, requires human approval of the initial plan before proceeding. """ - super().__init__("magentic_orchestrator", participant_registry) + super().__init__(self.MANAGER_NAME, participant_registry) self._manager = manager self._require_plan_signoff = require_plan_signoff diff --git a/python/samples/03-workflows/orchestrations/magentic.py b/python/samples/03-workflows/orchestrations/magentic.py index 263408aa5b0..cc16eb96a05 100644 --- a/python/samples/03-workflows/orchestrations/magentic.py +++ b/python/samples/03-workflows/orchestrations/magentic.py @@ -14,6 +14,7 @@ ) from agent_framework.foundry import FoundryChatClient from agent_framework.orchestrations import GroupChatRequestSentEvent, MagenticBuilder, MagenticProgressLedger +from agent_framework_orchestrations import MagenticOrchestrator from azure.identity import AzureCliCredential from dotenv import load_dotenv @@ -113,17 +114,21 @@ async def main() -> None: print("\nStarting workflow execution...") # Keep track of the last executor to format output nicely in streaming mode - last_response_id: str | None = None + last_message_id: str | None = None output_event: WorkflowEvent | None = None async for event in workflow.run(task, stream=True): - if event.type in ("intermediate", "output") and isinstance(event.data, AgentResponseUpdate): - response_id = event.data.response_id - if response_id != last_response_id: - if last_response_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_response_id = response_id - print(event.data, end="", flush=True) + if event.type in ("intermediate", "output"): + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -137,21 +142,20 @@ async def main() -> None: # Block to allow user to read the plan/progress before continuing # Note: this is for demonstration only and is not the recommended way to handle human interaction. # Please refer to `with_plan_review` for proper human interaction during planning phases. - await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") + # await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") elif event.type == "group_chat" and isinstance(event.data, GroupChatRequestSentEvent): print(f"\n[REQUEST SENT ({event.data.round_index})] to agent: {event.data.participant_name}") - elif event.type == "output": - output_event = event + if not output_event: + raise RuntimeError("Workflow did not produce a final output event.") - if output_event: - # The output of the magentic workflow is a collection of chat messages from all participants - outputs = cast(list[Message], output_event.data) - print("\n" + "=" * 80) - print("\nFinal Conversation Transcript:\n") - for message in outputs: - print(f"{message.author_name or message.role}: {message.text}\n") + print("\n\nWorkflow completed!") + print("Final Output:") + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) if __name__ == "__main__": diff --git a/python/samples/04-hosting/foundry-hosted-agents/README.md b/python/samples/04-hosting/foundry-hosted-agents/README.md index 3efbbbb22a3..e766966bceb 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/README.md @@ -19,11 +19,10 @@ This directory contains samples that demonstrate how to use hosted [Agent Framew | 6 | [Files](responses/files/) | An agent demonstrating how to work with files in a hosted agent session, including uploading files to a hosted agent session and having the agent read and manipulate those files at runtime. | | 7 | [Observability](responses/observability/) | A sample demonstrating how to enable observability for the agent deployed to Foundry. | | 8 | [Azure AI Search RAG](responses/azure_search_rag/) | An agent with Retrieval Augmented Generation (RAG) capabilities backed by Azure AI Search, grounding answers in documents indexed in a pre-provisioned search index. | -| 9 | [Foundry Skills](responses/foundry_skills/) | An agent that uploads `SKILL.md` files to the Foundry Skills REST API and downloads them at startup, decoupling tone/policy guidelines from agent code. | -| 10 | [Foundry Memory](responses/foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | -| 11 | [Monty CodeAct](responses/monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the beta `agent-framework-monty` package. | -| 12 | [Foundry Toolbox MCP Skills](responses/foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | -| 13 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | +| 9 | [Foundry Memory](responses/foundry_memory/) | An agent with persistent semantic memory backed by a Microsoft Foundry Memory Store, using `FoundryMemoryProvider` to remember user facts across sessions. | +| 10 | [Monty CodeAct](responses/monty_codeact/) | An agent with a Monty-backed CodeAct context provider, exposing a single `execute_code` tool that runs Python in a [pydantic-monty](https://github.com/pydantic/monty) interpreter and invokes typed host tools (`compute`, `fetch_data`) from inside the sandbox. Uses the beta `agent-framework-monty` package. | +| 11 | [Foundry Toolbox MCP Skills](responses/foundry_toolbox_mcp_skills/) | An agent that discovers MCP-based skills attached to a Foundry Toolbox and serves them via `SkillsProvider(MCPSkillsSource(...))`, fetching `SKILL.md` bodies and supplementary resources on demand. | +| 12 | [Using deployed agent](responses/using_deployed_agent.py) | A sample demonstrating how to invoke an agent that has already been deployed to Foundry, showing how to interact with a hosted agent in code. | ### Invocations API diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/files/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/files/main.py index e5b59cc7753..a3378661d99 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/files/main.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/files/main.py @@ -2,69 +2,16 @@ import asyncio import os -from collections.abc import Callable -from urllib.parse import urlsplit -import httpx -from agent_framework import Agent, MCPStreamableHTTPTool, tool -from agent_framework.foundry import FoundryChatClient, ResponsesHostServer -from azure.identity import DefaultAzureCredential, get_bearer_token_provider +from agent_framework import Agent, tool +from agent_framework.foundry import FoundryChatClient, FoundryToolbox, ResponsesHostServer +from azure.identity import DefaultAzureCredential from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() -def resolve_toolbox_endpoint() -> str: - """Resolve the toolbox MCP endpoint URL. - - Prefers the explicit ``TOOLBOX_ENDPOINT`` env var (set in ``agent.yaml`` or - ``agent.manifest.yaml`` and via ``azd env set TOOLBOX_ENDPOINT`` after the toolbox - is created); falls back to constructing the URL from ``FOUNDRY_PROJECT_ENDPOINT`` - and ``TOOLBOX_NAME``. - """ - if (endpoint := os.environ.get("TOOLBOX_ENDPOINT")) is not None: - if not endpoint: - raise ValueError("TOOLBOX_ENDPOINT is set but empty") - return endpoint - try: - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"].rstrip("/") - toolbox_name = os.environ["TOOLBOX_NAME"] - except KeyError as e: - raise ValueError( - "Either set TOOLBOX_ENDPOINT, or set both FOUNDRY_PROJECT_ENDPOINT " - "and TOOLBOX_NAME to build the toolbox MCP endpoint." - ) from e - return f"{project_endpoint}/toolboxes/{toolbox_name}/mcp?api-version=v1" - - -def _toolbox_name_from_endpoint(endpoint: str) -> str: - """Extract the toolbox name from a toolbox MCP endpoint URL. - - Handles both the versioned (``.../toolboxes//versions//mcp``) and - unversioned (``.../toolboxes//mcp``) endpoint shapes that Foundry - produces. Falls back to ``"toolbox"`` when the path has no ``toolboxes`` - segment. - """ - segments = urlsplit(endpoint).path.split("/") - if "toolboxes" in segments: - idx = segments.index("toolboxes") - if idx + 1 < len(segments) and segments[idx + 1]: - return segments[idx + 1] - return "toolbox" - - -class ToolboxAuth(httpx.Auth): - """Injects a fresh bearer token on every request.""" - - def __init__(self, token_provider: Callable[[], str]): - self._get_token = token_provider - - def auth_flow(self, request: httpx.Request): - request.headers["Authorization"] = f"Bearer {self._get_token()}" - yield request - - @tool(description="Get the current working directory.", approval_mode="never_require") def get_cwd() -> str: """Get the current working directory.""" @@ -96,48 +43,35 @@ def read_file(file_path: str) -> str: async def main(): credential = DefaultAzureCredential() - # Create the toolbox - token_provider = get_bearer_token_provider(credential, "https://ai.azure.com/.default") - - # Resolve the endpoint once and derive a friendly tool name from it. When - # ``TOOLBOX_NAME`` isn't set, extract the toolbox name from the URL path so - # the tool's local name matches the upstream toolbox. - toolbox_endpoint = resolve_toolbox_endpoint() - toolbox_name = os.environ.get("TOOLBOX_NAME") or _toolbox_name_from_endpoint(toolbox_endpoint) - - async with httpx.AsyncClient( - auth=ToolboxAuth(token_provider), - timeout=120.0, - ) as http_client: - toolbox = MCPStreamableHTTPTool( - name=toolbox_name, - url=toolbox_endpoint, - http_client=http_client, - load_prompts=False, - ) - - # Create the chat client - client = FoundryChatClient( - project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], - credential=credential, - ) - - agent = Agent( - client=client, - instructions=( - "You are a friendly assistant. Keep your answers brief. " - "Make sure all mathematical calculations are performed using the code interpreter " - "instead of mental arithmetic." - ), - tools=[get_cwd, list_files, read_file, toolbox], - # History will be managed by the hosting infrastructure, thus there - # is no need to store history by the service. Learn more at: - # https://developers.openai.com/api/reference/resources/responses/methods/create - default_options={"store": False}, - ) - server = ResponsesHostServer(agent) - await server.run_async() + # FoundryToolbox resolves the toolbox endpoint from the environment + # (TOOLBOX_ENDPOINT, or FOUNDRY_PROJECT_ENDPOINT + TOOLBOX_NAME), authenticates + # every request with the credential, and transparently forwards the platform + # per-request call-id to the toolbox. The hosting server enters the agent, which + # connects the toolbox on first use and closes it at shutdown. + toolbox = FoundryToolbox(credential) + + # Create the chat client + client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], + credential=credential, + ) + + agent = Agent( + client=client, + instructions=( + "You are a friendly assistant. Keep your answers brief. " + "Make sure all mathematical calculations are performed using the code interpreter " + "instead of mental arithmetic." + ), + tools=[get_cwd, list_files, read_file, toolbox], + # History will be managed by the hosting infrastructure, thus there + # is no need to store history by the service. Learn more at: + # https://developers.openai.com/api/reference/resources/responses/methods/create + default_options={"store": False}, + ) + server = ResponsesHostServer(agent) + await server.run_async() if __name__ == "__main__": diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.dockerignore b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.dockerignore deleted file mode 100644 index d7a4f0d7fa3..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -.venv -__pycache__ -*.pyc -*.pyo -*.pyd -.Python -.env -provision_skills.py -skills -downloaded_skills diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.env.example b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.env.example deleted file mode 100644 index b80b157d96c..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.env.example +++ /dev/null @@ -1,6 +0,0 @@ -FOUNDRY_PROJECT_ENDPOINT="..." -AZURE_AI_MODEL_DEPLOYMENT_NAME="..." -# Comma-separated list of Foundry skill names to download at startup. -SKILL_NAMES="support-style,escalation-policy" -# Optional writable directory for downloaded skills. Defaults to the system temp directory. -# DOWNLOADED_SKILLS_DIR="/tmp/maf_downloaded_skills" diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.gitignore b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.gitignore deleted file mode 100644 index ae8a1dfbe89..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/.gitignore +++ /dev/null @@ -1 +0,0 @@ -downloaded_skills/ diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/Dockerfile b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/Dockerfile deleted file mode 100644 index 0cc939d9b3a..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.12-slim - -WORKDIR /app - -COPY . user_agent/ -WORKDIR /app/user_agent - -RUN if [ -f requirements.txt ]; then \ - pip install -r requirements.txt; \ - else \ - echo "No requirements.txt found"; \ - fi - -EXPOSE 8088 - -CMD ["python", "main.py"] diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/README.md b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/README.md deleted file mode 100644 index d11ea8096db..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/README.md +++ /dev/null @@ -1,139 +0,0 @@ -# What this sample demonstrates - -An [Agent Framework](https://github.com/microsoft/agent-framework) agent that loads its behavioral guidelines from [**Foundry Skills**](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/tools/skills?view=foundry&pivots=python) at startup, hosted using the **Responses protocol**. Skills are authored once as `SKILL.md` files, uploaded to your Foundry project through `AIProjectClient.beta.skills`, and downloaded by the agent on boot so updates ship without code changes. - -## How It Works - -### Authoring skills - -Each skill is a Markdown file with a YAML front matter block. This sample ships two source skills under [`skills/`](skills/): - -| Skill | Purpose | -|---|---| -| [`support-style`](skills/support-style/SKILL.md) | Voice, formatting, and signature rules for Contoso Outdoors support replies. | -| [`escalation-policy`](skills/escalation-policy/SKILL.md) | When and how to escalate a customer ticket. | - -Each `SKILL.md` includes a unique `*-CANARY-*` token that the model is asked to echo, so you can prove the skill was loaded from Foundry (not hallucinated) by checking the response. - -> The `name` and `description` values in the YAML front matter must be **unquoted** — quoting them causes the Skills REST API to return HTTP 500 on import. - -### Uploading skills with `AIProjectClient` - -[`provision_skills.py`](provision_skills.py) walks `skills/*/SKILL.md`, packages each file as an in-memory ZIP (with `SKILL.md` at the archive root), and imports it through [`AIProjectClient.beta.skills.create_from_package`](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/tools/skills?view=foundry&pivots=python#option-2-import-from-a-skillmd-zip). The client is constructed with `allow_preview=True` (Skills is a preview feature) and authenticates with `DefaultAzureCredential`. Existing skills are deleted first via `beta.skills.delete` so the script is safe to re-run after editing a `SKILL.md`, and `beta.skills.list` is called at the end to verify each skill round-trips. - -### Downloading skills at agent startup - -[`main.py`](main.py) reads the comma-separated `SKILL_NAMES` env var, opens an `AIProjectClient` (also with `allow_preview=True`), and for each skill name streams the ZIP archive from `beta.skills.download(name)` and unpacks it into a **separate writable runtime directory**. By default this directory is created under the system temp folder as `maf_downloaded_skills//`, which works in hosted containers where the application directory may be read-only. Set `DOWNLOADED_SKILLS_DIR` to override the location. - -A [`SkillsProvider`](../../../../../packages/core/agent_framework/_skills.py) is then built over the downloaded skills directory and attached to the `Agent` as a context provider. The provider follows the [Agent Skills](https://agentskills.io/) progressive-disclosure pattern: - -1. **Advertise** — skill names and descriptions are injected into the system prompt at session start (~100 tokens per skill). -2. **Load** — the model calls the `load_skill` tool when it decides a skill is relevant to the user's turn, and the full `SKILL.md` body is returned. - -This means the model only pays the token cost for a skill's full body when it actually needs it, and updating a skill in Foundry + restarting the agent is enough to pick up the change — no code redeploy required. - -### Agent Hosting - -The agent is hosted using the [Agent Framework](https://github.com/microsoft/agent-framework) with the `ResponsesHostServer`, which provisions a REST API endpoint compatible with the OpenAI Responses protocol. - -## Prerequisites - -- A Microsoft Foundry project with a deployed model (e.g., `gpt-4.1-mini`) -- Azure CLI logged in (`az login`) - -### Required RBAC - -Your identity (or the Managed Identity running the container in production) needs **Azure AI User** on the Foundry project scope. This single role covers both authoring skills with `provision_skills.py` and downloading them from `main.py`. - -## Provisioning the skills (one time) - -From this directory, with the venv activated and `az login` done: - -```bash -export FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" -python provision_skills.py -``` - -Or in PowerShell: - -```powershell -$env:FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" -python provision_skills.py -``` - -Expected output: - -```text -Provisioning skill 'escalation-policy' from skills/escalation-policy/SKILL.md... - Imported skill 'escalation-policy' (id=skill_..., has_blob=True). -Provisioning skill 'support-style' from skills/support-style/SKILL.md... - Imported skill 'support-style' (id=skill_..., has_blob=True). -Done. -``` - -Re-running the script after editing a `SKILL.md` re-imports the skill, replacing the previous version. - -> To remove a skill manually, call `project.beta.skills.delete("")` on an `AIProjectClient` constructed with `allow_preview=True`. - -## Running the Agent Host - -Follow the instructions in the [Running the Agent Host Locally](../../README.md#running-the-agent-host-locally) section of the README in the parent directory to run the agent host. - -In addition to the standard environment variables, this sample requires: - -```bash -export SKILL_NAMES="support-style,escalation-policy" -``` - -Or in PowerShell: - -```powershell -$env:SKILL_NAMES="support-style,escalation-policy" -``` - -You can also place these in a `.env` file next to `main.py` — see [`.env.example`](.env.example). - -On startup you should see: - -```text -Downloading skill 'support-style' from Foundry... -Downloading skill 'escalation-policy' from Foundry... -``` - -The downloaded `SKILL.md` files land under `DOWNLOADED_SKILLS_DIR//SKILL.md`. The directory is recreated from scratch on every run, so deleting it manually is never necessary. - -By default, the sample uses the system temp directory, for example `/tmp/maf_downloaded_skills` on Linux. To choose a different writable location, set `DOWNLOADED_SKILLS_DIR` before startup. - -## Interacting with the agent - -> Depending on how you run the agent host, you can invoke the agent using `curl` (`Invoke-WebRequest` in PowerShell) or `azd`. Please refer to the [parent README](../../README.md) for more details. Use this README for sample queries you can send to the agent. - -Send a POST request to the server with a JSON body containing an `"input"` field to interact with the agent. For example: - -```bash -curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" -d '{"input": "Hi, I am Alex. I just want to confirm I can return my tent within 30 days."}' -curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" -d '{"input": "I want a $750 refund on Order #A-1042 right now or I am calling my lawyer."}' -``` - -| Prompt mentions | Skill that should drive the response | -|---|---| -| Routine return / shipping / care question | Model loads `support-style` (canary `STYLE-CANARY-3318`) — no escalation. | -| Injury, legal threat, press, or refund > $500 | Model loads `escalation-policy` (canary `ESC-CANARY-7742`) **and** `support-style`. | - -Because skills are loaded on demand, the canary token in a response also proves the model actually invoked `load_skill` for the matching skill (not just saw its name in the advertised list). - -## Deploying the Agent to Foundry - -To host the agent on Foundry, follow the instructions in the [Deploying the Agent to Foundry](../../README.md#deploying-the-agent-to-foundry) section of the README in the parent directory. - -When deploying, make sure `SKILL_NAMES` is set in your `azd` environment so it gets injected into the hosted container per [`agent.manifest.yaml`](agent.manifest.yaml): - -```bash -azd env set SKILL_NAMES "support-style,escalation-policy" -``` - -If it is not set, running `azd ai agent init -m ` will prompt you to enter it interactively. - -The deployed agent's Managed Identity needs **Azure AI User** on the Foundry project to download skills at startup. Make sure you have run `provision_skills.py` against the same Foundry project before deploying — otherwise the agent will fail to start with HTTP 404 on the skill download. - -> The `skills/` source folder is **not** deployed to Foundry — only the downloaded skills are used at runtime. The `provision_skills.py` step is required to upload the skills to Foundry before the agent can download them. \ No newline at end of file diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.manifest.yaml b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.manifest.yaml deleted file mode 100644 index 16fe4ef66fd..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.manifest.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: agent-framework-agent-foundry-skills-responses -description: > - An Agent Framework agent that downloads its instructions from the Foundry - Skills REST API at startup, demonstrating how to decouple behavioral - guidelines (tone, escalation policy, etc.) from agent code. -metadata: - tags: - - Agent Framework - - AI Agent Hosting - - Azure AI AgentServer - - Responses Protocol - - Foundry Skills -template: - name: agent-framework-agent-foundry-skills-responses - kind: hosted - protocols: - - protocol: responses - version: 2.0.0 - environment_variables: - - name: AZURE_AI_MODEL_DEPLOYMENT_NAME - value: "{{AZURE_AI_MODEL_DEPLOYMENT_NAME}}" - - name: SKILL_NAMES - value: "{{SKILL_NAMES}}" -parameters: - properties: - - name: SKILL_NAMES - secret: false - description: Comma-separated list of Foundry skill names to download at startup (e.g., support-style,escalation-policy) -resources: - - kind: model - id: gpt-4.1-mini - name: AZURE_AI_MODEL_DEPLOYMENT_NAME diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.yaml b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.yaml deleted file mode 100644 index 3a7a14d0bcc..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml -kind: hosted -name: agent-framework-agent-foundry-skills-responses -protocols: - - protocol: responses - version: 2.0.0 -resources: - cpu: "0.25" - memory: "0.5Gi" -environment_variables: - - name: AZURE_AI_MODEL_DEPLOYMENT_NAME - value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME} - - name: SKILL_NAMES - value: ${SKILL_NAMES} diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/main.py b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/main.py deleted file mode 100644 index 26ecf1fa617..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/main.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -"""Foundry Skills hosted agent sample. - -At startup, this agent downloads each Foundry Skill named in -``SKILL_NAMES`` from the project's ``beta.skills`` API, unpacks each -one into a separate writable runtime directory and wires -that directory into a :class:`SkillsProvider` so the agent advertises the -skills to the model and loads them on demand (progressive disclosure). - -Upload the skills to Foundry once with ``provision_skills.py`` before running -this sample. -""" - -import asyncio -import io -import logging -import os -import shutil -import tempfile -import zipfile -from pathlib import Path -from typing import Final - -from agent_framework import Agent, SkillsProvider -from agent_framework.foundry import FoundryChatClient, ResponsesHostServer -from azure.ai.projects.aio import AIProjectClient -from azure.identity.aio import DefaultAzureCredential -from dotenv import load_dotenv - -load_dotenv() - -# Runtime directory where skills downloaded from Foundry are unpacked. -# Kept separate from the static ``skills/`` source folder so the two never -# get confused: the source folder is the input to ``provision_skills.py`` -# and the runtime folder is the output of this script's bootstrap step. -# Defaults to a system temp location because hosted containers may mount the -# application directory read-only. Set DOWNLOADED_SKILLS_DIR to override it. -_DEFAULT_DOWNLOADED_SKILLS_DIR: Final = Path(tempfile.gettempdir()) / "maf_downloaded_skills" -_DOWNLOADED_SKILLS_DIR_ENV = os.environ.get("DOWNLOADED_SKILLS_DIR") -DOWNLOADED_SKILLS_DIR: Final = Path((_DOWNLOADED_SKILLS_DIR_ENV or "").strip() or _DEFAULT_DOWNLOADED_SKILLS_DIR) - -logger = logging.getLogger(__name__) - - -def _safe_extract_zip(zf: zipfile.ZipFile, dest_dir: Path) -> None: - """Extract ``zf`` into ``dest_dir``, rejecting entries that escape it (zip-slip guard).""" - dest_root = dest_dir.resolve() - for member in zf.infolist(): - member_path = (dest_root / member.filename).resolve() - if dest_root != member_path and dest_root not in member_path.parents: - raise RuntimeError(f"Refusing to extract unsafe path '{member.filename}' outside of '{dest_root}'.") - zf.extractall(dest_dir) - - -async def _bootstrap_skills(endpoint: str, skill_names: list[str], target_dir: Path) -> None: - """Download each named skill via ``project.beta.skills`` and unpack it as ``//SKILL.md``.""" - if target_dir.exists(): # noqa: ASYNC240 - shutil.rmtree(target_dir) - target_dir.mkdir(parents=True) # noqa: ASYNC240 - - async with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project, - ): - for name in skill_names: - logger.info(f"Downloading skill '{name}' from Foundry...") - stream = await project.beta.skills.download(name) - zip_bytes = b"".join([chunk async for chunk in stream]) - skill_dir = target_dir / name - skill_dir.mkdir() - with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf: - _safe_extract_zip(zf, skill_dir) - if not (skill_dir / "SKILL.md").is_file(): - raise RuntimeError(f"Downloaded archive for '{name}' did not contain a SKILL.md at the root.") - - -async def main() -> None: - project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - skill_names = [name.strip() for name in os.environ["SKILL_NAMES"].split(",") if name.strip()] - if not skill_names: - raise RuntimeError("SKILL_NAMES must list at least one skill name.") - - # Pull the latest copy of each skill from Foundry into a runtime-only folder. - await _bootstrap_skills(project_endpoint, skill_names, DOWNLOADED_SKILLS_DIR) - - # Build a SkillsProvider over the unpacked folder. The provider advertises - # each skill's name + description to the model and exposes the ``load_skill`` - # tool the model uses to retrieve the full SKILL.md body on demand. No - # script_runner is configured because the skills in this sample are - # instruction-only. - skills_provider = SkillsProvider.from_paths(skill_paths=str(DOWNLOADED_SKILLS_DIR)) - - async with DefaultAzureCredential() as credential: - client = FoundryChatClient( - project_endpoint=project_endpoint, - model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], - credential=credential, - ) - - agent = Agent( - client=client, - instructions="You are a customer-support assistant for Contoso Outdoors.", - context_providers=[skills_provider], - # History will be managed by the hosting infrastructure, thus there - # is no need to store history by the service. Learn more at: - # https://developers.openai.com/api/reference/resources/responses/methods/create - default_options={"store": False}, - ) - server = ResponsesHostServer(agent) - await server.run_async() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/provision_skills.py b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/provision_skills.py deleted file mode 100644 index 5f501d61e12..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/provision_skills.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -"""Provision Foundry Skills used by this sample. - -For each ``skills//SKILL.md`` file in this directory, this script packages -the file as an in-memory ZIP and imports it through the Foundry project's -:class:`~azure.ai.projects.aio.AIProjectClient` so the skill becomes downloadable -by any hosted agent in the project. - -If a skill with the same name already exists in Foundry, it is deleted first -so the script is safe to re-run after editing a ``SKILL.md`` file. - -Usage (from this directory, with the venv activated and ``az login`` done): - - python provision_skills.py - -Required env vars (also read from a local ``.env`` file if present): - - FOUNDRY_PROJECT_ENDPOINT e.g. https://.services.ai.azure.com/api/projects/ - -Your identity needs the ``Azure AI User`` role on the Foundry project. -""" - -import asyncio -import io -import os -import zipfile -from pathlib import Path - -from azure.ai.projects.aio import AIProjectClient -from azure.ai.projects.models import CreateSkillVersionFromFilesBody -from azure.core.exceptions import ResourceNotFoundError -from azure.identity.aio import DefaultAzureCredential -from dotenv import load_dotenv - -SKILLS_DIR = Path(__file__).parent / "skills" - - -def _zip_skill_md(skill_md: Path) -> bytes: - """Return the bytes of a ZIP archive containing ``SKILL.md`` at the root.""" - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: - zf.writestr("SKILL.md", skill_md.read_text(encoding="utf-8")) - return buffer.getvalue() - - -async def _delete_skill_if_exists(project: AIProjectClient, name: str) -> None: - try: - await project.beta.skills.delete(name) - except ResourceNotFoundError: - return - print(f" Deleted existing skill '{name}'.") - - -async def main() -> None: - load_dotenv() - - endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] - - skill_files = sorted(SKILLS_DIR.glob("*/SKILL.md")) - if not skill_files: - raise RuntimeError(f"No SKILL.md files found under {SKILLS_DIR}.") - - async with ( - DefaultAzureCredential() as credential, - AIProjectClient(endpoint=endpoint, credential=credential, allow_preview=True) as project, - ): - for skill_md in skill_files: - name = skill_md.parent.name - print(f"Provisioning skill '{name}' from {skill_md.relative_to(SKILLS_DIR.parent)}...") - await _delete_skill_if_exists(project, name) - imported = await project.beta.skills.create_from_files( - name, - content=CreateSkillVersionFromFilesBody( - files=[(f"{name}.zip", _zip_skill_md(skill_md), "application/zip")] - ), - ) - print(f" Imported skill '{imported.name}' (id={imported.skill_id}, version={imported.version}).") - - print("Verifying skills via project.beta.skills.list()...") - listed = {skill.name: skill async for skill in project.beta.skills.list()} - for skill_md in skill_files: - name = skill_md.parent.name - skill = listed.get(name) - if skill is None: - raise RuntimeError(f"Skill '{name}' was imported but is not present in the project listing.") - print( - f" OK '{skill.name}': id={skill.id}, " - f"description={skill.description!r}, default_version={skill.default_version}" - ) - - print("Done.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/requirements.txt b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/requirements.txt deleted file mode 100644 index 15061699375..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -agent-framework-foundry -agent-framework-foundry-hosting>=1.0.0a260630 -azure-ai-projects diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/escalation-policy/SKILL.md b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/escalation-policy/SKILL.md deleted file mode 100644 index ec04c9aa86d..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/escalation-policy/SKILL.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: escalation-policy -description: When and how to escalate Contoso Outdoors customer-support tickets. ---- - -# Contoso Outdoors Escalation Policy - -You must follow this escalation policy on every conversation. - -## Escalate immediately when the customer - -- Reports an injury, allergic reaction, or other safety incident. -- Mentions legal action, regulators, or the press. -- Has waited more than 14 days for a refund that was already approved. -- Requests a refund larger than $500. - -## How to escalate - -1. Acknowledge the issue in one sentence. -2. Tell the customer you are escalating to a senior specialist. -3. Provide the escalation reference `ESC-CANARY-7742` and the SLA: a senior - specialist will reply within 1 business day. -4. Do not promise a specific outcome (refund, replacement, compensation) on - escalated tickets — only the senior specialist can commit to one. - -## Do not escalate - -- Routine returns within the standard 30-day window. -- Shipping status questions. -- Product care and usage questions. diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/support-style/SKILL.md b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/support-style/SKILL.md deleted file mode 100644 index a516811a488..00000000000 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_skills/skills/support-style/SKILL.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: support-style -description: Contoso Outdoors customer-support tone and formatting guidelines. ---- - -# Contoso Outdoors Support Style - -You are speaking on behalf of Contoso Outdoors customer support. - -## Voice - -- Warm, concise, and confident — never apologetic in a hand-wringing way. -- Use the customer's name when it is known. -- Sign every response with `— Contoso Outdoors Support`. - -## Formatting - -- Keep replies to 1–3 short paragraphs unless the customer asks for detail. -- Use bullet lists only when enumerating concrete steps or options. -- Always reference order numbers as `Order #` (e.g. `Order #A-1042`). - -## Canary - -To prove this skill was loaded, include the token `STYLE-CANARY-3318` on a -separate line at the bottom of every response, prefixed with `# `. diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_toolbox_mcp_skills/README.md b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_toolbox_mcp_skills/README.md index 489fc493727..e4a5e7af4c0 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_toolbox_mcp_skills/README.md +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/foundry_toolbox_mcp_skills/README.md @@ -11,13 +11,9 @@ The `FoundryToolbox` is attached to the agent and its skills are exposed through 1. **Advertise** — each skill's name and description are injected into the system prompt so the model knows what is available (~100 tokens per skill). 2. **Load** — when the model decides a skill is relevant, it retrieves the full `SKILL.md` body on demand via `resources/read`. -> The Agent Skills spec defines a third stage — **read resources** — where a skill fetches supplementary files (reference documents, assets) on demand. That stage requires skills to be served as `type: skill-md` with sibling resources, but Foundry serves ZIP-uploaded (multi-file) skills as `type: archive`, which toolbox skill discovery does not currently surface. So this sample keeps both skills as single-file `SKILL.md` (advertise + load only). See the [`foundry_skills`](../foundry_skills/README.md) sample for the same instruction-only pattern via direct download. +> The Agent Skills spec defines a third stage — **read resources** — where a skill fetches supplementary files (reference documents, assets) on demand. That stage requires skills to be served as `type: skill-md` with sibling resources, but Foundry serves ZIP-uploaded (multi-file) skills as `type: archive`, which toolbox skill discovery does not currently surface. So this sample keeps both skills as single-file `SKILL.md` (advertise + load only). -## Toolbox MCP skills vs. Foundry Skills - -Foundry exposes skills in two ways, and this sample uses the second one. - -**Foundry Skills** are downloaded directly into an agent: the agent pulls each `SKILL.md` from the Skills API at startup and serves the bodies from local files. See the [`foundry_skills`](../foundry_skills/README.md) sample. +## Toolbox MCP skills **Toolbox MCP skills** are accessed through a toolbox over the MCP protocol. A toolbox bundles a curated set of skills (and optionally tools) behind one MCP endpoint, and any MCP client discovers them automatically. Skill bodies are fetched on demand. The same `SKILL.md` files power both modes — the difference is only in delivery. @@ -37,8 +33,6 @@ The agent is hosted with the `ResponsesHostServer`, which provisions a REST API ## The bundled skills -This sample ships two source skills under [`skills/`](skills/), reused from the [`foundry_skills`](../foundry_skills/README.md) sample so you can compare the two delivery modes side by side: - | Skill | Purpose | |---|---| | [`support-style`](skills/support-style/SKILL.md) | Voice, formatting, and signature rules for Contoso Outdoors support replies. | diff --git a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py index 9cef754ef3f..6f574b41252 100644 --- a/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py +++ b/python/samples/04-hosting/foundry-hosted-agents/responses/using_deployed_agent.py @@ -4,8 +4,7 @@ import asyncio import os -from collections.abc import Mapping -from typing import Any, cast +from typing import cast from agent_framework import AgentSession from agent_framework.foundry import FoundryAgent @@ -35,47 +34,37 @@ """ -def get_hosted_session_agents(project_client: AIProjectClient) -> Any: - """Return the hosted-session operations for azure-ai-projects 2.2 or 2.3.""" - session_agents = cast(Any, project_client.agents) - if hasattr(session_agents, "create_session"): - return session_agents - return cast(Any, project_client.beta.agents) - - async def create_hosted_agent_session( *, agent: FoundryAgent, project_client: AIProjectClient, agent_name: str, agent_version: str | None, - isolation_key: str, ) -> AgentSession: """Create a hosted-agent service session and wrap it in an AgentSession.""" - create_session_kwargs: dict[str, Any] = { - "agent_name": agent_name, - "isolation_key": isolation_key, - } resolved_agent_version = agent_version if resolved_agent_version is None: - agent_details = await cast(Any, project_client.beta.agents).get( # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] - agent_name=agent_name - ) - versions = getattr(agent_details, "versions", None) - if not isinstance(versions, Mapping): - raise ValueError("Hosted agent details did not include a versions mapping.") - latest_version = getattr(cast(Any, versions.get("latest")), "version", None) - if not isinstance(latest_version, str) or not latest_version: - raise ValueError("Hosted agent details did not include a latest version string.") - resolved_agent_version = latest_version + agent_details = await project_client.agents.get(agent_name) + resolved_agent_version = agent_details.versions.latest.version + + service_session = await project_client.agents.create_session( + agent_name, + version_indicator=VersionRefIndicator(agent_version=resolved_agent_version), + ) + return agent.get_session(service_session.agent_session_id) - create_session_kwargs["version_indicator"] = VersionRefIndicator(agent_version=resolved_agent_version) - service_session = await get_hosted_session_agents(project_client).create_session(**create_session_kwargs) - agent_session_id = getattr(service_session, "agent_session_id", None) - if not isinstance(agent_session_id, str) or not agent_session_id: - raise ValueError("Hosted agent session creation did not return a non-empty agent_session_id.") - return agent.get_session(agent_session_id) +async def delete_hosted_agent_session( + *, + project_client: AIProjectClient, + agent_name: str, + session: AgentSession, +) -> None: + """Delete a hosted-agent service session.""" + await project_client.agents.delete_session( + agent_name, + cast(str, session.service_session_id), + ) async def main() -> None: @@ -83,7 +72,6 @@ async def main() -> None: project_endpoint = os.environ["FOUNDRY_PROJECT_ENDPOINT"] agent_name = os.environ["FOUNDRY_AGENT_NAME"] agent_version = os.getenv("FOUNDRY_AGENT_VERSION") - isolation_key = "my-isolation-key" project_client = AIProjectClient( endpoint=project_endpoint, @@ -104,7 +92,6 @@ async def main() -> None: project_client=project_client, agent_name=agent_name, agent_version=agent_version, - isolation_key=isolation_key, ) try: @@ -132,12 +119,11 @@ async def main() -> None: if chunk.text: print(chunk.text, end="", flush=True) finally: - if isinstance(session.service_session_id, str): - await get_hosted_session_agents(project_client).delete_session( - agent_name=agent_name, - session_id=session.service_session_id, - isolation_key=isolation_key, - ) + await delete_hosted_agent_session( + project_client=project_client, + agent_name=agent_name, + session=session, + ) if __name__ == "__main__": diff --git a/python/samples/autogen-migration/orchestrations/04_magentic_one.py b/python/samples/autogen-migration/orchestrations/04_magentic_one.py index 9a7dec30eee..736d07408ce 100644 --- a/python/samples/autogen-migration/orchestrations/04_magentic_one.py +++ b/python/samples/autogen-migration/orchestrations/04_magentic_one.py @@ -10,7 +10,7 @@ Message, WorkflowEvent, ) -from agent_framework.orchestrations import MagenticProgressLedger +from agent_framework.orchestrations import MagenticOrchestrator, MagenticProgressLedger from dotenv import load_dotenv """AutoGen MagenticOneGroupChat vs Agent Framework MagenticBuilder. @@ -25,7 +25,6 @@ async def run_autogen() -> None: """AutoGen's MagenticOneGroupChat for orchestrated collaboration.""" - from autogen_agentchat.agents import AssistantAgent from autogen_agentchat.teams import MagenticOneGroupChat from autogen_agentchat.ui import Console @@ -62,7 +61,7 @@ async def run_autogen() -> None: team = MagenticOneGroupChat( participants=[researcher, coder, reviewer], model_client=client, # Coordinator uses this client - max_turns=20, + max_turns=10, max_stalls=3, ) @@ -109,7 +108,7 @@ async def run_agent_framework() -> None: instructions="You coordinate a team to complete complex tasks efficiently.", description="Orchestrator for team coordination", ), - max_round_count=20, + max_round_count=10, max_stall_count=3, max_reset_count=1, ).build() @@ -119,14 +118,18 @@ async def run_agent_framework() -> None: output_event: WorkflowEvent | None = None print("[Agent Framework] Magentic conversation:") async for event in workflow.run("Research Python async patterns and write a simple example", stream=True): - if event.type == "output" and isinstance(event.data, AgentResponseUpdate): - message_id = event.data.message_id - if message_id != last_message_id: - if last_message_id is not None: - print("\n") - print(f"- {event.executor_id}:", end=" ", flush=True) - last_message_id = message_id - print(event.data, end="", flush=True) + if event.type == "output": + if event.executor_id == MagenticOrchestrator.MANAGER_NAME: + output_event = event + else: + event_data = cast(AgentResponseUpdate, event.data) + message_id = event_data.message_id + if message_id != last_message_id: + if last_message_id is not None: + print("\n") + print(f"- {event.executor_id}:", end=" ", flush=True) + last_message_id = message_id + print(event_data, end="", flush=True) elif event.type == "magentic_orchestrator": print(f"\n[Magentic Orchestrator Event] Type: {event.data.event_type.name}") @@ -142,19 +145,15 @@ async def run_agent_framework() -> None: # Please refer to `with_plan_review` for proper human interaction during planning phases. await asyncio.get_event_loop().run_in_executor(None, input, "Press Enter to continue...") - elif event.type == "output": - output_event = event - if not output_event: raise RuntimeError("Workflow did not produce a final output event.") + print("\n\nWorkflow completed!") print("Final Output:") - # The output of the Magentic workflow is a list of ChatMessages with only one final message - # generated by the orchestrator. - output_messages = cast(list[Message], output_event.data) - if output_messages: - output = output_messages[-1].text - print(output) + # The output of the Magentic workflow is an AgentResponse or AgentResponseUpdate, + # which contains the final message text. + output_message = cast(AgentResponseUpdate, output_event.data) + print(output_message.text) async def main() -> None: diff --git a/python/samples/autogen-migration/single_agent/01_basic_agent.py b/python/samples/autogen-migration/single_agent/01_basic_agent.py index 204f50b93d5..a9bcb3da359 100644 --- a/python/samples/autogen-migration/single_agent/01_basic_agent.py +++ b/python/samples/autogen-migration/single_agent/01_basic_agent.py @@ -4,10 +4,11 @@ # "agent-framework-openai", # "autogen-agentchat", # "autogen-ext[openai]", +# "python-dotenv", # ] # /// # Run with any PEP 723 compatible runner, e.g.: -# uv run samples/autogen-migration/single_agent/01_basic_assistant_agent.py +# uv run samples/autogen-migration/single_agent/01_basic_agent.py # Copyright (c) Microsoft. All rights reserved. diff --git a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py index 75cebac2f45..8cae2710e2f 100644 --- a/python/samples/autogen-migration/single_agent/02_agent_with_tool.py +++ b/python/samples/autogen-migration/single_agent/02_agent_with_tool.py @@ -1,3 +1,15 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "agent-framework-openai", +# "autogen-agentchat", +# "autogen-ext[openai]", +# "python-dotenv", +# ] +# /// +# Run with any PEP 723 compatible runner, e.g.: +# uv run samples/autogen-migration/single_agent/02_agent_with_tool.py + # Copyright (c) Microsoft. All rights reserved. import asyncio diff --git a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py index 34549142b66..d4493b22a91 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py +++ b/python/samples/semantic-kernel-migration/chat_completion/01_basic_chat_completion.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py index 7a5dfc6c279..21397b62c9a 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py +++ b/python/samples/semantic-kernel-migration/chat_completion/02_chat_completion_with_tool.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py index 57789c09c94..b6fe966fe67 100644 --- a/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py +++ b/python/samples/semantic-kernel-migration/chat_completion/03_chat_completion_thread_and_stream.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py index 363a92c54bd..240a03440dd 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py @@ -2,11 +2,21 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/01_basic_copilot_studio_agent.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Call a Copilot Studio agent with SK and Agent Framework.""" diff --git a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py index 0b96492e057..991b4b5bf1c 100644 --- a/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +++ b/python/samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py @@ -2,11 +2,21 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-copilotstudio", +# "python-dotenv", # "semantic-kernel", # ] # /// # Run with any PEP 723 compatible runner, e.g.: # uv run samples/semantic-kernel-migration/copilot_studio/02_copilot_studio_streaming.py +# +# NOTE: The metadata above resolves the Agent Framework half only. +# The Semantic Kernel half (run_semantic_kernel) requires the older +# dot-namespace Microsoft Agents SDK (microsoft.agents.copilotstudio.client and +# microsoft.agents.core, from microsoft-agents-copilotstudio-client<0.3), while +# Agent Framework requires the newer underscore-namespace SDK +# (microsoft_agents.copilotstudio.client, from +# microsoft-agents-copilotstudio-client>=0.3.1). These two generations cannot be +# installed in the same environment, so run each half in its own isolated env. # Copyright (c) Microsoft. All rights reserved. """Stream responses from Copilot Studio agents in SK and AF.""" diff --git a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py index 18b8d3ba4b6..b360da34e8f 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py +++ b/python/samples/semantic-kernel-migration/openai_responses/01_basic_responses_agent.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py index cb4ffd83575..09b68fb7d00 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py +++ b/python/samples/semantic-kernel-migration/openai_responses/02_responses_agent_with_tool.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py index d5e0e58e8e7..04fad5754ce 100644 --- a/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py +++ b/python/samples/semantic-kernel-migration/openai_responses/03_responses_agent_structured_output.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-openai", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py index f5c6682c806..1a9470b7135 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/concurrent_basic.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py index c13408bf316..8c64bcb4c13 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/group_chat.py +++ b/python/samples/semantic-kernel-migration/orchestrations/group_chat.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/handoff.py b/python/samples/semantic-kernel-migration/orchestrations/handoff.py index 3e34c8e51f3..08f77400611 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/handoff.py +++ b/python/samples/semantic-kernel-migration/orchestrations/handoff.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/magentic.py b/python/samples/semantic-kernel-migration/orchestrations/magentic.py index 3a9204f8c72..5e03f0a3037 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/magentic.py +++ b/python/samples/semantic-kernel-migration/orchestrations/magentic.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/orchestrations/sequential.py b/python/samples/semantic-kernel-migration/orchestrations/sequential.py index 59baa191651..c40792561d2 100644 --- a/python/samples/semantic-kernel-migration/orchestrations/sequential.py +++ b/python/samples/semantic-kernel-migration/orchestrations/sequential.py @@ -3,6 +3,7 @@ # dependencies = [ # "agent-framework-openai", # "agent-framework-orchestrations", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py index a911eb3461a..4463de36c40 100644 --- a/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py +++ b/python/samples/semantic-kernel-migration/processes/fan_out_fan_in_process.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/samples/semantic-kernel-migration/processes/nested_process.py b/python/samples/semantic-kernel-migration/processes/nested_process.py index 24572023e12..61a9aef7ee6 100644 --- a/python/samples/semantic-kernel-migration/processes/nested_process.py +++ b/python/samples/semantic-kernel-migration/processes/nested_process.py @@ -2,6 +2,7 @@ # requires-python = ">=3.10" # dependencies = [ # "agent-framework-core", +# "python-dotenv", # "semantic-kernel", # ] # /// diff --git a/python/scripts/sample_validation/__main__.py b/python/scripts/sample_validation/__main__.py index 948fed3a307..f0b93b2b67c 100644 --- a/python/scripts/sample_validation/__main__.py +++ b/python/scripts/sample_validation/__main__.py @@ -17,6 +17,7 @@ import argparse import asyncio +import logging import os import sys import time @@ -29,6 +30,8 @@ from sample_validation.report import save_report from sample_validation.workflow import ValidationConfig, create_validation_workflow +logging.basicConfig(level=logging.INFO) + def parse_arguments() -> argparse.Namespace: """Parse command line arguments.""" @@ -82,6 +85,33 @@ def parse_arguments() -> argparse.Namespace: help="Subdirectory paths to exclude (relative to the search directory set by --subdir)", ) + parser.add_argument( + "--playbooks-dir", + type=str, + default="./sample_validation/playbooks", + help=( + "Directory (relative to samples/) where cached per-sample playbooks are stored and " + "reused across runs (default: ./sample_validation/playbooks)" + ), + ) + + parser.add_argument( + "--no-cache", + action="store_true", + help="Ignore cached playbooks and always validate every sample with the agent", + ) + + parser.add_argument( + "--agent-timeout", + type=int, + default=120, + help=( + "Per-turn timeout in seconds for the GitHub Copilot agent while validating a sample. " + "Increase for long-running samples such as hosted agents that start a server and make " + "multiple calls (default: 120)" + ), + ) + return parser.parse_args() @@ -107,14 +137,23 @@ async def main() -> int: ) # Create validation config + playbooks_dir = (samples_dir / args.playbooks_dir).resolve() config = ValidationConfig( samples_dir=samples_dir, python_root=python_root, subdir=args.subdir, exclude=args.exclude, max_parallel_workers=max(1, args.max_parallel_workers), + playbooks_dir=playbooks_dir, + use_cache=not args.no_cache, + agent_timeout=max(1, args.agent_timeout), ) + if config.use_cache: + print(f"Playbook cache: {playbooks_dir}") + else: + print("Playbook cache: disabled (--no-cache)") + # Create and run the workflow workflow = create_validation_workflow(config) diff --git a/python/scripts/sample_validation/create_dynamic_workflow_executor.py b/python/scripts/sample_validation/create_dynamic_workflow_executor.py index 6ebe25a8d4e..e7436cf64ad 100644 --- a/python/scripts/sample_validation/create_dynamic_workflow_executor.py +++ b/python/scripts/sample_validation/create_dynamic_workflow_executor.py @@ -3,10 +3,12 @@ import logging from collections import deque from dataclasses import dataclass +from pathlib import Path from agent_framework import ( Executor, Message, + SkillsProvider, Workflow, WorkflowBuilder, WorkflowContext, @@ -18,25 +20,47 @@ from copilot.session_events import PermissionRequest from pydantic import BaseModel from sample_validation.const import WORKER_COMPLETED -from sample_validation.discovery import DiscoveryResult from sample_validation.models import ( ExecutionResult, + ReplayResult, RunResult, RunStatus, SampleInfo, ValidationConfig, WorkflowCreationResult, ) +from sample_validation.playbook import ( + Playbook, + PlaybookStore, + compute_sample_hash, + sample_files, +) from typing_extensions import Never logger = logging.getLogger(__name__) +# Directory containing file-based skills used by the validation agents. +SKILLS_DIR = Path(__file__).parent / "skills" + + +class PlaybookScript(BaseModel): + """Reproducible Python validation script the agent returns so future runs can skip the agent. + + ``script`` is a complete, self-contained Python program that reproduces a successful + validation without any AI assistance. The harness runs it with the active interpreter from + the Python root and treats a zero exit code as success. + """ + + script: str + timeout: int = 120 + env: dict[str, str] = {} + class AgentResponseFormat(BaseModel): status: str output: str error: str - fix: str + playbook: PlaybookScript | None = None @dataclass @@ -58,17 +82,47 @@ class BatchCompletion: "Analyze the sample code and execute it as it is. Based on the execution result, determine " "if it runs successfully, fails, or is missing_setup. Use `missing_setup` if the sample reports " "missing required environment variables. The environment you're given should contain the necessary " - "variables. Don't create new environment variables nor modify the sample code.\n" + "variables. Don't create new environment variables nor modify the sample code, unless an available " + "skill instructs you to do so for the setup issue you detected. When a skill applies to the problem, " + "follow its guidance to resolve the setup and then re-run the sample.\n" "Feel free to install any required dependencies if needed.\n" "The sample can be interactive. If it is interactive, respond to the sample when prompted " "based on your analysis of the code. You do not need to consult human on what to respond.\n" - "If the sample fails, investigate the error and suggest a fix.\n" + "Fail fast and do not attempt to fix the sample unless instructed by a skill.\n" + "When (and only when) the status is `success`, also return a `playbook`: a self-contained " + "Python script that reproduces this successful validation WITHOUT any AI assistance, so future " + "runs can replay it deterministically. `playbook.script` is a COMPLETE Python program with " + "these rules:\n" + " - It is run with the SAME Python interpreter, and its WORKING DIRECTORY is the python/ " + "directory (the repo's Python root, where the shared .env lives). Reference the sample with a " + 'path relative to python/, e.g. "samples/04-hosting/foundry-hosted-agents/responses/01_basic".\n' + " - It MUST exit 0 when validation passes and exit non-zero (raise or sys.exit(1)) when it " + "fails. The harness maps exit code 0 to success and anything else to failure.\n" + " - It may import ONLY: the sample's own already-installed dependencies, the Python standard " + "library, and `httpx`. Do not require any other third-party package.\n" + " - It must be non-interactive and deterministic (no prompts, no reliance on you).\n" + " - For a normal run-to-completion sample: launch it (e.g. subprocess.run([sys.executable, " + '"/main.py"], ...) feeding any needed stdin via input=), then assert on the exit code ' + "and/or expected output.\n" + " - For a LONG-LIVED SERVER sample (an HTTP server that never returns on its own, e.g. a " + "hosted agent listening on a port): start it in the BACKGROUND (subprocess.Popen), POLL until " + "the port accepts connections (bounded readiness loop), send the real HTTP request(s) with " + "httpx, and assert HTTP 200 plus a non-empty/expected response. When the sample keeps " + "conversational state, exercise MULTIPLE TURNS (e.g. turn 1 provides a fact like a name, turn 2 " + "asks it to be recalled) and assert the recall. ALWAYS terminate the server in a finally block " + "so no process or port is leaked (the harness also force-kills the process group as a backstop).\n" + " - If you had to edit any sample file to make it run, BAKE that edit into the script (apply it " + "in code before running); the harness restores sample files after replay.\n" + " - On failure, print the captured server/sample output before exiting non-zero to aid debugging.\n" + "Also set `playbook.timeout` (whole-run seconds; the harness caps it) and optional " + "`playbook.env` (extra environment overrides; the CI environment already provides the real " + "credentials, so usually leave this empty).\n" "Return ONLY valid JSON with this schema:\n" "{\n" ' "status": "success|failure|missing_setup",\n' ' "output": "short summary of the result and what you did if the sample was interactive",\n' ' "error": "error details or empty string",\n' - ' "fix": "suggested code fix if the sample failed, otherwise empty string"\n' + ' "playbook": {"script": "", "timeout": 120, "env": {}}\n' "}\n\n" ) @@ -116,9 +170,16 @@ class CustomAgentExecutor(Executor): # Retry in case GitHub Copilot agent encounters transient errors unrelated to the sample execution. RETRY_COUNT = 1 - def __init__(self, agent: GitHubCopilotAgent): + def __init__( + self, + agent: GitHubCopilotAgent, + store: PlaybookStore | None = None, + python_root: Path | None = None, + ): super().__init__(id=agent.id) self.agent = agent + self._store = store + self._python_root = python_root self._session = agent.create_session() @handler @@ -126,6 +187,9 @@ async def handle_task( self, sample: SampleInfo, ctx: WorkflowContext[WorkerFreed | RunResult] ) -> None: """Execute one sample task and notify collector + coordinator.""" + # Snapshot the sample's pristine content so we can restore the working tree afterwards + # (the agent, or a baked-in playbook edit, may modify sample files while validating). + snapshot = self._snapshot_sample(sample) current_retry = 0 while True: try: @@ -144,8 +208,12 @@ async def handle_task( status=status_from_text(result_payload.status), output=result_payload.output, error=result_payload.error, - fix=result_payload.fix, ) + if result.status == RunStatus.SUCCESS: + # Restore the sample to its committed content before hashing so the playbook's + # sample_hash matches what a fresh checkout (and future staleness check) sees. + self._restore_snapshot(snapshot) + self._save_playbook(sample, result_payload) break except Exception as ex: if current_retry < self.RETRY_COUNT: @@ -167,7 +235,6 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=f"Original error: {ex}. Restart error: {restart_ex}", - fix="", ) break @@ -177,15 +244,58 @@ async def handle_task( status=RunStatus.FAILURE, output="", error=str(ex), - fix="", ) break + # Always restore the working tree so a run never leaves the repository dirty + # (on success this is a no-op because we already restored above). + self._restore_snapshot(snapshot) + await ctx.send_message(result, target_id="collector") await ctx.send_message(WorkerFreed(worker_id=self.id), target_id="coordinator") await ctx.add_event(WorkflowEvent(WORKER_COMPLETED, sample)) # type: ignore + def _snapshot_sample(self, sample: SampleInfo) -> dict[Path, bytes]: + """Capture the current on-disk bytes of every file that makes up a sample.""" + snapshot: dict[Path, bytes] = {} + for file in sample_files(sample): + try: + snapshot[file] = file.read_bytes() + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not snapshot {file}: {ex}") + return snapshot + + def _restore_snapshot(self, snapshot: dict[Path, bytes]) -> None: + """Restore snapshotted files to their pristine bytes if the agent changed them.""" + for file, original in snapshot.items(): + try: + if file.read_bytes() != original: + file.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {file}: {ex}") + + def _save_playbook(self, sample: SampleInfo, payload: AgentResponseFormat) -> None: + """Persist a cached playbook for a sample the agent validated successfully.""" + if self._store is None or payload.playbook is None: + return + spec = payload.playbook + if not spec.script.strip(): + logger.warning(f"Agent returned no replay script for {sample.relative_path}; skipping playbook.") + return + try: + playbook = Playbook( + sample=sample.relative_path, + sample_hash=compute_sample_hash(sample), + script=spec.script, + timeout=int(spec.timeout), + env=dict(spec.env), + ) + path = self._store.save(playbook) + logger.info(f"Saved playbook for {sample.relative_path} -> {path}") + except Exception as ex: # pragma: no cover - defensive; never fail validation over caching + logger.warning(f"Could not save playbook for {sample.relative_path}: {ex}") + class BatchCoordinatorExecutor(Executor): """Dispatch sample tasks to worker executors in bounded batches.""" @@ -263,40 +373,60 @@ class CreateConcurrentValidationWorkflowExecutor(Executor): def __init__(self, config: ValidationConfig): super().__init__(id="create_dynamic_workflow") self.config = config + self._store = ( + PlaybookStore(config.playbooks_dir) + if config.use_cache and config.playbooks_dir is not None + else None + ) @handler async def create( self, - discovery: DiscoveryResult, + replay: ReplayResult, ctx: WorkflowContext[WorkflowCreationResult], ) -> None: """Create a nested workflow with a coordinator + worker fan-out/fan-in.""" - sample_count = len(discovery.samples) - print(f"\nCreating nested batched workflow for {sample_count} samples...") + samples = replay.remaining_samples + cached_results = replay.cached_results + sample_count = len(samples) + print( + f"\nCreating nested batched workflow for {sample_count} samples " + f"({len(cached_results)} served from cache)..." + ) if sample_count == 0: await ctx.send_message( - WorkflowCreationResult(samples=[], workflow=None, agents=[]) + WorkflowCreationResult( + samples=[], workflow=None, agents=[], cached_results=cached_results + ) ) return agents: list[GitHubCopilotAgent] = [] workers: list[CustomAgentExecutor] = [] - for index, sample in enumerate(discovery.samples, start=1): + for index, sample in enumerate(samples, start=1): agent_id = f"sample_validator_{index}({sample.relative_path})" agent = GitHubCopilotAgent( id=agent_id, name=agent_id, instructions=AgentInstruction, + context_providers=[SkillsProvider.from_paths( + skill_paths=str(SKILLS_DIR), + disable_load_skill_approval=True, + disable_read_skill_resource_approval=True, + disable_run_skill_script_approval=True, + )], default_options={ "on_permission_request": prompt_permission, - "timeout": 120, + "timeout": self.config.agent_timeout, }, # type: ignore ) agents.append(agent) - workers.append(CustomAgentExecutor(agent)) + workers.append( + CustomAgentExecutor(agent, store=self._store, python_root=self.config.python_root) + ) coordinator = BatchCoordinatorExecutor( worker_ids=[worker.id for worker in workers], @@ -314,8 +444,9 @@ async def create( await ctx.send_message( WorkflowCreationResult( - samples=discovery.samples, + samples=samples, workflow=nested_workflow, agents=agents, + cached_results=cached_results, ) ) diff --git a/python/scripts/sample_validation/discovery.py b/python/scripts/sample_validation/discovery.py index c5424dd6ee9..492843ada56 100644 --- a/python/scripts/sample_validation/discovery.py +++ b/python/scripts/sample_validation/discovery.py @@ -58,7 +58,7 @@ def discover_samples( exclude: list[str] | None = None, ) -> list[SampleInfo]: """ - Find all Python sample files in the samples directory. + Find all samples in the samples directory. Args: samples_dir: Root samples directory @@ -80,38 +80,48 @@ def discover_samples( # Resolve excluded paths to absolute for reliable comparison exclude_paths = {(search_dir / exc).resolve() for exc in (exclude or [])} - python_files: list[Path] = [] + samples: list[Path] = [] # Walk through all subdirectories and find .py files for root, dirs, files in os.walk(search_dir): - # Skip directories that start with _, __pycache__, or excluded paths + # Skip directories that start with _ or ., __pycache__, virtual envs, or excluded paths. + # Dot-directories (e.g. .venv) may be created in a sample folder during validation and + # must never be treated as samples. dirs[:] = [ d for d in dirs if not d.startswith("_") - and d != "__pycache__" + and not d.startswith(".") + and d not in ("__pycache__", "venv", "node_modules") and (Path(root) / d).resolve() not in exclude_paths ] + # If the whole directory is a sample, add the directory itself and do NOT descend into + # it: everything under a main.py/app.py entry point belongs to that one sample. + if any(file in ("main.py", "app.py") for file in files): + samples.append(Path(root)) + dirs[:] = [] + continue + for file in files: # Skip files that start with _ and include only scripts with a main entrypoint guard if file.endswith(".py") and not file.startswith("_"): file_path = Path(root) / file if _has_main_entrypoint_guard(file_path): - python_files.append(file_path) + samples.append(file_path) # Sort files for consistent execution order - python_files = sorted(python_files) + samples = sorted(samples) # Convert to SampleInfo objects - samples: list[SampleInfo] = [] - for path in python_files: + samples_info: list[SampleInfo] = [] + for path in samples: try: - samples.append(SampleInfo.from_path(path, samples_dir)) + samples_info.append(SampleInfo.from_path(path, samples_dir)) except Exception as e: print(f"Warning: Could not read {path}: {e}") - return samples + return samples_info class DiscoverSamplesExecutor(Executor): diff --git a/python/scripts/sample_validation/models.py b/python/scripts/sample_validation/models.py index ff45b5909ba..3740e4d491b 100644 --- a/python/scripts/sample_validation/models.py +++ b/python/scripts/sample_validation/models.py @@ -20,6 +20,9 @@ class ValidationConfig: subdir: str | None = None exclude: list[str] | None = None max_parallel_workers: int = 10 + playbooks_dir: Path | None = None + use_cache: bool = True + agent_timeout: int = 120 @dataclass @@ -28,7 +31,6 @@ class SampleInfo: path: Path relative_path: str - code: str @classmethod def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": @@ -36,7 +38,6 @@ def from_path(cls, path: Path, samples_dir: Path) -> "SampleInfo": return cls( path=path, relative_path=str(path.relative_to(samples_dir)), - code=path.read_text(encoding="utf-8"), ) @@ -47,6 +48,19 @@ class DiscoveryResult: samples: list[SampleInfo] +@dataclass +class ReplayResult: + """Outcome of attempting to replay cached playbooks for discovered samples. + + Samples whose cached playbook replayed successfully are captured in + ``cached_results`` and skip the agent entirely. Everything else flows to the + agent-driven validation as ``remaining_samples``. + """ + + remaining_samples: list[SampleInfo] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] + + @dataclass class WorkflowCreationResult: """Result of creating a nested per-sample concurrent workflow.""" @@ -54,6 +68,7 @@ class WorkflowCreationResult: samples: list[SampleInfo] workflow: Workflow | None agents: list[GitHubCopilotAgent] + cached_results: list["RunResult"] = field(default_factory=list) # type: ignore[assignment] class RunStatus(Enum): @@ -72,7 +87,6 @@ class RunResult: status: RunStatus output: str error: str - fix: str @dataclass @@ -154,7 +168,6 @@ def to_dict(self) -> dict[str, object]: "status": r.status.value, "output": r.output, "error": r.error, - "fix": r.fix, } for r in self.results ], diff --git a/python/scripts/sample_validation/playbook.py b/python/scripts/sample_validation/playbook.py new file mode 100644 index 00000000000..751240c101b --- /dev/null +++ b/python/scripts/sample_validation/playbook.py @@ -0,0 +1,328 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Cached validation playbooks. + +A *playbook* is a small JSON recipe that captures exactly how a sample was +successfully validated so future runs can replay it deterministically without +invoking an LLM agent. The agent produces a playbook the first time it +validates a sample; subsequent runs replay it directly and only fall back to +the agent when the playbook is missing, stale (the sample changed), or the +replay fails. + +The playbook payload is an agent-authored **Python validation script**. Replaying +a playbook writes that script to a temporary file and runs it with the active +interpreter, treating a zero exit code as success. A single script can validate +any sample shape: a plain sample that runs to completion, or a long-lived server +sample (e.g. a hosted agent) that must be started in the background, exercised +over HTTP, asserted, and then torn down. +""" + +import asyncio +import hashlib +import json +import logging +import os +import signal +import subprocess +import sys +import tempfile +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from pathlib import Path + +from sample_validation.models import RunResult, RunStatus, SampleInfo + +logger = logging.getLogger(__name__) + +# Hard cap on how long a replayed sample may run, regardless of the value the +# agent recorded, to protect the CI job from a runaway process. +MAX_REPLAY_TIMEOUT = 600 + + +@dataclass +class Playbook: + """A cached, replayable recipe for validating a single sample. + + The recipe is an agent-authored Python ``script`` that reproduces a + successful validation without any AI assistance. Replaying runs the script + with the active interpreter from the Python root and maps a zero exit code to + success. The script is self-contained: it launches/exercises the sample, + performs its own assertions, bakes in any edits it needs, and (for a server + sample) starts and tears down the server itself. + """ + + sample: str + sample_hash: str + script: str + timeout: int = 120 + env: dict[str, str] = field(default_factory=dict) # type: ignore + expected_status: str = RunStatus.SUCCESS.value + generated_at: str = field(default_factory=lambda: datetime.now().isoformat()) + + def to_dict(self) -> dict[str, object]: + """Serialize to a JSON-friendly dict.""" + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, object]) -> "Playbook": + """Deserialize from a dict, tolerating unknown/missing optional keys.""" + return cls( + sample=str(data["sample"]), + sample_hash=str(data["sample_hash"]), + script=str(data.get("script") or ""), + timeout=int(data.get("timeout") or 120), # type: ignore[arg-type] + env={str(k): str(v) for k, v in (data.get("env") or {}).items()}, # type: ignore[union-attr] + expected_status=str(data.get("expected_status") or RunStatus.SUCCESS.value), + generated_at=str(data.get("generated_at") or datetime.now().isoformat()), + ) + + +def sample_files(sample: SampleInfo) -> list[Path]: + """Return the files that make up a sample. + + For a single-file sample this is just that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it is every ``.py`` file in the tree. + + Note: we only consider source code files, not data files, nor deployment + artifacts (Dockerfile, requirements.txt, etc.) for now. + """ + path = sample.path + if path.is_dir(): + return sorted(p for p in path.rglob("*.py") if "__pycache__" not in p.parts) + return [path] + + +def compute_sample_hash(sample: SampleInfo) -> str: + """Compute a stable content hash for a sample. + + For a single-file sample the hash covers that file. For a directory sample + (``main.py``/``app.py`` entrypoint) it covers every ``.py`` file in the + directory tree so any change to the sample invalidates the cached playbook. + """ + hasher = hashlib.sha256() + path = sample.path + for file in sample_files(sample): + try: + hasher.update(file.relative_to(path.parent).as_posix().encode("utf-8")) + hasher.update(b"\0") + hasher.update(file.read_bytes()) + hasher.update(b"\0") + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not hash {file}: {ex}") + + return f"sha256:{hasher.hexdigest()}" + + +class PlaybookStore: + """Loads and saves per-sample playbooks under a directory. + + Each sample gets one JSON file whose name is derived from the sample's + relative path (path separators replaced so it stays flat and filesystem + safe). + """ + + def __init__(self, directory: Path) -> None: + self.directory = directory + + @staticmethod + def _slug(relative_path: str) -> str: + return relative_path.replace("\\", "/").replace("/", "__") + ".json" + + def _path_for(self, relative_path: str) -> Path: + return self.directory / self._slug(relative_path) + + def load(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook for a sample, or ``None`` if absent/invalid.""" + path = self._path_for(sample.relative_path) + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + playbook = Playbook.from_dict(data) + except Exception as ex: + logger.warning(f"Ignoring unreadable playbook {path}: {ex}") + return None + return playbook + + def save(self, playbook: Playbook) -> Path: + """Persist a playbook to disk and return its path.""" + self.directory.mkdir(parents=True, exist_ok=True) + path = self._path_for(playbook.sample) + path.write_text(json.dumps(playbook.to_dict(), indent=2), encoding="utf-8") + return path + + def is_valid_for(self, sample: SampleInfo) -> Playbook | None: + """Return the stored playbook only if it matches the sample's current hash.""" + playbook = self.load(sample) + if playbook is None: + return None + current_hash = compute_sample_hash(sample) + if playbook.sample_hash != current_hash: + logger.info(f"Playbook for {sample.relative_path} is stale (sample changed); will re-validate.") + return None + return playbook + + +def _new_process_group_kwargs() -> dict[str, object]: + """Return subprocess kwargs that launch the child in its own process group. + + Isolating the replay in a new group/session lets us tear down the whole tree + (for example a background server the script spawned) even if the script exits + without cleaning up after itself. + """ + if os.name == "posix": + return {"start_new_session": True} + # Windows: a new process group so the whole tree can be terminated together. + return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} + + +def _terminate_process_tree(proc: "asyncio.subprocess.Process", pgid: int | None) -> None: + """Best-effort kill of a replay process and any descendants it leaked. + + This always runs, even when the script itself already exited cleanly, because + the whole point of the backstop is to reap a background server the script may + have started but failed to tear down. On POSIX we signal the saved process + group id (captured at spawn time, since the leader may already be reaped); + on Windows we kill the process tree by PID. + """ + try: + if os.name == "posix": + if pgid is not None: + os.killpg(pgid, signal.SIGKILL) + elif proc.returncode is None: + proc.kill() + else: + # Kill by PID (not image name) including the whole child tree. + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + check=False, + ) + except (ProcessLookupError, OSError) as ex: # pragma: no cover - defensive + logger.debug(f"Could not terminate replay process tree {proc.pid}: {ex}") + + +async def replay_playbook(playbook: Playbook, python_root: Path) -> RunResult: + """Deterministically replay a playbook and return a ``RunResult``. + + Writes the recorded Python script to a temporary file and runs it with the + active interpreter, using the Python root as the working directory. A zero + exit code maps to ``SUCCESS``; anything else yields a ``FAILURE`` so the + caller falls back to the agent. + + Sample files are snapshotted and restored so a replay never leaves the + repository dirty (the script may edit the sample to make it run), and the + replay process is launched in its own process group so a background server it + starts is always torn down afterwards, even if the script fails to clean up. + """ + sample_path = playbook.sample + sample_info = SampleInfo(path=python_root / sample_path, relative_path=sample_path) + + if not playbook.script.strip(): + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error="Playbook has an empty validation script.", + ) + + # Snapshot every file that makes up the sample so we can restore it byte-exact + # no matter how we exit (the script is allowed to edit the sample in place). + snapshot: dict[Path, bytes] = {} + for file in sample_files(sample_info): + try: + snapshot[file] = file.read_bytes() + except OSError: + pass + + script_file = tempfile.NamedTemporaryFile( # noqa: SIM115 - closed explicitly below + prefix="playbook_", suffix=".py", delete=False + ) + script_path = Path(script_file.name) + proc: asyncio.subprocess.Process | None = None + pgid: int | None = None + try: + script_file.write(playbook.script.encode("utf-8")) + script_file.close() + + env = {**os.environ, **playbook.env} + timeout = min(max(1, playbook.timeout), MAX_REPLAY_TIMEOUT) + + start = time.perf_counter() + try: + proc = await asyncio.create_subprocess_exec( + sys.executable, + str(script_path), + cwd=str(python_root), + env=env, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + **_new_process_group_kwargs(), # type: ignore[arg-type] + ) + except OSError as ex: + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay could not start the script: {ex}", + ) + + # Capture the process-group id now (== proc.pid, since we start a new + # session/group) so teardown can kill leaked descendants even after the + # leader process has exited and been reaped. + if os.name == "posix": + try: + pgid = os.getpgid(proc.pid) + except OSError: # pragma: no cover - defensive + pgid = None + + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except (TimeoutError, asyncio.TimeoutError): + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay timed out after {timeout}s.", + ) + + duration = time.perf_counter() - start + output_text = stdout.decode("utf-8", errors="replace") if stdout else "" + + if proc.returncode == 0: + return RunResult( + sample=sample_info, + status=RunStatus.SUCCESS, + output=f"Replayed cached playbook successfully in {duration:.1f}s.", + error="", + ) + + # Non-zero exit: surface a truncated tail to aid debugging; caller retries with the agent. + tail = output_text[-1000:] + return RunResult( + sample=sample_info, + status=RunStatus.FAILURE, + output="", + error=f"Replay exited with code {proc.returncode}. Output tail: {tail}", + ) + finally: + # Always tear down the process group (even if the script exited cleanly) + # so a background server it leaked is killed, then reap the leader. + if proc is not None: + _terminate_process_tree(proc, pgid) + try: + await proc.wait() + except Exception: # pragma: no cover - defensive + pass + for target, original in snapshot.items(): + try: + if target.read_bytes() != original: + target.write_bytes(original) + except OSError as ex: # pragma: no cover - defensive + logger.warning(f"Could not restore {target} after replay: {ex}") + try: + script_path.unlink() + except OSError: # pragma: no cover - defensive + pass diff --git a/python/scripts/sample_validation/replay_executor.py b/python/scripts/sample_validation/replay_executor.py new file mode 100644 index 00000000000..6806243d6d3 --- /dev/null +++ b/python/scripts/sample_validation/replay_executor.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Executor that replays cached playbooks before falling back to the agent.""" + +import asyncio +import logging + +from agent_framework import Executor, WorkflowContext, handler + +from sample_validation.models import ( + DiscoveryResult, + ReplayResult, + RunResult, + RunStatus, + SampleInfo, + ValidationConfig, +) +from sample_validation.playbook import PlaybookStore, replay_playbook + +logger = logging.getLogger(__name__) + + +class ReplayCachedPlaybooksExecutor(Executor): + """Replay cached playbooks and split samples into cached vs. agent-bound. + + For every discovered sample we look for a playbook whose recorded hash still + matches the sample's current content. Matching playbooks are replayed + deterministically (no LLM). Samples that pass are returned as cached + results; samples without a valid playbook, or whose replay fails, are passed + on for agent-driven validation (which will refresh the playbook). + """ + + def __init__(self, config: ValidationConfig) -> None: + super().__init__(id="replay_cached_playbooks") + self.config = config + self._store = PlaybookStore(config.playbooks_dir) if config.playbooks_dir else None + + @handler + async def replay(self, discovery: DiscoveryResult, ctx: WorkflowContext[ReplayResult]) -> None: + """Attempt cached replay for each sample, then forward the remainder.""" + samples = discovery.samples + + if not self.config.use_cache or self._store is None or not samples: + await ctx.send_message(ReplayResult(remaining_samples=samples, cached_results=[])) + return + + store = self._store + candidates: list[tuple[SampleInfo, object]] = [] + remaining: list[SampleInfo] = [] + + for sample in samples: + playbook = store.is_valid_for(sample) + if playbook is None: + remaining.append(sample) + else: + candidates.append((sample, playbook)) + + print( + f"\nCache check: {len(candidates)} sample(s) have a valid playbook, " + f"{len(remaining)} need the agent." + ) + + cached_results: list[RunResult] = [] + if candidates: + semaphore = asyncio.Semaphore(max(1, self.config.max_parallel_workers)) + + async def _run(sample: SampleInfo, playbook: object) -> tuple[SampleInfo, RunResult]: + async with semaphore: + result = await replay_playbook(playbook, self.config.python_root) # type: ignore[arg-type] + return sample, result + + for coro in asyncio.as_completed([_run(s, pb) for s, pb in candidates]): + sample, result = await coro + if result.status == RunStatus.SUCCESS: + cached_results.append(result) + print(f"[cache hit] {sample.relative_path}") + else: + # Replay failed: hand the sample to the agent to re-validate and refresh the playbook. + logger.info(f"Cached replay failed for {sample.relative_path}: {result.error}") + print(f"[cache miss] {sample.relative_path} (replay failed, will use agent)") + remaining.append(sample) + + await ctx.send_message(ReplayResult(remaining_samples=remaining, cached_results=cached_results)) diff --git a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py index c7244cff2a2..143a8214caf 100644 --- a/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py +++ b/python/scripts/sample_validation/run_dynamic_validation_workflow_executor.py @@ -36,8 +36,11 @@ async def run( self, creation: WorkflowCreationResult, ctx: WorkflowContext[ExecutionResult] ) -> None: """Run the nested workflow and emit execution results.""" + cached_results = list(creation.cached_results) + if creation.workflow is None: - await ctx.send_message(ExecutionResult(results=[])) + # Nothing left for the agent; emit whatever the cache produced. + await ctx.send_message(ExecutionResult(results=cached_results)) return print("\nRunning nested batched workflow...") @@ -50,10 +53,10 @@ async def run( CoordinatorStart(samples=creation.samples), stream=True ): if event.type == "output" and isinstance(event.data, ExecutionResult): - result = event.data # type: ignore - elif event.type == WORKER_COMPLETED and isinstance( + result = event.data + elif event.type == WORKER_COMPLETED and isinstance( # type: ignore event.data, SampleInfo - ): # type: ignore + ): remaining_sample_counts -= 1 print( f"Completed validation for sample: {event.data.relative_path:<80} | " @@ -61,7 +64,9 @@ async def run( ) if result is not None: - await ctx.send_message(result) + await ctx.send_message( + ExecutionResult(results=cached_results + result.results) + ) else: fallback_results = [ RunResult( @@ -69,10 +74,11 @@ async def run( status=RunStatus.FAILURE, output="", error="Nested workflow did not return an ExecutionResult.", - fix="", ) for sample in creation.samples ] - await ctx.send_message(ExecutionResult(results=fallback_results)) + await ctx.send_message( + ExecutionResult(results=cached_results + fallback_results) + ) finally: await stop_agents(creation.agents) diff --git a/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md new file mode 100644 index 00000000000..eb4e75746e0 --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-config-setup/SKILL.md @@ -0,0 +1,52 @@ +--- +name: foundry-config-setup +description: Resolve missing setup caused by a hardcoded Foundry project endpoint or model in a sample. Use when a sample fails because it uses a placeholder/hardcoded project_endpoint (for example "https://your-project.services.ai.azure.com") or a hardcoded model instead of reading them from the environment. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +## Usage + +Some samples (notably those under `01-get-started`) hardcode the Foundry +project endpoint and model directly in the `FoundryChatClient` constructor +using placeholder values, for example: + +```python +client = FoundryChatClient( + project_endpoint="https://your-project.services.ai.azure.com", + model="gpt-4o", + credential=AzureCliCredential(), +) +``` + +These placeholder values are not real and the sample cannot run as written. +The validation environment provides the real values through environment +variables, so the sample must read them from the environment instead. + +When you detect a hardcoded/placeholder `project_endpoint` (or `model`) that +is causing the sample to fail with missing or invalid setup: + +1. Replace the hardcoded `project_endpoint` value with a read from the + `FOUNDRY_PROJECT_ENDPOINT` environment variable. +2. Replace the hardcoded `model` value with a read from the `FOUNDRY_MODEL` + environment variable. +3. Ensure `import os` is present at the top of the file. + +The corrected constructor should look like: + +```python +import os + +client = FoundryChatClient( + project_endpoint=os.environ["FOUNDRY_PROJECT_ENDPOINT"], + model=os.environ["FOUNDRY_MODEL"], + credential=AzureCliCredential(), +) +``` + +These samples are intentionally written with hardcoded placeholders, so this +is expected setup—not a defect in the sample. After applying the change, +re-run the sample and report the result as a `success` if it now runs. diff --git a/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md new file mode 100644 index 00000000000..de8f9a9841d --- /dev/null +++ b/python/scripts/sample_validation/skills/foundry-hosted-agent-validation/SKILL.md @@ -0,0 +1,316 @@ +--- +name: foundry-hosted-agent-validation +description: > + Step-by-step process for validating a Python Foundry hosted agent sample + (under python/samples/04-hosting/foundry-hosted-agents/) end to end — running + it locally (native runtime and `azd ai agent run`) and after deploying it to + an Azure AI Foundry project with `azd`. Use this when asked to validate a hosted + agent sample. +license: MIT +compatibility: Works with any model that supports tool use. +metadata: + author: agent-framework-samples + version: "1.0" +--- + +# Validating a Foundry Hosted Agent Sample + +A hosted agent sample is "validated" when it passes **three** independent +checks, plus cleanup: + +1. **Local, native runtime** — run the sample's own entry point + (`python main.py`) and invoke it over HTTP. +2. **Local, via `azd ai agent run`** — the `azd` local dev loop. +3. **Deployed** — `azd deploy` to Foundry, then invoke the hosted agent. + +Each check must succeed for **single-turn** and **multi-turn** (session / +`previous_response_id`) conversation. Always end with **cleanup** (delete the +deployed agent, remove the temp `azd` project, restore the sample dir). + +> Read the sample's own `README.md` and the parent +> `.../foundry-hosted-agents/README.md` first — they define the run/deploy +> commands and any sample-specific payload. This skill captures the process and +> the non-obvious gotchas the READMEs don't. + +--- + +## Inputs you need before starting + +Gather these: + +- **Foundry project endpoint**, e.g. + `https://.services.ai.azure.com/api/projects/`. +- **Foundry project resource id** (for non-interactive `azd ai agent init`): + `/subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts//projects/`. + Find it with `az cognitiveservices account list` + the project name. +- **A real, deployed model name** in that project (e.g. `gpt-4.1-mini`). This is + often **different** from the model id in `agent.manifest.yaml` — the actual + deployment name wins. +- **An existing ACR to reuse** for deployment (login server, e.g. + `myacr.azurecr.io`). Reusing one avoids `azd provision` creating resources. +- Whether a like-named agent **already exists** in the project (remove it first + for a clean validation — see below). + +## Tooling / auth + +- `az` (logged in: `az login`) and `azd` (logged in: `azd auth login`). +- `azd` **agents extension**: `azd extension list` should show + `azure.ai.agents`; install with `azd extension install azure.ai.agents`. +- `uv` for the native-Python local run. **`python` need not be on PATH** — `uv` + and `azd ai agent run` provision their own interpreter. +- Docker is **not** required when you reuse an ACR (`remoteBuild: true` builds + in ACR Tasks). + +--- + +## Phase 0 — Understand the sample + +A responses/invocations sample folder typically contains: +`main.py` (entry point + `ResponsesHostServer`/`InvocationsHostServer`), +`agent.manifest.yaml` (used by `azd ai agent init`), `agent.yaml` (the deployed +agent definition), `requirements.txt`, `Dockerfile`, `.env.example`. + +**The sample is the whole directory whose entry point is `main.py` — not every +`.py` file in it.** Other Python files in (or alongside) a sample folder are +**helper/companion scripts**, not standalone samples. Do **not** treat a helper +script as an individual sample — validate the sample via its `main.py` host, and +run a helper only when the sample's `README.md` calls for it as a setup. + +Note the **protocol** (`responses` or `invocations`) from `agent.yaml` / +manifest — it changes the invoke command (`--protocol invocations`) and the HTTP +path (`/responses` vs the invocations route). + +--- + +## Phase 1 — Local validation, native runtime (Python) + +Run from the sample directory. + +```bash +uv venv .venv --python 3.12 # 3.12 matches the sample Dockerfile +uv pip install --python .venv/... -r requirements.txt +``` + +Create `.env` from `.env.example` with the **real** values: + +``` +FOUNDRY_PROJECT_ENDPOINT="https://.services.ai.azure.com/api/projects/" +AZURE_AI_MODEL_DEPLOYMENT_NAME="" +``` + +Start the server (`python main.py`) — it listens on `http://localhost:8088`. +`main.py` uses `DefaultAzureCredential`, so `az login` must be current. + +Invoke (single turn), capture the returned `response_id`, then reuse it for a +follow-up turn to confirm memory: + +```bash +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "My name is Tao. Remember it."}' +# take response_id from the JSON, then: +curl -X POST http://localhost:8088/responses -H "Content-Type: application/json" \ + -d '{"input": "What is my name?", "previous_response_id": ""}' +``` + +PowerShell: use `Invoke-WebRequest -Uri http://localhost:8088/responses -Method POST -ContentType application/json -Body '...'`. + +**Pass:** HTTP 200, non-empty `output[].content[].text`, and the second turn +recalls the name. Stop the server afterward. + +### Recording the playbook (cached replay) + +The sample-validation harness caches a **playbook** so future runs replay your +result without an agent. For a hosted-agent sample the playbook is an +agent-authored **Python script** that reproduces **only this Phase 1 +native-local smoke test** — never the `azd`/deploy phases (those are +credential- and deployment-bound, non-deterministic, and must not enter the +replay cache). + +Emit a self-contained script that the harness runs with the active interpreter +from the `python/` directory (exit code 0 = success). It may import only the +sample's own installed deps, the Python stdlib, and `httpx`. It must: + +1. Start the sample server in the **background** with + `subprocess.Popen([sys.executable, "/main.py"])` (path relative to + `python/`; the shared `python/.env` and the process env supply credentials). +2. **Poll** `http://localhost:8088` until the port accepts a connection (bounded + readiness loop with a timeout), failing if it never comes up. +3. POST turn 1 to the protocol's route (`/responses` for `responses` samples; + the invocations route for `invocations` samples), capture the `response_id`, + then POST turn 2 with `previous_response_id` to confirm recall. +4. **Assert** HTTP 200, non-empty `output[].content[].text`, and that turn 2 + recalls the fact from turn 1. +5. **Always** terminate the server in a `finally` block so port 8088 is freed + (the harness force-kills the process group as a backstop, but the script must + still clean up). On any failure, print the captured server output before + exiting non-zero. + +Keep the deep three-phase validation (below) for a full manual/`azd` pass; it is +out of scope for the cached playbook. + +--- + +## Phase 2 — Local validation via `azd ai agent run` + +### Init the azd project (once) + +Run in an **empty temp directory outside the repo** (short path avoids Windows +path-length issues, e.g. `C:\afval\`). Point `-m` at the **local** +manifest so it validates the working-tree sample: + +```bash +azd ai agent init -m /agent.manifest.yaml \ + --project-id "" \ + --model-deployment "" \ + --agent-name "" \ + --no-prompt --force +``` + +`init` downloads the template into a **subfolder named after the agent**, so the +azd project root is `//`. `cd` there for all later `azd` +commands. + +> **Before init, remove any `.venv` you created in the sample dir** — `init` +> copies the entire manifest directory into `src/`. (`.venv` is excluded from +> deploy packaging by `.agentignore`/`.dockerignore`, so it is harmless but +> bloats/slows the copy.) + +### Fix the model deployment name (critical — see Gotcha 1) + +```bash +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME "" +``` + +### Run and invoke locally + +```bash +azd ai agent run --no-inspector # auto-creates a uv venv + installs deps; listens on :8088 +azd ai agent invoke --local --new-session "My name is Tao. Remember it." +azd ai agent invoke --local "What is my name?" # same session is reused automatically +``` + +**Pass:** both invokes return text; the second recalls the name (same +`Session:` id). Stop the `run` process afterward. + +--- + +## Removing a pre-existing agent (do this before deploying) + +`init` prints a warning if the agent name already exists in the project. To +delete it, note that **`azd ai agent delete`/`show` resolve the deployed agent +name from an azd env var, not from the positional argument.** The var is +`AGENT_{SERVICEKEY}_NAME`, where `SERVICEKEY` = the `azure.yaml` service name +uppercased with `-`/spaces → `_`. + +Example for service `agent-framework-agent-basic-responses`: + +```bash +azd env set AGENT_AGENT_FRAMEWORK_AGENT_BASIC_RESPONSES_NAME agent-framework-agent-basic-responses +azd ai agent delete --force --no-prompt --output json +# -> {"object":"agent.deleted","name":"...","deleted":true} +``` + +(After a successful `azd deploy`, this var is set automatically, so later +`show`/`delete`/`invoke` work without setting it.) + +--- + +## Phase 3 — Deploy and validate + +### Reuse an existing ACR (avoid provisioning) + +For an existing project + model, **do not run `azd provision`/`azd up`** — the +generated `azure.yaml` has a `deployments` block for the manifest's model +(often an auto-selected `GlobalProvisionedManaged` PTU SKU) that provision would +try to create (costly / quota failures). Instead reuse an ACR: + +```bash +azd env set AZURE_CONTAINER_REGISTRY_ENDPOINT # e.g. myacr.azurecr.io +azd deploy +``` + +`azd deploy` fails with _"could not determine container registry endpoint"_ if +this is unset and no ACR is provisioned. + +### Verify the deployed model env var, then invoke + +```bash +azd ai agent show --output json # check definition.environment_variables.AZURE_AI_MODEL_DEPLOYMENT_NAME +azd ai agent invoke --new-session "My name is Tao. Remember it." +azd ai agent invoke "What is my name?" +``` + +Use `--output raw` on invoke to see raw SSE events and any failure, e.g.: + +``` +event: response.failed +... "code": "DeploymentNotFound" ... 404 ... +``` + +`DeploymentNotFound` means the deployed `AZURE_AI_MODEL_DEPLOYMENT_NAME` points +at a model that isn't deployed → fix per Gotcha 1 and redeploy (creates a new +version). + +**Pass:** agent reaches `status: active`, invoke returns text (not empty, no +`response.failed`), and multi-turn recalls the name. + +--- + +## Cleanup (always) + +- Delete the deployed agent: `azd ai agent delete --force --no-prompt`. +- Delete the temp `azd` project directory. +- Remove `.env`/`.venv` you created in the sample dir; confirm the sample dir is + pristine (`git status --porcelain ` is empty — `.env`/`.venv` are + gitignored). +- Stop any leftover local server still holding port 8088: + `Get-NetTCPConnection -LocalPort 8088 -State Listen` → `Stop-Process -Id ` + (Linux/macOS: `lsof -ti:8088 | xargs kill`). Stopping the shell may leave the + child interpreter running. + +--- + +## Gotchas (the parts that waste the most time) + +1. **The deployed model name comes from `agent.yaml`, not the azd env.** + `azd ai agent init` ignores `--model-deployment` in `--no-prompt` mode and + writes the **manifest's** model id (e.g. `gpt-4.1-mini`) as a **literal** into + both the azd env `AZURE_AI_MODEL_DEPLOYMENT_NAME` and the generated + `src//agent.yaml` env var. Local runs read the azd env (so + `azd env set` fixes them), but **deployment injects `agent.yaml`'s value**. + Fix by setting the generated `agent.yaml` env var to the template + `value: ${AZURE_AI_MODEL_DEPLOYMENT_NAME}` (what the repo sample already uses; + `init` flattens it) **and** `azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME `, + then redeploy. A literal `value: ` also works. + +2. **`azd ai agent delete/show` need the `AGENT_{SERVICEKEY}_NAME` env var** — a + bare positional agent name is treated as the _service_ name and the deployed + agent name is looked up from that env var (see "Removing a pre-existing + agent"). + +3. **`azd provision`/`azd up` will try to create the manifest's model + deployment** (from `azure.yaml`'s `deployments` block). Prefer `azd deploy` + with a reused ACR when the project + model already exist. + +4. **`python` on PATH is not required.** `uv venv` and `azd ai agent run` + provision their own interpreter and install `requirements.txt`. + +5. **`init` copies the whole manifest directory into `src/`.** Remove a local + `.venv` from the sample dir first to keep the copy clean/fast. + +6. **Port 8088 can stay bound after stopping the shell** — kill the interpreter + by PID (see Cleanup). + +--- + +## Success checklist + +- [ ] Native local run: 200 + non-empty text + multi-turn recall. +- [ ] `azd ai agent run` local: text returned + session reused across invokes. +- [ ] Pre-existing agent removed (if any). +- [ ] `azd deploy` succeeds; agent `status: active`. +- [ ] `azd ai agent show` confirms `AZURE_AI_MODEL_DEPLOYMENT_NAME` = the real + deployed model. +- [ ] Deployed invoke: text returned (no `response.failed`) + multi-turn recall. +- [ ] Cleanup done: agent deleted, temp project removed, sample dir pristine, + port 8088 free. diff --git a/python/scripts/sample_validation/workflow.py b/python/scripts/sample_validation/workflow.py index 10187c069be..49ded23aa4a 100644 --- a/python/scripts/sample_validation/workflow.py +++ b/python/scripts/sample_validation/workflow.py @@ -12,6 +12,7 @@ CreateConcurrentValidationWorkflowExecutor, ) from sample_validation.discovery import DiscoverSamplesExecutor, ValidationConfig +from sample_validation.replay_executor import ReplayCachedPlaybooksExecutor from sample_validation.report import GenerateReportExecutor from sample_validation.run_dynamic_validation_workflow_executor import ( RunDynamicValidationWorkflowExecutor, @@ -31,13 +32,15 @@ def create_validation_workflow( Configured Workflow instance """ discover = DiscoverSamplesExecutor(config) + replay = ReplayCachedPlaybooksExecutor(config) create_dynamic_workflow = CreateConcurrentValidationWorkflowExecutor(config) run_dynamic_workflow = RunDynamicValidationWorkflowExecutor() generate = GenerateReportExecutor() return ( WorkflowBuilder(start_executor=discover) - .add_edge(discover, create_dynamic_workflow) + .add_edge(discover, replay) + .add_edge(replay, create_dynamic_workflow) .add_edge(create_dynamic_workflow, run_dynamic_workflow) .add_edge(run_dynamic_workflow, generate) .build()