Add automated SQL Assignment 2 marking

.github/scripts/grade_assignments.py

@@ -0,0 +1,47 @@
+"""Run each changed A2 submission and always produce a fresh combined report."""
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+
+
+def grade_files(files, output):
+    """Grade exactly one changed Assignment 2 SQL file and write its report."""
+    if len(files) != 1:
+        if files:
+            error = "More than one assignment2.sql file changed. Submit only your cohort's file."
+        else:
+            error = "No changed assignment2.sql file was found."
+        reports = [{"file": "Submission selection", "results": [], "runner_error": error}]
+        Path(output).write_text(json.dumps(reports, indent=2))
+        return 1
+
+    file_path = files[0]
+    with tempfile.TemporaryDirectory(prefix="a2-reports-") as folder:
+        report = Path(folder) / "report.json"
+        error = None
+        try:
+            result = subprocess.run(
+                [sys.executable, "-m", "pytest", "03_instructional_team/tests/test_assignment.py",
+                 f"--file_path={file_path}", f"--report_path={report}",
+                 "--tb=short", "--disable-warnings"], timeout=120, check=False,
+            )
+            if result.returncode:
+                error = f"Grading runner exited with status {result.returncode}."
+            answers = json.loads(report.read_text()) if report.exists() else []
+            if not isinstance(answers, list) or len(answers) != 12:
+                raise ValueError("Grading runner did not produce a complete 12-question report.")
+        except Exception as exception:
+            answers = []
+            error = str(exception)
+
+    reports = [{"file": file_path, "results": answers, "runner_error": error}]
+    Path(output).write_text(json.dumps(reports, indent=2))
+    return int(error is not None)
+
+
+if __name__ == "__main__":
+    changed = Path(sys.argv[1]).read_bytes().decode().split("\0")
+    files = sorted({path for path in changed if Path(path).name == "assignment2.sql"})
+    sys.exit(grade_files(files, sys.argv[2]))

.github/workflows/sql_assignment_runner.yml

@@ -1,12 +1,16 @@
 name: SQL Unit Tests
 
 on:
+  pull_request:
+    branches:
+      - main
+    types: [opened, reopened, synchronize]
   workflow_dispatch:  # Manual trigger from GitHub UI
     inputs:
       branch:
-        description: 'Branch to use to run assignment for, i.e. assignment-one, assignment-2'
+        description: 'Assignment 2 branch to grade, usually assignment-two'
         required: true
-        default: assignment-one
+        default: assignment-two
         type: string
       pr_number:
         description: 'Pull request number. Extract a number from the end of pull request url, i.e. https://github.com/khsergvl/sql/pull/17'
@@ -19,8 +23,10 @@ permissions:
 
 jobs:
   run-assignment-queries:
-    # uncomment once a change to PR trigger will be implemented
-    # if: startsWith(github.head_ref, 'assignment-')
+    if: >-
+      github.event_name == 'workflow_dispatch' ||
+      ((github.head_ref == 'assignment-two' || github.head_ref == 'assignment-2') &&
+      github.repository_owner != 'UofT-DSI')
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -30,7 +36,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.branch }}
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || github.event.pull_request.head.sha }}
+          fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -41,95 +48,92 @@ jobs:
         run: |
           pip install -r ./.github/scripts/requirements.txt
 
+      - name: Verify grader regressions
+        run: python -m pytest 03_instructional_team/tests/test_grader_regressions.py -q
+
       - name: Get changed files
-        id: changes
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git fetch
-          git diff --name-only origin/main > changed_files.txt
-
-          echo "Changed files:"
-          cat changed_files.txt
-          # The goal here to search for a file modified during PR that is match to a pattern
-          # and give priority in next order 2, 1 as it quite often happens when students submit
-          # PR without following assignments order and git flow. It will to run latest / greatest
-          assignment_changed=""
-
-          # Priority: assignment2 > assignment1
-          if grep -qE '(^|/)assignment2\.sql$' changed_files.txt; then
-            assignment_changed=$(grep -E '(^|/)assignment2\.sql$' changed_files.txt | head -n1)
-          elif grep -qE '(^|/)assignment1\.sql$' changed_files.txt; then
-            assignment_changed=$(grep -E '(^|/)assignment1\.sql$' changed_files.txt | head -n1)
-          fi
-
-          if [ -n "$assignment_changed" ]; then
-            echo "assignment_changed=$assignment_changed" >> "$GITHUB_OUTPUT"
-          fi
+          base_ref="${BASE_SHA:-origin/main}"
+          git diff --name-only -z "$base_ref"...HEAD > changed_files.txt
 
-      - name: Run tests
-        id: pytest
-        run: |
-          pytest tests/test_assignment.py --file_path="${{steps.changes.outputs.assignment_changed}}" --tb=short --disable-warnings \
-                 --junitxml=pytest-report.xml || true
+      - name: Grade changed assignments
+        id: grading
+        continue-on-error: true
+        run: python .github/scripts/grade_assignments.py changed_files.txt test-results.json
 
-      - name: Post test results to PR
+      - name: Post grading feedback to PR
+        if: always() && steps.grading.outcome != 'skipped'
         uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || github.event.pull_request.number }}
         with:
           script: |
-            function jsonToMarkdownTable(data) {
-              if (!Array.isArray(data) || data.length === 0) {
-                return 'No data';
-              }
-
-              const headers = Object.keys(data[0]);
-
-              // Header row
-              const headerRow = `| ${headers.join(' | ')} |`;
-
-              // Separator row
-              const separatorRow = `| ${headers.map(() => '---').join(' | ')} |`;
-
-              // Data rows
-              const rows = data.map(row =>
-                `| ${headers.map(h => formatValue(row[h])).join(' | ')} |`
-              );
-
-              return [headerRow, separatorRow, ...rows].join('\n');
+            const fs = require('fs');
+            const labels = {
+              correct: '✅ Correct', incorrect: '❌ Incorrect', error: '❌ Error',
+              missing: '❌ Missing answer', blocked: '⚪ Cannot assess: prerequisite failed',
+              manual_review: '⚪ Instructor review needed'
+            };
+            function fenced(value, language = '') {
+              const text = String(value);
+              const runs = text.match(/`+/g) || [];
+              const fence = '`'.repeat(Math.max(3, ...runs.map(run => run.length + 1)));
+              return `${fence}${language}\n${text}\n${fence}\n`;
             }
-
-            function formatValue(value) {
-              if (value === null || value === undefined) return '';
-              if (typeof value === 'object') return `\`${JSON.stringify(value)}\``;
-              return String(value);
+            let reports;
+            try {
+              reports = JSON.parse(fs.readFileSync('test-results.json', 'utf8'));
+              if (!Array.isArray(reports)) throw new Error('Invalid report format');
+            } catch (error) {
+              reports = [{file: 'Grading runner', results: [], runner_error: error.message}];
             }
-
-            const fs = require('fs')
-            const file_read_result = fs.readFileSync('test-results.json', 'utf8')
-            const results = JSON.parse(file_read_result)
-
-            let body = `### 🧪 SQL Queries Run Results (up to 3 rows)\n\n`
-            body += `<details> <summary>Click to expand/collapse assignment queries execution results</summary>\n\n`
-
-            for (const result of results) {
-              const isArrayResult = Array.isArray(result.result);
-
-              if (result.error) {
-                body += `❌ Query ${result.number}: \n\n *${result.query}*, \n\n **Error**:  \n${result.error}\n\n`
-              } else if (isArrayResult && result.result.length === 0) {
-                body += `⚪ Query ${result.number}: \n\n *${result.query}*, \n\n No rows returned \n\n`
-              } else if (isArrayResult && result.result.length > 0) {
-                body += `✅ Query ${result.number}: \n\n *${result.query}*, \n\n **Results**:  \n`
-                const table = jsonToMarkdownTable(result.result)
-                body += `${table} \n\n`
+            if (!reports.length) return;
+            let body = '### SQL Assignment 2 feedback\n\n';
+            body += 'SQL checks only. ERDs, written responses, and rubric point totals require instructor marking.\n\n';
+            const truncationNotice = '\nFeedback truncated; inspect the workflow report for remaining results.';
+            const maxBodyLength = 60000;
+            let truncated = false;
+            function appendSection(section) {
+              if (body.length + section.length + truncationNotice.length > maxBodyLength) {
+                truncated = true;
+                return false;
               }
-
-              body += `-------------------------------------------------------- \n`;
+              body += section;
+              return true;
             }
-
-            body += `</details>`
-
+            reportLoop:
+            for (const report of reports) {
+              let reportSection = fenced(report.file);
+              if (report.runner_error) reportSection += `Grading could not finish: ${report.runner_error}\n\n`;
+              if (!appendSection(reportSection)) break;
+              for (const result of report.results) {
+                let resultSection = `**Query ${result.number}: ${labels[result.status] || '⚪ Instructor review needed'}**\n\n`;
+                resultSection += `${result.feedback}\n\n`;
+                if (result.error) resultSection += fenced(result.error);
+                if (result.query) resultSection += fenced(result.query, 'sql');
+                if (result.result?.length) resultSection += fenced(JSON.stringify(result.result, null, 2), 'json');
+                resultSection += '\n';
+                if (!appendSection(resultSection)) break reportLoop;
+              }
+            }
+            if (truncated) body += truncationNotice;
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: ${{ inputs.pr_number }},
+              issue_number: Number(process.env.PR_NUMBER),
               body
-            })
+            });
+
+      - name: Upload grading reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: a2-grading-results
+          path: test-results.json
+          if-no-files-found: ignore
+
+      - name: Preserve runner failure status
+        if: always() && steps.grading.outcome == 'failure'
+        run: exit 1

.gitignore

@@ -2,3 +2,5 @@
 .vscode/
 *.db-journal
 .venv
+__pycache__/
+*.pyc

02_activities/assignments/DC_Cohort/Assignment2.md

@@ -14,6 +14,7 @@
     * Open a private window in your browser. Copy and paste the link to your pull request into the address bar. Make sure you can see your pull request properly. This helps the technical facilitator and learning support staff review your submission easily.
 
 Checklist:
+- [ ] 🚨🚨🚨 After forking, open the **Actions** tab in your own SQL repository. If prompted, click **I understand my workflows, go ahead and enable them** before opening your assignment pull request. This enables automatic SQL feedback on your pull request.
 - [ ] Create a branch called `assignment-two`.
 - [ ] Ensure that the repository is public.
 - [ ] Review [the PR description guidelines](https://github.com/UofT-DSI/onboarding/blob/main/onboarding_documents/submissions.md#guidelines-for-pull-request-descriptions) and adhere to them.

02_activities/assignments/Microcredential_Cohort/Assignment2.md

@@ -14,6 +14,7 @@
     * Open a private window in your browser. Copy and paste the link to your pull request into the address bar. Make sure you can see your pull request properly. This helps the technical facilitator and learning support staff review your submission easily.
 
 Checklist:
+- [ ] 🚨🚨🚨 After forking, open the **Actions** tab in your own SQL repository. If prompted, click **I understand my workflows, go ahead and enable them** before opening your assignment pull request. This enables automatic SQL feedback on your pull request.
 - [ ] Create a branch called `assignment-two`.
 - [ ] Ensure that the repository is public.
 - [ ] Review [the PR description guidelines](https://github.com/UofT-DSI/onboarding/blob/main/onboarding_documents/submissions.md#guidelines-for-pull-request-descriptions) and adhere to them.

02_activities/assignments/Microcredential_Cohort/assignment2.sql

@@ -62,7 +62,7 @@ HINT: Do not use the previous visit dates filter. */
 /* 3. Using a COUNT() window function, include a value along with each row of the 
 customer_purchases table that indicates how many different times that customer has purchased that product_id. 
 
-You can make this a running count by including an ORDER BY within the PARTITION BY if desired.
+You can make this a running count by including an ORDER BY in the window function if desired.
 Filter the visits to dates before April 29, 2022. */
 --QUERY 4
 

tests/conftest.py → 03_instructional_team/tests/conftest.py

@@ -4,12 +4,13 @@
 
 @pytest.fixture
 def sqlite_db():
+    """Provide each test with an in-memory copy of the course database."""
     source_db = Path("05_src/sql/farmersmarket.db")
     # Create in-memory DB
     conn = sqlite3.connect(":memory:")
 
     # Load schema.db into memory
-    disk_db = sqlite3.connect(source_db)
+    disk_db = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
     conn.row_factory = sqlite3.Row
     disk_db.backup(conn)
     disk_db.close()
@@ -18,9 +19,16 @@ def sqlite_db():
     conn.close()
 
 def pytest_addoption(parser):
-    parser.addoption("--file_path", action="store", default="02_activities/assignments/DC_Cohort/assignment1.sql")
+    """Add the command-line option that selects the Assignment 2 SQL file."""
+    parser.addoption("--report_path", action="store", default="test-results.json")
+    parser.addoption("--file_path", action="store", default="02_activities/assignments/DC_Cohort/assignment2.sql")
 
 def pytest_generate_tests(metafunc):
+    """Pass the selected assignment path into tests that request file_path."""
     option_value = metafunc.config.option.file_path
     if 'file_path' in metafunc.fixturenames and option_value is not None:
         metafunc.parametrize("file_path", [option_value])
+
+@pytest.fixture
+def report_path(request):
+    return request.config.getoption("--report_path")