diff --git a/.github/scripts/grade_assignments.py b/.github/scripts/grade_assignments.py new file mode 100644 index 000000000..0d5a3942c --- /dev/null +++ b/.github/scripts/grade_assignments.py @@ -0,0 +1,47 @@ +"""Run each changed A2 submission and always produce a fresh combined report.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile + + +def grade_files(files, output): + """Grade exactly one changed Assignment 2 SQL file and write its report.""" + if len(files) != 1: + if files: + error = "More than one assignment2.sql file changed. Submit only your cohort's file." + else: + error = "No changed assignment2.sql file was found." + reports = [{"file": "Submission selection", "results": [], "runner_error": error}] + Path(output).write_text(json.dumps(reports, indent=2)) + return 1 + + file_path = files[0] + with tempfile.TemporaryDirectory(prefix="a2-reports-") as folder: + report = Path(folder) / "report.json" + error = None + try: + result = subprocess.run( + [sys.executable, "-m", "pytest", "03_instructional_team/tests/test_assignment.py", + f"--file_path={file_path}", f"--report_path={report}", + "--tb=short", "--disable-warnings"], timeout=120, check=False, + ) + if result.returncode: + error = f"Grading runner exited with status {result.returncode}." + answers = json.loads(report.read_text()) if report.exists() else [] + if not isinstance(answers, list) or len(answers) != 12: + raise ValueError("Grading runner did not produce a complete 12-question report.") + except Exception as exception: + answers = [] + error = str(exception) + + reports = [{"file": file_path, "results": answers, "runner_error": error}] + Path(output).write_text(json.dumps(reports, indent=2)) + return int(error is not None) + + +if __name__ == "__main__": + changed = Path(sys.argv[1]).read_bytes().decode().split("\0") + files = sorted({path for path in changed if Path(path).name == "assignment2.sql"}) + sys.exit(grade_files(files, sys.argv[2])) diff --git a/.github/workflows/sql_assignment_runner.yml b/.github/workflows/sql_assignment_runner.yml index 928761494..4beb330cc 100644 --- a/.github/workflows/sql_assignment_runner.yml +++ b/.github/workflows/sql_assignment_runner.yml @@ -1,12 +1,16 @@ name: SQL Unit Tests on: + pull_request: + branches: + - main + types: [opened, reopened, synchronize] workflow_dispatch: # Manual trigger from GitHub UI inputs: branch: - description: 'Branch to use to run assignment for, i.e. assignment-one, assignment-2' + description: 'Assignment 2 branch to grade, usually assignment-two' required: true - default: assignment-one + default: assignment-two type: string pr_number: description: 'Pull request number. Extract a number from the end of pull request url, i.e. https://github.com/khsergvl/sql/pull/17' @@ -19,8 +23,10 @@ permissions: jobs: run-assignment-queries: - # uncomment once a change to PR trigger will be implemented - # if: startsWith(github.head_ref, 'assignment-') + if: >- + github.event_name == 'workflow_dispatch' || + ((github.head_ref == 'assignment-two' || github.head_ref == 'assignment-2') && + github.repository_owner != 'UofT-DSI') runs-on: ubuntu-latest permissions: contents: read @@ -30,7 +36,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - ref: ${{ inputs.branch }} + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || github.event.pull_request.head.sha }} + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 @@ -41,95 +48,92 @@ jobs: run: | pip install -r ./.github/scripts/requirements.txt + - name: Verify grader regressions + run: python -m pytest 03_instructional_team/tests/test_grader_regressions.py -q + - name: Get changed files - id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | - git fetch - git diff --name-only origin/main > changed_files.txt - - echo "Changed files:" - cat changed_files.txt - # The goal here to search for a file modified during PR that is match to a pattern - # and give priority in next order 2, 1 as it quite often happens when students submit - # PR without following assignments order and git flow. It will to run latest / greatest - assignment_changed="" - - # Priority: assignment2 > assignment1 - if grep -qE '(^|/)assignment2\.sql$' changed_files.txt; then - assignment_changed=$(grep -E '(^|/)assignment2\.sql$' changed_files.txt | head -n1) - elif grep -qE '(^|/)assignment1\.sql$' changed_files.txt; then - assignment_changed=$(grep -E '(^|/)assignment1\.sql$' changed_files.txt | head -n1) - fi - - if [ -n "$assignment_changed" ]; then - echo "assignment_changed=$assignment_changed" >> "$GITHUB_OUTPUT" - fi + base_ref="${BASE_SHA:-origin/main}" + git diff --name-only -z "$base_ref"...HEAD > changed_files.txt - - name: Run tests - id: pytest - run: | - pytest tests/test_assignment.py --file_path="${{steps.changes.outputs.assignment_changed}}" --tb=short --disable-warnings \ - --junitxml=pytest-report.xml || true + - name: Grade changed assignments + id: grading + continue-on-error: true + run: python .github/scripts/grade_assignments.py changed_files.txt test-results.json - - name: Post test results to PR + - name: Post grading feedback to PR + if: always() && steps.grading.outcome != 'skipped' uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || github.event.pull_request.number }} with: script: | - function jsonToMarkdownTable(data) { - if (!Array.isArray(data) || data.length === 0) { - return 'No data'; - } - - const headers = Object.keys(data[0]); - - // Header row - const headerRow = `| ${headers.join(' | ')} |`; - - // Separator row - const separatorRow = `| ${headers.map(() => '---').join(' | ')} |`; - - // Data rows - const rows = data.map(row => - `| ${headers.map(h => formatValue(row[h])).join(' | ')} |` - ); - - return [headerRow, separatorRow, ...rows].join('\n'); + const fs = require('fs'); + const labels = { + correct: '✅ Correct', incorrect: '❌ Incorrect', error: '❌ Error', + missing: '❌ Missing answer', blocked: '⚪ Cannot assess: prerequisite failed', + manual_review: '⚪ Instructor review needed' + }; + function fenced(value, language = '') { + const text = String(value); + const runs = text.match(/`+/g) || []; + const fence = '`'.repeat(Math.max(3, ...runs.map(run => run.length + 1))); + return `${fence}${language}\n${text}\n${fence}\n`; } - - function formatValue(value) { - if (value === null || value === undefined) return ''; - if (typeof value === 'object') return `\`${JSON.stringify(value)}\``; - return String(value); + let reports; + try { + reports = JSON.parse(fs.readFileSync('test-results.json', 'utf8')); + if (!Array.isArray(reports)) throw new Error('Invalid report format'); + } catch (error) { + reports = [{file: 'Grading runner', results: [], runner_error: error.message}]; } - - const fs = require('fs') - const file_read_result = fs.readFileSync('test-results.json', 'utf8') - const results = JSON.parse(file_read_result) - - let body = `### 🧪 SQL Queries Run Results (up to 3 rows)\n\n` - body += `
Click to expand/collapse assignment queries execution results\n\n` - - for (const result of results) { - const isArrayResult = Array.isArray(result.result); - - if (result.error) { - body += `❌ Query ${result.number}: \n\n *${result.query}*, \n\n **Error**: \n${result.error}\n\n` - } else if (isArrayResult && result.result.length === 0) { - body += `⚪ Query ${result.number}: \n\n *${result.query}*, \n\n No rows returned \n\n` - } else if (isArrayResult && result.result.length > 0) { - body += `✅ Query ${result.number}: \n\n *${result.query}*, \n\n **Results**: \n` - const table = jsonToMarkdownTable(result.result) - body += `${table} \n\n` + if (!reports.length) return; + let body = '### SQL Assignment 2 feedback\n\n'; + body += 'SQL checks only. ERDs, written responses, and rubric point totals require instructor marking.\n\n'; + const truncationNotice = '\nFeedback truncated; inspect the workflow report for remaining results.'; + const maxBodyLength = 60000; + let truncated = false; + function appendSection(section) { + if (body.length + section.length + truncationNotice.length > maxBodyLength) { + truncated = true; + return false; } - - body += `-------------------------------------------------------- \n`; + body += section; + return true; } - - body += `
` - + reportLoop: + for (const report of reports) { + let reportSection = fenced(report.file); + if (report.runner_error) reportSection += `Grading could not finish: ${report.runner_error}\n\n`; + if (!appendSection(reportSection)) break; + for (const result of report.results) { + let resultSection = `**Query ${result.number}: ${labels[result.status] || '⚪ Instructor review needed'}**\n\n`; + resultSection += `${result.feedback}\n\n`; + if (result.error) resultSection += fenced(result.error); + if (result.query) resultSection += fenced(result.query, 'sql'); + if (result.result?.length) resultSection += fenced(JSON.stringify(result.result, null, 2), 'json'); + resultSection += '\n'; + if (!appendSection(resultSection)) break reportLoop; + } + } + if (truncated) body += truncationNotice; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: ${{ inputs.pr_number }}, + issue_number: Number(process.env.PR_NUMBER), body - }) + }); + + - name: Upload grading reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: a2-grading-results + path: test-results.json + if-no-files-found: ignore + + - name: Preserve runner failure status + if: always() && steps.grading.outcome == 'failure' + run: exit 1 diff --git a/.gitignore b/.gitignore index b7ffb5651..ec819d823 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .vscode/ *.db-journal .venv +__pycache__/ +*.pyc diff --git a/02_activities/assignments/DC_Cohort/Assignment2.md b/02_activities/assignments/DC_Cohort/Assignment2.md index be0eff3a5..47a854540 100644 --- a/02_activities/assignments/DC_Cohort/Assignment2.md +++ b/02_activities/assignments/DC_Cohort/Assignment2.md @@ -14,6 +14,7 @@ * Open a private window in your browser. Copy and paste the link to your pull request into the address bar. Make sure you can see your pull request properly. This helps the technical facilitator and learning support staff review your submission easily. Checklist: +- [ ] 🚨🚨🚨 After forking, open the **Actions** tab in your own SQL repository. If prompted, click **I understand my workflows, go ahead and enable them** before opening your assignment pull request. This enables automatic SQL feedback on your pull request. - [ ] Create a branch called `assignment-two`. - [ ] Ensure that the repository is public. - [ ] Review [the PR description guidelines](https://github.com/UofT-DSI/onboarding/blob/main/onboarding_documents/submissions.md#guidelines-for-pull-request-descriptions) and adhere to them. diff --git a/02_activities/assignments/Microcredential_Cohort/Assignment2.md b/02_activities/assignments/Microcredential_Cohort/Assignment2.md index 2c39be5ff..2ff945602 100644 --- a/02_activities/assignments/Microcredential_Cohort/Assignment2.md +++ b/02_activities/assignments/Microcredential_Cohort/Assignment2.md @@ -14,6 +14,7 @@ * Open a private window in your browser. Copy and paste the link to your pull request into the address bar. Make sure you can see your pull request properly. This helps the technical facilitator and learning support staff review your submission easily. Checklist: +- [ ] 🚨🚨🚨 After forking, open the **Actions** tab in your own SQL repository. If prompted, click **I understand my workflows, go ahead and enable them** before opening your assignment pull request. This enables automatic SQL feedback on your pull request. - [ ] Create a branch called `assignment-two`. - [ ] Ensure that the repository is public. - [ ] Review [the PR description guidelines](https://github.com/UofT-DSI/onboarding/blob/main/onboarding_documents/submissions.md#guidelines-for-pull-request-descriptions) and adhere to them. diff --git a/02_activities/assignments/Microcredential_Cohort/assignment2.sql b/02_activities/assignments/Microcredential_Cohort/assignment2.sql index 4079c18ae..f4a2b999a 100644 --- a/02_activities/assignments/Microcredential_Cohort/assignment2.sql +++ b/02_activities/assignments/Microcredential_Cohort/assignment2.sql @@ -62,7 +62,7 @@ HINT: Do not use the previous visit dates filter. */ /* 3. Using a COUNT() window function, include a value along with each row of the customer_purchases table that indicates how many different times that customer has purchased that product_id. -You can make this a running count by including an ORDER BY within the PARTITION BY if desired. +You can make this a running count by including an ORDER BY in the window function if desired. Filter the visits to dates before April 29, 2022. */ --QUERY 4 diff --git a/tests/conftest.py b/03_instructional_team/tests/conftest.py similarity index 55% rename from tests/conftest.py rename to 03_instructional_team/tests/conftest.py index 5415915f8..f17c8dd5e 100644 --- a/tests/conftest.py +++ b/03_instructional_team/tests/conftest.py @@ -4,12 +4,13 @@ @pytest.fixture def sqlite_db(): + """Provide each test with an in-memory copy of the course database.""" source_db = Path("05_src/sql/farmersmarket.db") # Create in-memory DB conn = sqlite3.connect(":memory:") # Load schema.db into memory - disk_db = sqlite3.connect(source_db) + disk_db = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True) conn.row_factory = sqlite3.Row disk_db.backup(conn) disk_db.close() @@ -18,9 +19,16 @@ def sqlite_db(): conn.close() def pytest_addoption(parser): - parser.addoption("--file_path", action="store", default="02_activities/assignments/DC_Cohort/assignment1.sql") + """Add the command-line option that selects the Assignment 2 SQL file.""" + parser.addoption("--report_path", action="store", default="test-results.json") + parser.addoption("--file_path", action="store", default="02_activities/assignments/DC_Cohort/assignment2.sql") def pytest_generate_tests(metafunc): + """Pass the selected assignment path into tests that request file_path.""" option_value = metafunc.config.option.file_path if 'file_path' in metafunc.fixturenames and option_value is not None: metafunc.parametrize("file_path", [option_value]) + +@pytest.fixture +def report_path(request): + return request.config.getoption("--report_path") diff --git a/03_instructional_team/tests/test_assignment.py b/03_instructional_team/tests/test_assignment.py new file mode 100644 index 000000000..d0492ac04 --- /dev/null +++ b/03_instructional_team/tests/test_assignment.py @@ -0,0 +1,621 @@ +import json +from datetime import datetime, timedelta, timezone +import time +import re +import sqlite3 +from collections import Counter +from pathlib import Path + + +PRODUCT_COLUMNS = ( + "product_id", "product_name", "product_size", + "product_category_id", "product_qty_type", +) + + +def load_queries(sql_file): + """Extract exactly one closed answer block per required question.""" + queries = {} + number = None + buffer = [] + for line in Path(sql_file).read_text().splitlines(): + stripped = line.strip() + start = re.fullmatch(r"--\s*QUERY\s+(\d+)\s*", stripped, re.I) + end = re.fullmatch(r"--\s*END\s+QUERY\s*", stripped, re.I) + if start: + if number is not None: + raise ValueError("An answer block was not closed before the next QUERY marker.") + number = int(start[1]) + if number not in range(1, 13) or number in queries: + raise ValueError(f"Unknown or duplicate QUERY number: {number}.") + buffer = [] + elif end: + if number is None: + raise ValueError("END QUERY has no matching QUERY marker.") + queries[number] = "\n".join(buffer).strip() + number = None + elif re.match(r"--\s*(QUERY|END\s+QUERY)\b", stripped, re.I): + raise ValueError(f"Invalid answer marker: {stripped}") + elif number is not None: + buffer.append(line) + if number is not None: + raise ValueError(f"QUERY {number} has no END QUERY marker.") + return [{"number": n, "query": queries.get(n, "")} for n in range(1, 13)] + + +def sql_tokens(query): + """Return unquoted SQL tokens, excluding comments and string literals.""" + pattern = r"--[^\n]*|/\*[\s\S]*?\*/|'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`[^`]*`|\[[^]]*\]|[A-Za-z_][A-Za-z_0-9]*|[^\s]" + return [part.upper() for part in re.findall(pattern, query) + if not part.startswith(("--", "/*", "'", '\"', "`", "["))] + + +def technique_feedback(number, query): + """Confirm required syntax independently from result correctness.""" + tokens = sql_tokens(query) + pairs = set(zip(tokens, tokens[1:])) + requirements = {1: ("COALESCE", "("), 4: ("COUNT", "("), + 5: ("SUBSTR", "("), 8: ("CROSS", "JOIN"), + 10: ("INSERT", "INTO"), 11: ("DELETE", "FROM")} + required = requirements.get(number) + if required and required not in pairs: + return f"The required SQL technique ({required[0]}) was not found." + if number in (2, 3, 4) and ("OVER", "(") not in pairs: + return "Use a window function for this answer." + if number == 6 and "REGEXP" not in tokens: + return "Use REGEXP as requested." + if number == 7 and "UNION" not in tokens: + return "Use UNION as requested." + if number == 12 and "UPDATE" not in tokens: + return "Use UPDATE as requested." + return None + + +def register_functions(conn, clock=None): + """Add the REGEXP function required by Assignment 2 to SQLite.""" + # Python's SQLite does not include REGEXP by default, but Assignment 2 Question 6 requires it. + conn.create_function( + "REGEXP", 2, + lambda pattern, value: value is not None and re.search(pattern, str(value)) is not None, + ) + + if clock is not None: + conn.create_function("current_timestamp", 0, lambda: clock[0]) + conn.create_function("current_date", 0, lambda: clock[0].split()[0]) + conn.create_function("current_time", 0, lambda: clock[0].split()[1]) + # Delegate SQLite date semantics while making 'now' deterministic. + for name in ("date", "time", "datetime", "strftime", "julianday", "unixepoch"): + def date_function(*args, function=name): + values = tuple(clock[0] if arg == "now" else arg for arg in args) + if not values and function != "strftime": + values = (clock[0],) + placeholders = ",".join("?" for _ in values) + delegate = sqlite3.connect(":memory:") + try: + return delegate.execute(f"SELECT {function}({placeholders})", values).fetchone()[0] + finally: + delegate.close() + conn.create_function(name, -1, date_function) + + +def clone_database(source, clock=None): + """Create an isolated in-memory copy of the course database.""" + clone = sqlite3.connect(":memory:") + clone.row_factory = sqlite3.Row + source.backup(clone) + register_functions(clone, clock) + clone.set_authorizer(lambda action, *_: sqlite3.SQLITE_DENY + if action in (sqlite3.SQLITE_ATTACH, sqlite3.SQLITE_DETACH) + else sqlite3.SQLITE_OK) + return clone + + +def run_query(conn, query): + """Execute SQL statements without splitting quoted semicolons or comments.""" + deadline = time.monotonic() + 10 + conn.set_progress_handler(lambda: int(time.monotonic() > deadline), 1000) + cursor = conn.cursor() + buffer = "" + try: + for character in query: + buffer += character + if character == ";" and sqlite3.complete_statement(buffer): + if any(token != ";" for token in sql_tokens(buffer)): + cursor.execute(buffer) + buffer = "" + if any(token != ";" for token in sql_tokens(buffer)): + cursor.execute(buffer) + return [dict(row) for row in cursor.fetchall()] + finally: + conn.set_progress_handler(None, 0) + + +def fetch(conn, query, parameters=()): + """Run an internal grading query and return its rows as dictionaries.""" + return [dict(row) for row in conn.execute(query, parameters).fetchall()] + + +def normalize_value(value): + """Normalize floating-point values so result comparisons are reliable.""" + return round(value, 6) if isinstance(value, float) else value + + +def count_result_rows(rows, keys): + """Count normalized result rows using only the requested columns.""" + return Counter(tuple(normalize_value(row[key]) for key in keys) for row in rows) + + +def extract_result_rows(rows, columns, last_fallback=False): + """Extract comparable columns from student results using accepted column names.""" + if not rows: + return Counter() + actual_keys = [] + lower_keys = {key.lower(): key for key in rows[0]} + for index, choices in enumerate(columns): + key = next((lower_keys[name.lower()] for name in choices if name.lower() in lower_keys), None) + if key is None and last_fallback and index == len(columns) - 1: + key = list(rows[0])[-1] + if key is None: + return None + actual_keys.append(key) + return Counter(tuple(normalize_value(row[key]) for key in actual_keys) for row in rows) + + +def grading_outcome(passed, success, failure): + """Return a grading status and the matching feedback message.""" + return ("correct", success) if passed else ("incorrect", failure) + + +def grade_question_1(rows, reference): + """Grade the COALESCE product-description answer.""" + expected = fetch(reference, """ + SELECT product_name || ', ' || COALESCE(product_size, '') || + ' (' || COALESCE(product_qty_type, 'unit') || ')' AS value + FROM product + """) + # Each expected row contains one complete product description. + expected_descriptions = [] + for row in expected: + expected_descriptions.append(row["value"]) + + # The student must also return one description column per row. + has_one_column_per_row = bool(rows) + for row in rows: + if len(row) != 1: + has_one_column_per_row = False + break + + actual_descriptions = None + if has_one_column_per_row: + actual_descriptions = [] + for row in rows: + values = list(row.values()) + description = values[0] + actual_descriptions.append(normalize_value(description)) + + # Count descriptions so duplicates matter, while row order does not. + answers_match = False + if actual_descriptions is not None: + actual_counts = Counter(actual_descriptions) + expected_counts = Counter(expected_descriptions) + answers_match = actual_counts == expected_counts + + return grading_outcome( + answers_match, + "All product descriptions and NULL replacements are correct.", + "The formatted product descriptions do not match the expected results.", + ) + + +def grade_question_2(rows, reference): + """Grade the ascending customer visit-number answer.""" + actual = extract_result_rows( + rows, (("customer_id",), ("market_date",), ("visit_number",)), True + ) + full = fetch(reference, """ + SELECT customer_id, market_date, + DENSE_RANK() OVER (PARTITION BY customer_id ORDER BY market_date) AS visit_number + FROM customer_purchases WHERE market_date < '2022-04-29' + """) + unique = fetch(reference, """ + SELECT customer_id, market_date, + ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY market_date) AS visit_number + FROM (SELECT DISTINCT customer_id, market_date FROM customer_purchases + WHERE market_date < '2022-04-29') + """) + keys = ("customer_id", "market_date", "visit_number") + accepted = [count_result_rows(result, keys) for result in (full, unique)] + return grading_outcome( + actual in accepted, + "Visit dates are numbered correctly for each customer.", + "The visit numbers or returned dates are incorrect.", + ) + + +def grade_question_3(rows, reference): + """Grade the most-recent customer visit answer.""" + actual = extract_result_rows( + rows, (("customer_id",), ("market_date",), ("visit_number",)), True + ) + full = fetch(reference, """ + SELECT customer_id, market_date, 1 AS visit_number + FROM customer_purchases AS p + WHERE market_date = (SELECT MAX(market_date) FROM customer_purchases + WHERE customer_id = p.customer_id) + """) + unique = fetch(reference, """ + SELECT customer_id, MAX(market_date) AS market_date, 1 AS visit_number + FROM customer_purchases GROUP BY customer_id + """) + keys = ("customer_id", "market_date", "visit_number") + accepted = [count_result_rows(result, keys) for result in (full, unique)] + return grading_outcome( + actual in accepted, + "Only each customer's most recent visit is returned.", + "The result does not contain only each customer's most recent visit.", + ) + + +def grade_question_4(rows, reference): + """Grade the customer product-purchase count answer.""" + columns = ( + ("customer_id",), ("vendor_id",), ("product_id",), + ("market_date",), ("transaction_time",), ("product_purchase_count",), + ) + actual = extract_result_rows(rows, columns, True) + fixed = fetch(reference, """ + SELECT customer_id, vendor_id, product_id, market_date, transaction_time, + COUNT(*) OVER (PARTITION BY customer_id, product_id) AS product_purchase_count + FROM customer_purchases WHERE market_date < '2022-04-29' + """) + running = fetch(reference, """ + SELECT customer_id, vendor_id, product_id, market_date, transaction_time, + COUNT(*) OVER (PARTITION BY customer_id, product_id + ORDER BY market_date, transaction_time + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS product_purchase_count + FROM customer_purchases WHERE market_date < '2022-04-29' + """) + keys = tuple(item[0] for item in columns) + accepted = [count_result_rows(result, keys) for result in (fixed, running)] + return grading_outcome( + actual in accepted, + "The purchase count is correct for every customer and product.", + "The purchase counts or returned rows are incorrect.", + ) + + +def grade_question_5(rows, reference): + """Grade the product-name description extraction answer.""" + expected = fetch(reference, """ + SELECT product_name, + CASE WHEN INSTR(product_name, '-') > 0 + THEN TRIM(SUBSTR(product_name, INSTR(product_name, '-') + 1)) + ELSE NULL END AS description + FROM product + """) + expected_rows = count_result_rows(expected, ("product_name", "description")) + # Students may name the description column differently; compare its values with each product. + matches = any( + extract_result_rows(rows, (("product_name",), (column,))) == expected_rows + for column in (rows[0] if rows else ()) + if column.lower() != "product_name" + ) + return grading_outcome( + matches, + "Product descriptions are extracted and trimmed correctly.", + "The descriptions do not match the text after each product-name hyphen.", + ) + + +def grade_question_6(rows, reference): + """Grade the numeric product-size REGEXP answer.""" + actual = extract_result_rows(rows, (("product_size",),)) + if actual is None and rows and all(len(row) == 1 for row in rows): + actual = Counter((normalize_value(next(iter(row.values()))),) for row in rows) + expected = fetch( + reference, "SELECT product_size FROM product WHERE product_size REGEXP '[0-9]'" + ) + return grading_outcome( + actual == count_result_rows(expected, ("product_size",)), + "Only product sizes containing a number are returned.", + "The result does not match all product sizes containing a number.", + ) + + +def grade_question_7(rows, reference): + """Grade the highest and lowest market-sales answer.""" + actual = extract_result_rows(rows, (("market_date",), ("sales", "total_sales", "revenue")), True) + expected = fetch(reference, """ + WITH daily AS ( + SELECT market_date, ROUND(SUM(quantity * cost_per_quantity), 2) AS sales + FROM customer_purchases GROUP BY market_date + ) + SELECT market_date, sales FROM daily + WHERE sales = (SELECT MIN(sales) FROM daily) + OR sales = (SELECT MAX(sales) FROM daily) + """) + if rows and all(len(row) == 1 for row in rows): + actual_dates = Counter(next(iter(row.values())) for row in rows) + return grading_outcome(actual_dates == Counter(row["market_date"] for row in expected), + "The highest and lowest sales dates are correct.", + "The highest and lowest sales dates are incorrect.") + if actual != count_result_rows(expected, ("market_date", "sales")) and rows: + for column in rows[0]: + if column == "market_date": + continue + candidate = extract_result_rows(rows, (("market_date",), (column,))) + if candidate == count_result_rows(expected, ("market_date", "sales")): + actual = candidate + break + return grading_outcome( + actual == count_result_rows(expected, ("market_date", "sales")), + "The highest and lowest sales dates and totals are correct.", + "The result does not match the highest and lowest sales dates.", + ) + + +def grade_question_8(rows, reference): + """Grade the possible vendor revenue answer.""" + actual = extract_result_rows( + rows, + (("vendor_name",), ("product_name",), ("possible_revenue", "sum(price)", "revenue")), + True, + ) + expected = fetch(reference, """ + SELECT vendor_name, product_name, SUM(price) AS possible_revenue + FROM ( + SELECT DISTINCT vendor_name, product_name, 5 * original_price AS price + FROM vendor_inventory JOIN vendor USING (vendor_id) JOIN product USING (product_id) + ) + CROSS JOIN (SELECT DISTINCT customer_id FROM customer) + GROUP BY vendor_name, product_name + """) + return grading_outcome( + actual == count_result_rows(expected, ("vendor_name", "product_name", "possible_revenue")), + "Possible revenue is correct for every vendor and product.", + "The possible revenue does not match five items per customer.", + ) + + +SELECT_GRADERS = { + 1: grade_question_1, + 2: grade_question_2, + 3: grade_question_3, + 4: grade_question_4, + 5: grade_question_5, + 6: grade_question_6, + 7: grade_question_7, + 8: grade_question_8, +} + + +def grade_select(number, rows, reference): + """Dispatch Questions 1 through 8 to their grading functions.""" + grader = SELECT_GRADERS.get(number) + if grader is None: + return "incorrect", "This query number is not part of Assignment 2." + return grader(rows, reference) + + +def product_units_state(conn): + """Return the product_units table structure and data when the table exists.""" + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='product_units'" + ).fetchone() + if not exists: + return None + columns = [row["name"] for row in fetch(conn, "PRAGMA table_info(product_units)")] + rows = fetch(conn, "SELECT * FROM product_units") + try: + identities = fetch(conn, "SELECT rowid AS identity FROM product_units") + rowids = [row["identity"] for row in identities] + except sqlite3.OperationalError: + rowids = None + return {"columns": columns, "rows": rows, "rowids": rowids} + + +def count_table_rows(rows, columns): + """Convert selected row values into a countable form for state comparisons.""" + return Counter( + tuple(normalize_value(row[column]) for column in columns) + for row in rows + ) + + +def timestamp_value(value): + timestamp = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + if timestamp.tzinfo is not None: + timestamp = timestamp.astimezone(timezone.utc).replace(tzinfo=None) + return timestamp + + +def valid_timestamp(value, context): + try: + return abs((timestamp_value(value) - timestamp_value(context["execution_time"])).total_seconds()) <= 5 + except (ValueError, TypeError): + return False + + +def grade_question_9(conn, reference, context, before): + state = product_units_state(conn) + required = set(PRODUCT_COLUMNS) | {"snapshot_timestamp"} + expected = fetch(reference, "SELECT * FROM product WHERE product_qty_type='unit'") + passed = (state is not None and set(state["columns"]) == required + and count_table_rows(state["rows"], PRODUCT_COLUMNS) == count_table_rows(expected, PRODUCT_COLUMNS) + and all(valid_timestamp(row["snapshot_timestamp"], context) for row in state["rows"])) + return grading_outcome(passed, "Unit products and current snapshot timestamps are correct.", + "Check the columns, unit products, and current snapshot timestamps.") + + +def grade_question_10(conn, reference, context, before): + state = product_units_state(conn) + if before is None or state is None or set(state["columns"]) != set(before["columns"]): + return "incorrect", "The product_units schema changed or is missing." + columns = before["columns"] + old = count_table_rows(before["rows"], columns) + new = count_table_rows(state["rows"], columns) + added = new - old + if old - new or sum(added.values()) != 1: + return "incorrect", "Insert exactly one row without changing existing rows." + row = dict(zip(columns, next(iter(added)))) + passed = (row["product_id"] is not None and row["product_category_id"] is not None + and isinstance(row["product_name"], str) and bool(row["product_name"].strip()) + and isinstance(row["product_qty_type"], str) and bool(row["product_qty_type"].strip()) + and valid_timestamp(row["snapshot_timestamp"], context)) + if passed: + previous_copies = [existing for existing in before["rows"] + if existing["product_id"] == row["product_id"]] + passed = all(timestamp_value(row["snapshot_timestamp"]) >= timestamp_value(existing["snapshot_timestamp"]) + for existing in previous_copies) + # Save the inserted row for Q11 so we can check that it survives the deletion. + if passed: + context["inserted_row"] = row + if before["rowids"] is not None and state["rowids"] is not None: + added_ids = set(state["rowids"]) - set(before["rowids"]) + if len(added_ids) == 1: + context["inserted_rowid"] = added_ids.pop() + return grading_outcome(passed, "One product with a current timestamp was inserted.", + "Check the inserted product fields and updated timestamp.") + + +def grade_question_11(conn, reference, context, before): + state = product_units_state(conn) + inserted = context.get("inserted_row") + if before is None or state is None or inserted is None: + return "blocked", "A valid preceding insertion is needed to assess the deletion." + columns = before["columns"] + if set(columns) != set(state["columns"]): + return "incorrect", "Deleting a record must preserve the table schema." + before_rows = count_table_rows(before["rows"], columns) + after_rows = count_table_rows(state["rows"], columns) + inserted_key = tuple(normalize_value(inserted[column]) for column in columns) + candidates = before_rows.copy() + candidates.subtract({inserted_key: 1}) + old_rows = [dict(zip(columns, key)) for key, count in candidates.items() + if count > 0 and key[columns.index("product_id")] == inserted["product_id"]] + if not old_rows: + return "manual_review", "The inserted product has no older copy; the deletion needs instructor review." + removed = before_rows - after_rows + if after_rows - before_rows or sum(removed.values()) != 1: + return "incorrect", "Remove exactly the older copy and retain the newer copy and other rows." + oldest = min(timestamp_value(row["snapshot_timestamp"]) for row in old_rows) + removed_row = dict(zip(columns, next(iter(removed)))) + passed = (removed_row["product_id"] == inserted["product_id"] + and timestamp_value(removed_row["snapshot_timestamp"]) == oldest + and after_rows[inserted_key] >= 1) + if context.get("inserted_rowid") is not None and state["rowids"] is not None: + passed = passed and context["inserted_rowid"] in state["rowids"] + return grading_outcome(passed, "Only the older copy was removed; the inserted copy remains.", + "Retain the inserted copy and remove only the older copy.") + + +def grade_question_12(conn, reference, context, before): + state = product_units_state(conn) + if before is None or state is None or set(state["columns"]) != set(before["columns"]) | {"current_quantity"}: + return "incorrect", "Add current_quantity while preserving the existing table columns." + if count_table_rows(before["rows"], before["columns"]) != count_table_rows(state["rows"], before["columns"]): + return "incorrect", "UPDATE must preserve all existing rows and non-quantity values." + for row in state["rows"]: + latest = fetch(reference, """ + SELECT COALESCE(quantity, 0) AS quantity FROM vendor_inventory + WHERE product_id=? AND market_date=( + SELECT MAX(market_date) FROM vendor_inventory WHERE product_id=?) + """, (row["product_id"], row["product_id"])) + allowed = {normalize_value(item["quantity"]) for item in latest} or {0} + if normalize_value(row["current_quantity"]) not in allowed: + return "incorrect", f"Product {row['product_id']} does not have its latest quantity." + return "correct", "All rows remain and current_quantity matches the latest inventory." + + +MUTATION_GRADERS = { + 9: grade_question_9, + 10: grade_question_10, + 11: grade_question_11, + 12: grade_question_12, +} + + +def grade_mutation(number, conn, reference, context, before): + """Dispatch Questions 9 through 12 to their grading functions.""" + grader = MUTATION_GRADERS.get(number) + if grader is None: + return "incorrect", "This query number is not part of Assignment 2." + return grader(conn, reference, context, before) + + +def build_result(number, query, status, feedback, rows=None, error=None): + """Build one consistently shaped entry for the JSON grading report.""" + return { + "number": number, + "query": query, + "status": status, + "feedback": feedback, + "result": (rows or [])[:3], + "error": str(error) if error is not None else None, + } + + +def test_assignment(sqlite_db, file_path, report_path): + """Provide the pytest entry point for grading the submitted assignment file.""" + run_assignment(sqlite_db, file_path, report_path) + + +def run_assignment(sqlite_db, file_path, report_path="test-results.json"): + """Write complete feedback, including malformed submissions and dependencies.""" + results = [] + reference = mutations = None + # Real grading can give Q9 and Q10 the same timestamp, so we control the clock and deliberately advance it. + clock = [datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0).isoformat(" ")] + start = datetime.fromisoformat(clock[0]) + context = {} + statuses = {} + try: + if Path(file_path).name.lower() != "assignment2.sql": + raise ValueError("This grader only supports Assignment 2.") + answers = load_queries(file_path) + reference = clone_database(sqlite_db) + mutations = clone_database(sqlite_db, clock) + for parsed in answers: + number, query = parsed["number"], parsed["query"] + rows, error = [], None + conn = None + try: + if not sql_tokens(query): + status, feedback = "missing", "No SQL answer was submitted." + elif number >= 10 and statuses.get(9) != "correct": + status, feedback = "blocked", "Question 9 must create a valid snapshot first." + elif number == 11 and statuses.get(10) != "correct": + status, feedback = "blocked", "Question 10 must insert a valid product first." + elif number == 12 and statuses.get(11) not in ("correct", "manual_review"): + status, feedback = "blocked", "Question 11 must preserve the intended product rows first." + else: + # Q9 starts at zero; each later mutation advances one second so older rows are identifiable. + clock[0] = (start + timedelta(seconds=max(0, number - 9))).isoformat(" ") + context["execution_time"] = clock[0] + conn = clone_database(sqlite_db, clock) if number < 9 else mutations + before = product_units_state(conn) if number >= 10 else None + rows = run_query(conn, query) + if number < 9: + status, feedback = grade_select(number, rows, reference) + else: + status, feedback = grade_mutation(number, conn, reference, context, before) + technique = technique_feedback(number, query) + if status == "correct" and technique: + status, feedback = "incorrect", technique + except Exception as exception: + status, feedback, error = "error", "The SQL could not be executed or assessed.", exception + finally: + if conn is not None and number < 9: + conn.close() + statuses[number] = status + results.append(build_result(number, query, status, feedback, rows, error)) + except Exception as exception: + results = [build_result(n, "", "error", "Submission could not be parsed or loaded.", error=exception) + for n in range(1, 13)] + finally: + if reference is not None: + reference.close() + if mutations is not None: + mutations.close() + Path(report_path).write_text(json.dumps(results, indent=2)) + return results diff --git a/03_instructional_team/tests/test_grader_regressions.py b/03_instructional_team/tests/test_grader_regressions.py new file mode 100644 index 000000000..8be05fdd7 --- /dev/null +++ b/03_instructional_team/tests/test_grader_regressions.py @@ -0,0 +1,277 @@ +"""Check that the autograder makes the right marking decisions. + +Public checks cover runner safety without exposing answers. When the private staff +rubric is supplied through A2_SOLUTION_RUBRIC, the same suite also creates correct +and deliberately wrong submissions to test every marking decision end to end. +All sample submissions and reports are created in temporary folders. +""" +from pathlib import Path +import hashlib +import json +import os +import re +import runpy +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +# Find the repository root, regardless of where this test file is launched from. +REPO = Path(__file__).resolve().parents[2] +# Load the real grader functions so these tests exercise the code used for marking. +G = runpy.run_path(str(Path(__file__).with_name("test_assignment.py"))) +SOLUTION_RUBRIC = os.environ.get("A2_SOLUTION_RUBRIC") +requires_private_solution = unittest.skipUnless( + SOLUTION_RUBRIC, + "Set A2_SOLUTION_RUBRIC to run private solution-based regression tests.", +) + + +class GraderRegressionTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Prepare the course database and optional private answers once for all tests.""" + cls.path = REPO / "05_src/sql/farmersmarket.db" + # Save a fingerprint of the database so we can detect accidental changes. + cls.original = hashlib.sha256(cls.path.read_bytes()).hexdigest() + # Read-only access protects the original; the grader works on copies. + cls.source = sqlite3.connect(f"file:{cls.path}?mode=ro", uri=True) + cls.blocks = [] + cls.valid = {} + if SOLUTION_RUBRIC: + # The private rubric includes explanations and multiple valid answers. + rubric = Path(SOLUTION_RUBRIC).read_text() + cls.blocks = re.findall(r"```\n(.*?)```", rubric, re.S) + # These positions pick one correct answer for each of the 12 questions. + cls.valid = dict(zip(range(1, 13), [cls.blocks[i].strip() for i in + (1, 2, 4, 6, 8, 9, 10, 11, 12, 13, 14, 16)])) + + @classmethod + def tearDownClass(cls): + """Close the database and confirm that testing did not change the original file.""" + cls.source.close() + assert hashlib.sha256(cls.path.read_bytes()).hexdigest() == cls.original + + def run_submission(self, changes=None, text=None): + """Make a temporary submission, run the grader, and return its feedback by question.""" + with tempfile.TemporaryDirectory() as folder: + folder = Path(folder) + submission = folder / "assignment2.sql" + # Start with correct answers and replace only the questions being tested. + # Supplying raw text instead lets us test missing or broken query markers. + answers = self.valid | (changes or {}) + if text is None: + if not self.valid: + raise RuntimeError("Private solution rubric is required for this test.") + text = "\n".join(f"--QUERY {n}\n{sql}\n--END QUERY" for n, sql in answers.items()) + submission.write_text(text) + report = folder / "report.json" + # Plant an old report first: every run must replace it with fresh feedback. + report.write_text('[{"stale": true}]') + G["run_assignment"](self.source, submission, report) + results = json.loads(report.read_text()) + # Every report must account for all 12 questions, even if answers are missing. + self.assertEqual([r["number"] for r in results], list(range(1, 13))) + return {r["number"]: r for r in results} + + @requires_private_solution + def test_all_instructor_answers_pass_without_timestamp_sleep(self): + """Correct instructor answers should all pass, even when run immediately in sequence.""" + self.assertTrue(all(r["status"] == "correct" for r in self.run_submission().values())) + + @requires_private_solution + def test_wrong_answers_are_rejected_with_valid_prerequisites(self): + """Give each question a wrong answer while keeping the other answers correct.""" + wrong = { + 1: 'SELECT product_name FROM product', + 2: 'SELECT customer_id,market_date,99 AS visit_number FROM customer_purchases', + 3: 'SELECT customer_id,market_date,99 AS visit_number FROM customer_purchases', + 4: 'SELECT *,99 AS product_purchase_count FROM customer_purchases', + 5: 'SELECT product_name,NULL AS description FROM product', + 6: 'SELECT * FROM product', + 7: 'SELECT market_date,0 AS sales FROM customer_purchases GROUP BY market_date LIMIT 2', + 8: 'SELECT vendor_name,product_name,0 AS revenue FROM vendor CROSS JOIN product', + 9: self.valid[9].replace('WHERE product_qty_type = "unit"', ''), + 10: 'SELECT 1', 11: 'DELETE FROM product_units WHERE 1=0', + 12: 'ALTER TABLE product_units ADD current_quantity INT; UPDATE product_units SET current_quantity=-1;', + } + # Test one wrong answer at a time so an earlier failure cannot hide the result. + for number, sql in wrong.items(): + with self.subTest(question=number): + self.assertEqual(self.run_submission({number: sql})[number]['status'], 'incorrect') + + def test_missing_blank_and_comment_only_answers_are_explicit(self): + """An empty file, comments, or omitted questions must be reported as missing answers.""" + for text in ('', '--QUERY 1\n-- unanswered\n--END QUERY'): + with self.subTest(text=text): + self.assertTrue(all(r['status'] == 'missing' for r in self.run_submission(text=text).values())) + results = self.run_submission(text='--QUERY 1\nSELECT 1\n--END QUERY') + self.assertEqual(results[12]['status'], 'missing') + + def test_malformed_submissions_always_replace_stale_report(self): + """Broken query markers must produce fresh errors rather than reuse an old report.""" + for text in ('--QUERY 1\nSELECT 1', '--QUERY 1\nSELECT 1\n--END QUERY\n--QUERY 1\nSELECT 1\n--END QUERY', + '--QUERY 99\nSELECT 1\n--END QUERY', '--QUERY nope', '--END QUERY', + '--QUERY 1\n--QUERY 2'): + with self.subTest(text=text): + self.assertTrue(all(r['status'] == 'error' for r in self.run_submission(text=text).values())) + + @requires_private_solution + def test_valid_window_alternatives(self): + """Accept the different visit-numbering and purchase-count approaches allowed by A2.""" + for number, index in ((2, 3), (3, 5), (4, 7)): + with self.subTest(question=number): + self.assertEqual(self.run_submission({number: self.blocks[index]})[number]['status'], 'correct') + + def test_semicolons_inside_literals_and_comments(self): + """A semicolon in a product name or comment must not break a valid SQL command.""" + conn = G['clone_database'](self.source) + try: + rows = G['run_query'](conn, "CREATE TEMP TABLE sample(value); INSERT INTO sample VALUES ('Pie; special'); -- comment;\n SELECT * FROM sample;; -- trailing;\n;") + self.assertEqual(rows, [{'value': 'Pie; special'}]) + finally: + conn.close() + + @requires_private_solution + def test_techniques_cannot_be_satisfied_by_comments_or_literals(self): + """Mentioning REGEXP or UNION in comments or text does not count as using it.""" + sql = "SELECT * FROM product WHERE product_size GLOB '*[0-9]*' -- REGEXP" + self.assertEqual(self.run_submission({6: sql})[6]['status'], 'incorrect') + self.assertIsNotNone(G['technique_feedback'](7, "SELECT 'UNION' /* UNION */")) + self.assertIsNone(G['technique_feedback'](7, self.valid[7])) + + @requires_private_solution + def test_description_alias_is_accepted(self): + """Accept correct Q5 descriptions when the student chooses a different column alias.""" + sql, replacements = re.subn(r'\bAS\s+description\b', 'AS product_description', + self.valid[5], flags=re.I) + self.assertEqual(replacements, 1) + self.assertEqual(self.run_submission({5: sql})[5]['status'], 'correct') + + @requires_private_solution + def test_sales_alias_and_dates_only_are_accepted(self): + """Accept a different sales column name or an answer showing just the requested dates.""" + alias = self.valid[7].replace('sales, sales_rank_desc AS sales_rank', 'sales AS gross_income, sales_rank_desc AS sales_rank') + date_only = re.sub(r'SELECT market_date, sales, sales_rank_desc AS sales_rank', 'SELECT market_date', self.valid[7]) + for sql in (alias, date_only): + self.assertEqual(self.run_submission({7: sql})[7]['status'], 'correct') + + @requires_private_solution + def test_reordered_snapshot_columns(self): + """Putting the required product columns in a different order should still pass.""" + sql = self.valid[9].replace('p.*', 'product_name,product_id,product_size,product_category_id,product_qty_type') + self.assertEqual(self.run_submission({9: sql})[9]['status'], 'correct') + + @requires_private_solution + def test_invalid_and_stale_timestamps_fail(self): + """Reject timestamps that are meaningless text or dates far in the past.""" + for n, value in ((9, "'not a timestamp'"), (10, "'1900-01-01'")): + self.assertEqual(self.run_submission({n: self.valid[n].replace('CURRENT_TIMESTAMP', value)})[n]['status'], 'incorrect') + + @requires_private_solution + def test_current_datetime_forms_use_the_same_controlled_clock(self): + """Equivalent ways to get the current time must follow the same grading clock.""" + for expression in ("DATETIME('now')", "STRFTIME('%Y-%m-%d %H:%M:%S','now')"): + changes = {n: self.valid[n].replace('CURRENT_TIMESTAMP', expression) for n in (9, 10)} + results = self.run_submission(changes) + self.assertTrue(all(results[n]['status'] == 'correct' for n in (9, 10, 11))) + + @requires_private_solution + def test_inserted_copy_cannot_have_an_older_timestamp(self): + """A newly inserted copy must not be dated earlier than the original copy.""" + expression = "DATETIME((SELECT MIN(snapshot_timestamp) FROM product_units WHERE product_id=7), '-1 second')" + results = self.run_submission({10: self.valid[10].replace('CURRENT_TIMESTAMP', expression)}) + self.assertEqual(results[10]['status'], 'incorrect') + + @requires_private_solution + def test_new_product_is_accepted_and_ambiguous_delete_needs_review(self): + """Allow a new product; ask an instructor to assess deletion when no older copy exists.""" + results = self.run_submission({10: "INSERT INTO product_units VALUES (999,'New Pie','10 inch',3,'unit',CURRENT_TIMESTAMP)", + 11: 'DELETE FROM product_units WHERE 1=0'}) + self.assertEqual(results[10]['status'], 'correct') + self.assertEqual(results[11]['status'], 'manual_review') + self.assertEqual(results[12]['status'], 'correct') + + @requires_private_solution + def test_delete_must_keep_inserted_copy(self): + """Deleting both product copies must fail: the newer copy needs to remain.""" + self.assertEqual(self.run_submission({11: 'DELETE FROM product_units WHERE product_id=7'})[11]['status'], 'incorrect') + + @requires_private_solution + def test_equal_timestamps_allow_one_copy_to_survive(self): + """When timestamps tie, accept deleting the original row and reject deleting the newer row.""" + same = self.valid[10].replace('CURRENT_TIMESTAMP', '(SELECT MIN(snapshot_timestamp) FROM product_units WHERE product_id=7)') + one = 'DELETE FROM product_units WHERE rowid=(SELECT MIN(rowid) FROM product_units WHERE product_id=7)' + self.assertEqual(self.run_submission({10: same, 11: one})[11]['status'], 'correct') + self.assertEqual(self.run_submission({10: same})[11]['status'], 'incorrect') + newer = one.replace('MIN(rowid)', 'MAX(rowid)') + self.assertEqual(self.run_submission({10: same, 11: newer})[11]['status'], 'incorrect') + + @requires_private_solution + def test_update_cannot_delete_or_modify_other_values(self): + """Updating quantities must not erase products or change their other details.""" + for sql in ('ALTER TABLE product_units ADD current_quantity INT; DELETE FROM product_units;', + self.valid[12] + "; UPDATE product_units SET product_name='Changed';"): + self.assertEqual(self.run_submission({12: sql})[12]['status'], 'incorrect') + + def test_prerequisite_failure_is_distinguished_from_wrong_answer(self): + """If table creation fails, explain that later questions cannot be assessed yet.""" + text = '--QUERY 9\nCREATE TABLE product_units AS SELECT * FROM missing_table\n--END QUERY' + text += ''.join(f'\n--QUERY {n}\nSELECT 1;\n--END QUERY' for n in (10, 11, 12)) + results = self.run_submission(text=text) + self.assertEqual(results[9]['status'], 'error') + self.assertTrue(all(results[n]['status'] == 'blocked' for n in (10, 11, 12))) + + def test_sql_cannot_attach_disk_database(self): + """Student SQL must not open another database file and write to it.""" + with tempfile.TemporaryDirectory() as folder: + target = Path(folder) / 'disk.db' + text = f"--QUERY 1\nATTACH DATABASE '{target}' AS disk; CREATE TABLE disk.test(value);\n--END QUERY" + results = self.run_submission(text=text) + self.assertEqual(results[1]['status'], 'error') + self.assertFalse(target.exists()) + + def test_long_sql_is_interrupted(self): + """Stop an excessively long query instead of letting it run without a time limit.""" + conn = G['clone_database'](self.source) + # Pretend time has advanced past the limit, avoiding a real ten-second wait. + ticks = iter((0, 11)) + try: + with patch('time.monotonic', side_effect=lambda: next(ticks, 11)): + with self.assertRaises(sqlite3.OperationalError): + G['run_query'](conn, "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<10000000) SELECT SUM(x) FROM n;") + finally: + conn.close() + + def test_runner_requires_one_file_and_reports_failures(self): + """Require one submission file and give explicit feedback for selection or runner failures.""" + runner = runpy.run_path(str(REPO / '.github/scripts/grade_assignments.py')) + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / 'combined.json' + calls = [] + # Stand in for pytest here so we can inspect which files the runner selects. + # The other tests exercise the actual grader; this one tests orchestration. + def fake_run(command, **kwargs): + calls.append(command) + report = Path(next(arg.split('=', 1)[1] for arg in command if arg.startswith('--report_path='))) + report.write_text(json.dumps([{'number': n} for n in range(1, 13)])) + return type('Result', (), {'returncode': 0})() + with patch('subprocess.run', fake_run): + self.assertEqual(runner['grade_files'](['DC/assignment2.sql'], output), 0) + self.assertEqual(len(calls), 1) + self.assertEqual(len(json.loads(output.read_text())), 1) + for files, message in (([], 'No changed'), + (['DC/assignment2.sql', 'Micro/assignment2.sql'], 'More than one')): + with self.subTest(files=files), patch('subprocess.run', fake_run): + calls.clear() + self.assertEqual(runner['grade_files'](files, output), 1) + self.assertFalse(calls) + self.assertIn(message, json.loads(output.read_text())[0]['runner_error']) + # Simulate a runner crash and check that an old success report is replaced. + with patch('subprocess.run', side_effect=RuntimeError('Runner unavailable')): + self.assertEqual(runner['grade_files'](['DC/assignment2.sql'], output), 1) + self.assertIn('Runner unavailable', json.loads(output.read_text())[0]['runner_error']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_assignment.py b/tests/test_assignment.py deleted file mode 100644 index 510b410a6..000000000 --- a/tests/test_assignment.py +++ /dev/null @@ -1,112 +0,0 @@ -import json -from pathlib import Path -import sqlite3 -import re - -def load_queries(sql_file): - lines = Path(sql_file).read_text().splitlines() - - queries = [] - current_query = None - buffer = [] - - for line in lines: - stripped = line.strip() - - # Start marker: -- QUERY - if stripped.lower().startswith("--query"): - if current_query is not None: - raise AssertionError("Nested QUERY blocks are not allowed") - - parts = stripped.split() - if len(parts) < 2: - raise AssertionError(f"Invalid QUERY marker: {line}") - - try: - current_query = int(parts[1]) - except ValueError: - raise AssertionError(f"Invalid QUERY number: {line}") - - buffer = [] - continue - - # End marker: -- END QUERY or -- END STATEMENT - if stripped.lower().startswith("--end"): - if current_query is None: - continue # ignore stray END - - query = "\n".join(buffer).strip().rstrip(";") - - queries.append({ - "number": current_query, - "query": query - }) - - current_query = None - buffer = [] - continue - - # Collect lines inside a query block - if current_query is not None: - buffer.append(line) - - if not queries: - raise AssertionError( - "No queries found. Use '-- QUERY ' and '-- END QUERY' markers." - ) - - return queries - -def run_query(conn, query): - cursor = conn.cursor() - try: - cursor.execute(query) - rows = cursor.fetchall() - return [dict(row) for row in rows] - - except sqlite3.ProgrammingError as e: - if "one statement at a time" in str(e).lower(): - # fallback: handle multiple statements - statements = [s.strip() for s in query.split(";") if s.strip()] - - if len(statements) == 0: - return [] - - for stmt in statements[:-1]: - cursor.execute(stmt) - - cursor.execute(statements[-1]) - rows = cursor.fetchall() - return [dict(row) for row in rows] - - else: - raise - -def test_assignment(sqlite_db, file_path): - run_assignment(sqlite_db, file_path) - -def run_assignment(sqlite_db, file_path): - with open("test-results.json", "w") as json_file: - queries = load_queries(file_path) - test_result = [] - for parsed_query in queries: - try: - rows = run_query(sqlite_db, parsed_query["query"]) - test_result.append({ - "number": parsed_query["number"], - "query": parsed_query["query"], - "result": rows[0:3], - "error": None - }) - except Exception as e: - test_result.append({ - "number": parsed_query["number"], - "query": parsed_query["query"], - "result": [], - "error": str(e) - }) - json.dump(test_result, json_file, indent=2) - - # The purpose of it to have report in the future in case DSI will want to go with unit testing style. - # assert True, "test execution query {} result {}".format(queries, test_result) -