diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1672fcc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,35 @@ +.git +.github +.gitignore +.idea +.vscode +__pycache__/ +*.py[cod] +.venv/ +venv/ +env/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ +coverage.xml +tests/ +test-sites/ +docs/ +*.md +.env +.env.* +data/ +*.db +*.db-journal +*.db-wal +*.db-shm +*.log +logs/ +node_modules/ +.DS_Store +Dockerfile +.dockerignore +.pre-commit-config.yaml +Makefile diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c650a9d --- /dev/null +++ b/.env.example @@ -0,0 +1,81 @@ +# ---- App ---- +APP_NAME="LinClean Open API" +APP_VERSION="0.1.0" +ENVIRONMENT=local +DEBUG=true + +# ---- Server ---- +HOST=0.0.0.0 +PORT=8000 +LOG_LEVEL=INFO +LOG_JSON=false + +# ---- CORS (JSON list) ---- +CORS_ORIGINS=["http://localhost:3000"] + +# ---- Database (SQLite) ---- +# Relative paths are resolved against the project root. +SQLITE_PATH=data/linclean.db +DB_ECHO=false + +# ---- Google Safe Browsing ---- +# Leave empty to disable GSB lookups in /api/v1/check. +GSB_API_KEY=SECRET_KEY +GSB_CLIENT_ID=linclean +GSB_CLIENT_VERSION=0.1.0 +GSB_TIMEOUT_SECONDS=5.0 + +# ---- URLhaus ---- +URLHAUS_RECENT_CSV_URL=https://urlhaus.abuse.ch/downloads/csv_recent/ +URLHAUS_ONLINE_CSV_URL=https://urlhaus.abuse.ch/downloads/csv_online/ +URLHAUS_REFRESH_INTERVAL_SECONDS=3600 +URLHAUS_DOWNLOAD_TIMEOUT_SECONDS=60.0 + +# ---- Scheduler / URLhaus sync ---- +# 스케줄러 켜면 URLHAUS_REFRESH_INTERVAL_SECONDS 주기로 CSV 동기화 실행. +# 부팅 시 1회 동기화는 URLHAUS_SYNC_ON_STARTUP 로 제어. 로컬/CI 에서 비활성화 가능. +SCHEDULER_ENABLED=true +URLHAUS_SYNC_ON_STARTUP=true + +# ---- RDAP ---- +RDAP_BOOTSTRAP_URL=https://rdap.org/domain/ +RDAP_TIMEOUT_SECONDS=5.0 +RDAP_CACHE_TTL_SECONDS=604800 +RDAP_NEW_DOMAIN_THRESHOLD_DAYS=30 + +# ---- Scoring ---- +SCORE_WEIGHT_GSB=50 +SCORE_WEIGHT_URLHAUS=50 +SCORE_WEIGHT_RDAP=20 +SCORE_MALICIOUS_THRESHOLD=50 +# 1~3단계 합산이 이 값 이상이면 콘텐츠 정적 분석(네트워크·AI 비용)을 건너뛴다. +SCORE_DANGER_THRESHOLD=61 + +# ---- Content Analysis (4단계) ---- +CONTENT_FETCH_TIMEOUT_SECONDS=8.0 +CONTENT_FETCH_CONNECT_TIMEOUT_SECONDS=3.0 +CONTENT_FETCH_MAX_BYTES=2097152 +CONTENT_EXTERNAL_LINK_RATIO_THRESHOLD=0.8 + +# 페치 실패 가중치 — timeout/connect_error/http_error 같은 도달 실패에만 적용. +# not_html / too_large / unexpected_redirect 는 정상 컨텐츠·파이프라인 이슈로 보고 가산 없이 시그널만. +SCORE_WEIGHT_CONTENT_FETCH_FAILED=10 + +# ---- AI Provider ---- +# auto : OPENAI_API_KEY 있으면 OpenAIProvider, 없으면 NullAIProvider +# openai : 강제 openai (키 없으면 NullAIProvider 로 폴백 + 경고) +# null : 비활성 (규칙 점수만 사용) +AI_PROVIDER=auto + +# ---- OpenAI ---- +# 비워두면 NullAIProvider — 규칙 기반 점수만 사용. +# 모델은 OpenAI 채팅 계열이면 전부 호환 (gpt-4o-mini / gpt-4o / gpt-4.1-mini 등). 교체는 OPENAI_MODEL 한 줄. +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o-mini +OPENAI_TIMEOUT_SECONDS=10.0 +OPENAI_MAX_OUTPUT_TOKENS=300 + +# ---- Spring 통신 ---- +# secrets.token_hex(32) 로 생성한 사전 공유 키. Spring 과 동일한 값 설정 필수. +INTERNAL_API_KEY=REPLACE_ME +SPRING_INTERNAL_URL=http://localhost:8080 diff --git "a/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" "b/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" new file mode 100644 index 0000000..8175bae --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" @@ -0,0 +1,24 @@ +--- +name: 기능 개발 +about: 새로운 기능을 추가하거나 수정 +title: "[FEATURE] 기능 개발" +labels: "✨ Feature" +assignees: + +--- + +## 기능 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 구현할 기능에 대한 설계를 마쳤는가? +- [ ] 테스트 코드를 작성했는가? +- [ ] API 명세서(Swagger 등)를 업데이트했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" "b/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" new file mode 100644 index 0000000..17800c0 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" @@ -0,0 +1,23 @@ +--- +name: 기타 업무 +about: 프로덕션 코드가 바뀌지 않고 개발 로직과 상관 없는 가벼운 일들 +title: "[CHORE] 기타 업무" +labels: "🗑 chore" +assignees: + +--- + +## 기타 업무 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 어떤 목적으로 수행되는 작업인지 명확히 작성했는가? +- [ ] 이 작업이 다른 개발 환경에 영향을 주지 않는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" "b/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" new file mode 100644 index 0000000..0e74675 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" @@ -0,0 +1,24 @@ +--- +name: 리팩토링 +about: production 코드를 리팩토링 +title: "[REFACTOR] 리팩토링" +labels: "🔧 Refactor" +assignees: + +--- + +## 리팩토링 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 로직의 가독성이 좋아졌는가? +- [ ] 단일 책임 원칙(SRP)을 잘 따랐는가? +- [ ] 기존 테스트가 모두 통과하는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..77f0fda --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 문서 작성 및 수정 +about: 도큐먼트/문서화 업데이트 +title: "[DOCS] 문서 작성 및 수정" +labels: "📄 Docs" +assignees: + +--- + +## 문서 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 문서의 내용이 최신 코드를 반영하고 있는가? +- [ ] 오타나 잘못된 링크는 없는가? +- [ ] 기술 용어가 일관되게 사용되었는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..b93f346 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 버그 수정 +about: 버그를 고침 +title: "[FIX] 버그 수정" +labels: "🐞 Bugfix" +assignees: + +--- + +## 버그 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 버그가 발생하는 원인을 분석했는가? +- [ ] 버그가 재현되는 테스트 코드를 작성했는가? +- [ ] 수정 사항이 기존 기능에 영향을 주지 않는지 확인했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" "b/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" new file mode 100644 index 0000000..aa696f7 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" @@ -0,0 +1,24 @@ +--- +name: 빌드 관련 +about: 빌드 관련 수정, 모듈 설치 또는 삭제 등 +title: "[BUILD] 빌드 관련" +labels: "🛠 Build" +assignees: + +--- + +## 빌드 관련 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 의존성이 올바르게 추가/삭제 되었는가? +- [ ] 로컬 환경에서 빌드(./gradlew build)가 성공하는가? +- [ ] 환경 변수나 설정 파일(.properties, .yml)의 변경이 필요한가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..e2a3733 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 시급한 버그 수정 +about: 시급한 버그를 고침 - 현 production에 critical +title: "[HOTFIX] 버그 수정" +labels: "🐞 HOTFIX" +assignees: + +--- + +## 시급한 버그 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 버그를 재현할 수 있는 테스트 코드를 작성했는가? +- [ ] 해결책이 기존 코드의 다른 기능에 영향을 주지 않는가? +- [ ] 로그나 에러 메시지를 충분히 확인했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" "b/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" new file mode 100644 index 0000000..ee9a6fb --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" @@ -0,0 +1,24 @@ +--- +name: 코드 형태 변경 +about: 주석, 프리티어 등 기능의 영향없이 코드 모양새만 변경되는 경우 +title: "[STYLE] 코드 형태 변경" +labels: "💄 Style" +assignees: + +--- + +## 코드 형태 변경 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 비즈니스 로직에 영향이 없는가? +- [ ] 정해진 코드 스타일 가이드를 준수했는가? +- [ ] 불필요한 주석이나 공백을 제거했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" "b/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" new file mode 100644 index 0000000..81a344c --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" @@ -0,0 +1,24 @@ +--- +name: 테스트 코드 +about: 테스트 코드 추가 및 업데이트 +title: "[TEST] 테스트" +labels: "✅ Test" +assignees: + +--- + +## 테스트 코드 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 테스트 케이스가 모든 엣지 케이스를 포함하는가? +- [ ] 테스트가 독립적으로 실행 가능한가? +- [ ] 테스트 코드의 네이밍이 이해하기 쉬운가? + +### 참고자료 + + \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..58b9b19 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +# 요약 + + +# 연관 이슈 및 Close 할 이슈 작성 +(fix #일이삼) + + +-- PR 시 다음과 같이 작성 +#123 + +close #123 + +# Pull Request 체크리스트 + +## TODO + +- [ ] 최종 결과물을 확인했는가? +- [ ] 의미 있는 커밋 메시지를 작성했는가? + \ No newline at end of file diff --git a/.github/workflows/auto-close-issues.yml b/.github/workflows/auto-close-issues.yml new file mode 100644 index 0000000..eeb3692 --- /dev/null +++ b/.github/workflows/auto-close-issues.yml @@ -0,0 +1,57 @@ +name: Auto Close Issues + +on: + pull_request: + types: [closed] + +jobs: + close-issues: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - name: Close linked issues + uses: actions/github-script@v7 + with: + script: | + const prBody = context.payload.pull_request.body || ''; + const prTitle = context.payload.pull_request.title || ''; + + // PR 본문과 제목에서 이슈 번호 찾기 + const issueRegex = /(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)\s+#(\d+)/gi; + const matches = [...prBody.matchAll(issueRegex), ...prTitle.matchAll(issueRegex)]; + + for (const match of matches) { + const issueNumber = parseInt(match[1]); + + try { + // 이슈 상태 확인 + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber + }); + + if (issue.data.state === 'open') { + // 이슈 종료 + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + state: 'closed' + }); + + // 백로그 라벨 추가 (선택사항) + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['백로그'] + }); + + console.log(`Closed issue #${issueNumber}`); + } + } catch (error) { + console.log(`Failed to close issue #${issueNumber}: ${error.message}`); + } + } \ No newline at end of file diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..56b58de --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,86 @@ +name: CD + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: # 수동 실행 (테스트/재배포/롤백) + +permissions: + id-token: write # OIDC + contents: read + +jobs: + build-and-push: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.vars.outputs.sha }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + - id: vars + run: echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: | + ${{ vars.IMAGE_NAME }}:latest + ${{ vars.IMAGE_NAME }}:${{ steps.vars.outputs.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + deploy: + needs: build-and-push + runs-on: ubuntu-latest + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + - name: Deploy via SSM SendCommand + run: | + cmd_id=$(aws ssm send-command \ + --instance-ids "${{ secrets.APP_INSTANCE_ID }}" \ + --document-name "AWS-RunShellScript" \ + --comment "deploy fastapi ${{ needs.build-and-push.outputs.sha }}" \ + --parameters commands='[ + "set -e", + "cd ${{ vars.DEPLOY_DIR }}", + "docker compose pull ${{ vars.DEPLOY_SERVICE }}", + "docker compose up -d ${{ vars.DEPLOY_SERVICE }}", + "docker image prune -f" + ]' \ + --query "Command.CommandId" --output text) + echo "CommandId=$cmd_id" + # 완료까지 폴링 후 상태 검증 + for i in $(seq 1 30); do + status=$(aws ssm get-command-invocation \ + --command-id "$cmd_id" \ + --instance-id "${{ secrets.APP_INSTANCE_ID }}" \ + --query "Status" --output text 2>/dev/null || echo Pending) + echo "status=$status" + case "$status" in + Success) break ;; + Failed|Cancelled|TimedOut) + aws ssm get-command-invocation --command-id "$cmd_id" \ + --instance-id "${{ secrets.APP_INSTANCE_ID }}" \ + --query "StandardErrorContent" --output text + exit 1 ;; + esac + sleep 10 + done + # 루프가 타임아웃(300s)으로 끝났는데 Success 가 아니면 배포 실패로 처리. + # break 없이 빠져나온 Pending/InProgress 상태를 성공으로 오인하지 않도록 명시 검증. + if [ "$status" != "Success" ]; then + echo "deploy did not reach Success within timeout (last status=$status)" + exit 1 + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ad8e82d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + pull_request: + branches: [dev] + push: + branches: [dev, main] + +jobs: + check: + runs-on: ubuntu-latest + env: + TLDEXTRACT_CACHE: /tmp/tldextract # 리눅스 러너 캐시 경로 보정 + INTERNAL_API_KEY: test-ci-key # 테스트용 더미값 — Settings 필수 필드라 없으면 import 단계에서 실패 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + - run: pip install -e ".[dev]" + - name: Ruff lint + run: ruff check app tests + - name: Pytest + run: pytest + - name: Docker build + run: docker build -t linclean-fastapi:ci . diff --git a/.gitignore b/.gitignore index f7296e0..d0425e6 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,12 @@ env/ htmlcov/ coverage.xml +# JavaScript test-site builds +test-sites/ +node_modules/ +.next/ +tsconfig.tsbuildinfo + # Logs *.log logs/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a9c0dcc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + # app 이 site-packages 에 설치돼 alembic/uvicorn 이 그쪽을 import 하므로 + # SQLite 경로를 절대경로로 고정 (PROJECT_ROOT 추론이 site-packages 를 가리켜 + # 쓰기 불가 위치에 DB 디렉토리를 만들려다 실패하는 문제 방지). + SQLITE_PATH=/app/data/linclean.db \ + TLDEXTRACT_CACHE=/tmp/tldextract + +WORKDIR /app + +# hatchling 이 app 패키지를 빌드해야 하므로 install 전에 소스 복사 +COPY pyproject.toml ./ +COPY app ./app +RUN pip install . + +# 마이그레이션 + 기동 스크립트 +COPY alembic.ini ./ +COPY alembic ./alembic +COPY entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +# non-root + 데이터 디렉토리(SQLite) 소유권 +RUN useradd -m -u 1000 appuser \ + && mkdir -p /app/data \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/v1/health').status==200 else 1)" + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/Makefile b/Makefile index 0c4654a..5c7c353 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,12 @@ install: ## Install runtime + dev dependencies pip install -e ".[dev]" pre-commit install -run: ## Run the API locally with hot reload - uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 +run: ## Run the API locally + uvicorn app.main:app --host 0.0.0.0 --port 8000 + +dev: ## Run the API locally with hot reload + uvicorn app.main:app --reload --reload-dir app --reload-exclude .venv \ + --host 0.0.0.0 --port 8000 test: ## Run the test suite pytest diff --git a/README.md b/README.md new file mode 100644 index 0000000..c4ae946 --- /dev/null +++ b/README.md @@ -0,0 +1,269 @@ +# LinClean FastAPI + +## 핵심 목표 + +LinClean FastAPI는 Spring 서비스가 전달한 URL을 열기 전에 분석해 `safe`, `caution`, `danger` verdict와 근거를 반환하는 URL 보안 엔진입니다. + +주요 목표는 다음과 같습니다. + +- 단축 URL과 리다이렉트를 풀어 실제 도착지를 확인합니다. +- Google Safe Browsing, URLhaus, 도메인 휴리스틱, 콘텐츠 분석, AI 보조 판정을 한 파이프라인에서 합산합니다. +- 사라진 페이지나 400번대 페이지는 무리하게 verdict를 만들지 않고 실패 상태로 Spring에 콜백합니다. +- `caution` 이상 판정에서는 AI 분석 근거와 사용자 행동 가이드를 기존 `ai_reason` 안에 100자 이내로 담습니다. + +## 기술 스택 + +| 구분 | 기술 | +|---|---| +| Language | Python 3.13 | +| API | FastAPI, Pydantic v2 | +| DB | SQLite, SQLAlchemy Async, Alembic | +| HTTP | httpx | +| Scheduler | APScheduler | +| HTML 분석 | BeautifulSoup4, lxml | +| 도메인 분석 | tldextract, RDAP | +| AI | OpenAI Chat Completions, NullAIProvider fallback | +| 테스트 | pytest, pytest-asyncio | +| 품질 도구 | Ruff, mypy | +| 선택 기능 | Playwright 렌더링 분석 | + +## 디렉토리 구조 + +```text +linclean-fastapi/ + app/ + api/ # FastAPI 라우터, 인증 의존성, 에러 핸들러 + core/ # 설정, 로깅, 스케줄러, DNS 캐시 + db/ # SQLAlchemy async engine/session + middleware/ # request_id, access log + models/ # URLhaus 캐시용 ORM 모델 + schemas/ # Pydantic 요청/응답 모델 + services/ + normalizer/ # URL 정규화 + unchainer/ # 리다이렉트 추적 + threat_db/ # GSB, URLhaus 조회 + domain_heuristic/ # 도메인/URL 휴리스틱 + content_analyzer/ # HTML fetch/extract/signals/AI + pipeline.py # DB 의존 전체 파이프라인 + db_independent_pipeline.py + analysis_callback.py # Spring 콜백 + page_unavailability.py + alembic/ # DB migration + data/ # SQLite 파일 위치 + reports/ # 날짜별 평가 결과, 커밋 제외 + scripts/ # 평가/운영 스크립트, 커밋 제외 + tests/ # 단위/통합 테스트 +``` + +## 파이프라인 구조 + +```text +입력 URL + -> 1. URL 정규화 + -> 2. 리다이렉트 언체이닝 + -> 3. 외부 위협 DB 조회 + -> 4. 도메인/URL 휴리스틱 + -> 5. 콘텐츠 분석 + -> 6. AI 보조 판정 + -> 점수 합산 및 verdict 산출 + -> 동기 응답 또는 Spring 콜백 +``` + +DB 의존 파이프라인은 GSB와 URLhaus를 포함합니다. DB 비의존 파이프라인은 외부 위협 DB 없이 정규화, 언체이닝, 도메인 휴리스틱, 콘텐츠/AI 분석만으로 판정합니다. + +## 각 단계별 상세 + +1. URL 정규화 + 입력 URL의 스킴, 호스트, 기본 포트, 경로, 인코딩, fragment를 정리합니다. 스킴이 없는 URL은 HTTPS 우선으로 분석 가능한 형태를 만듭니다. + +2. 리다이렉트 언체이닝 + 3xx Location 체인을 따라 최종 URL을 찾습니다. 같은 등록 도메인 안의 canonical redirect는 cross-origin으로 보지 않고, 등록 도메인이 달라질 때만 `REDIRECT_CROSS_ORIGIN` 신호를 남깁니다. + +3. 외부 위협 DB 조회 + Google Safe Browsing과 URLhaus 로컬 캐시를 조회합니다. 알려진 악성 URL이면 점수와 관계없이 `danger`로 판정할 수 있습니다. + +4. 도메인/URL 휴리스틱 + IP 직접 접근, userinfo 포함 URL, 오타 도메인, DGA 유사 도메인, suspicious TLD, open redirect 파라미터, 민감 경로, 무료 호스팅 유도 등을 점수화합니다. 신뢰 도메인은 DGA/typo 오탐을 줄이도록 보정합니다. + +5. 콘텐츠 분석 + 최종 URL의 HTML을 가져와 title, form, password field, 외부 form action, 민감정보 필드, 기관/브랜드 사칭 문구, 위험 다운로드, meta refresh, 외부 링크 비율을 분석합니다. 404 등 찾을 수 없는 페이지는 `PAGE_UNAVAILABLE` 실패로 조기 종료합니다. + +6. AI 보조 판정 + 구조화된 페이지 피처와 선행 단계 신호를 OpenAI에 전달합니다. AI 응답은 `phishing`, `suspicious`, `benign` 중 하나이며, `ai_reason`에는 분석 근거와 사용자 행동 가이드를 100자 이내 한 문장으로 담습니다. 예: `브랜드 사칭 정황이 있어 비밀번호나 결제 정보를 입력하지 마세요.` + +## 점수 산정 기준 + +모든 점수는 합산 후 0~100으로 제한합니다. 외부 위협 DB 매치는 고신뢰 신호로 보며, 휴리스틱과 콘텐츠 분석은 보조 신호로 누적합니다. + +| 영역 | 신호 | 점수 | +|---|---:|---:| +| 외부 DB | GSB match | 50 | +| 외부 DB | URLhaus match | 50 | +| 도메인 | IP 직접 접근 | 40 | +| 도메인 | URL userinfo | 45 | +| 도메인 | typo domain | 30 | +| 도메인 | punycode IDN | 35 | +| 도메인 | open redirect param | 31 | +| 도메인 | DGA-like | 31 | +| 도메인 | suspicious TLD | 25 | +| 도메인 | new domain | 25 | +| 도메인 | no HTTPS, subdomain/hyphen overuse, hosting platform, sensitive path | 각 20 | +| 도메인 | brand in URL | 30 | +| 도메인 | free hosting lure, URL shortener | 각 25 | +| 콘텐츠 | brand impersonation form | 50 | +| 콘텐츠 | external credential form | 45 | +| 콘텐츠 | sensitive ID field | 30 | +| 콘텐츠 | financial field, external meta refresh | 각 25 | +| 콘텐츠 | PII form, risky download, public agency lure, meta refresh | 각 20 | +| 콘텐츠 | Korean lure text | 15 | +| 콘텐츠 | logo alt impersonation | 10 | +| 콘텐츠 | external link overuse | 5 | +| 콘텐츠 | fetch failed | 15 | +| AI | phishing | 45 | +| AI | suspicious | 31 | + +보정 기준: + +- 도메인 휴리스틱 점수는 최대 80점입니다. +- 콘텐츠 분석 점수는 최대 100점입니다. +- 종합 점수는 최대 100점입니다. +- 선행 단계 점수가 61점 이상이면 콘텐츠 분석을 건너뛰고 `danger`로 확정할 수 있습니다. +- `not_html`, `too_large`, `unexpected_redirect`, `blocked_host`는 악성 근거가 아니라 분석 불가 성격이 강해 fetch failed 신호만 남기고 점수는 올리지 않습니다. +- 신뢰 도메인에서 AI `suspicious` 단독 판정은 점수를 올리지 않습니다. 단, 외부 form action, 민감정보 필드, 브랜드 사칭 등 강한 신호가 있으면 반영합니다. + +## verdict 종류 및 구간별 점수 + +| Verdict | 점수 구간 | 의미 | +|---|---:|---| +| `safe` | 0~30 | 현재 기준에서 뚜렷한 위험 신호가 낮음 | +| `caution` | 31~60 | 사용자가 링크, 입력 정보, 결제 정보를 한 번 더 확인해야 함 | +| `danger` | 61~100 | 접속, 로그인, 결제, 다운로드를 피해야 하는 위험 상태 | + +GSB 또는 URLhaus에 악성으로 등록된 경우에는 점수 구간과 별개로 `danger` verdict가 우선됩니다. + +## API 종류및 상세 + +공통 헤더: + +```http +X-Internal-Api-Key: +Content-Type: application/json +``` + +| Method | Path | 설명 | +|---|---|---| +| GET | `/api/v1/health` | 서버 상태 확인 | +| GET | `/api/v1/health/ready` | DB 등 readiness 확인 | +| POST | `/api/v1/analyze` | 비동기 분석 접수. 완료 후 Spring 콜백 전송 | +| POST | `/api/v1/analyze/sync` | GSB, URLhaus 포함 전체 파이프라인 동기 실행 | +| POST | `/api/v1/analyze/db-independent/sync` | 외부 DB 없이 동기 실행 | +| POST | `/api/v1/normalize` | URL 정규화와 리다이렉트 언체이닝 결과 확인 | +| POST | `/api/v1/threat-db` | GSB, URLhaus 조회만 실행 | +| POST | `/api/v1/domain-heuristic` | 도메인/URL 휴리스틱만 실행 | +| POST | `/api/v1/content-analysis` | 콘텐츠 분석과 AI 보조 판정만 실행 | +| POST | `/api/v1/content/fetch-extract` | HTML fetch와 feature 추출만 실행 | + +요청 예시: + +```json +{ + "url": "https://example.com" +} +``` + +비동기 분석 접수는 `analysisId`를 함께 보냅니다. + +```json +{ + "analysisId": "analysis-1", + "url": "https://example.com" +} +``` + +## JSON 응답 예시 + +동기 분석 성공: + +```json +{ + "status": "success", + "analysis_id": "analysis-1", + "original_url": "https://example.com", + "final_url": "https://example.com/", + "verdict": "caution", + "score": 31, + "timings": { + "total_seconds": 1.23, + "stages": { + "normalize": 0.001, + "unchain": 0.12, + "threat_db": 0.03, + "domain_heuristic": 0.2, + "content_analysis": 0.87 + } + }, + "stages": { + "content_analysis": { + "final_url": "https://example.com/", + "fetched": true, + "status_code": 200, + "score": 31, + "signals": [], + "ai_verdict": "suspicious", + "ai_reason": "로그인 유도 정황이 있어 비밀번호나 결제 정보를 입력하지 마세요.", + "ai_error": null, + "ai_model": "gpt-4o-mini" + } + } +} +``` + +찾을 수 없는 페이지: + +```json +{ + "status": "failed", + "analysis_id": "analysis-1", + "original_url": "https://missing.example", + "final_url": "https://missing.example", + "failed_at_stage": "content_analysis", + "error": "페이지를 찾을 수 없습니다.", + "error_code": "PAGE_UNAVAILABLE", + "status_code": 404 +} +``` + +비동기 접수 응답: + +```json +{ + "analysisId": "analysis-1", + "status": "queued" +} +``` + +Spring 성공 콜백은 camelCase로 전송합니다. + +```json +{ + "analysisId": "analysis-1", + "requestId": "request-1", + "status": "succeeded", + "originalUrl": "https://example.com", + "finalUrl": "https://example.com/", + "verdict": "caution", + "score": 31, + "summary": "로그인 유도 정황이 있어 비밀번호나 결제 정보를 입력하지 마세요.", + "stages": { + "contentAnalysis": { + "fetched": true, + "hasPasswordField": true, + "aiVerdict": "suspicious", + "aiReason": "로그인 유도 정황이 있어 비밀번호나 결제 정보를 입력하지 마세요." + } + }, + "engineVersion": "0.1.0", + "analyzedAt": "2026-05-29T00:00:00Z", + "elapsedMs": 1234 +} +``` diff --git a/alembic/env.py b/alembic/env.py index 7854f21..e25e4cc 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,11 +1,10 @@ from logging.config import fileConfig from alembic import context -from sqlalchemy import engine_from_config, pool - from app.core.config import settings from app.db.base import Base -from app.models import * # noqa: F401,F403 -- ensure all models are imported +from app.models import * # noqa: F403 -- ensure all models are imported +from sqlalchemy import engine_from_config, pool config = context.config config.set_main_option("sqlalchemy.url", settings.alembic_database_url) @@ -15,8 +14,7 @@ target_metadata = Base.metadata -# SQLite + Alembic: enable batch mode so ALTER TABLE migrations work, -# since SQLite has very limited native ALTER support. +# SQLite는 ALTER TABLE 지원이 빈약해서 batch 모드 필수. RENDER_AS_BATCH = True diff --git a/alembic/versions/20260415_0610_urlhaus_entry.py b/alembic/versions/20260415_0610_urlhaus_entry.py new file mode 100644 index 0000000..e02ea89 --- /dev/null +++ b/alembic/versions/20260415_0610_urlhaus_entry.py @@ -0,0 +1,61 @@ +"""urlhaus_entry + +Revision ID: 818a3f82f172 +Revises: +Create Date: 2026-04-15 06:10:17.609945+00:00 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "818a3f82f172" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "urlhaus_entries", + sa.Column("id", sa.Integer(), autoincrement=False, nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("host", sa.String(), nullable=False), + sa.Column("match_key", sa.String(), nullable=False), + sa.Column("threat", sa.String(), nullable=True), + sa.Column("tags", sa.String(), nullable=True), + sa.Column("url_status", sa.String(), nullable=True), + sa.Column("date_added", sa.DateTime(), nullable=True), + sa.Column("last_online", sa.DateTime(), nullable=True), + sa.Column("urlhaus_link", sa.String(), nullable=True), + sa.Column("reporter", sa.String(), nullable=True), + sa.Column("synced_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_urlhaus_entries")), + ) + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.create_index(batch_op.f("ix_urlhaus_entries_host"), ["host"], unique=False) + batch_op.create_index( + "ix_urlhaus_entries_host_match_key", ["host", "match_key"], unique=False + ) + batch_op.create_index( + batch_op.f("ix_urlhaus_entries_match_key"), ["match_key"], unique=False + ) + batch_op.create_index(batch_op.f("ix_urlhaus_entries_url"), ["url"], unique=True) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_url")) + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_match_key")) + batch_op.drop_index("ix_urlhaus_entries_host_match_key") + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_host")) + + op.drop_table("urlhaus_entries") + # ### end Alembic commands ### diff --git a/alembic/versions/20260416_drop_unused_urlhaus_indexes.py b/alembic/versions/20260416_drop_unused_urlhaus_indexes.py new file mode 100644 index 0000000..d25c66b --- /dev/null +++ b/alembic/versions/20260416_drop_unused_urlhaus_indexes.py @@ -0,0 +1,34 @@ +"""drop unused urlhaus indexes + +Revision ID: a1b2c3d4e5f6 +Revises: 818a3f82f172 +Create Date: 2026-04-16 00:00:00.000000+00:00 + +현재 조회 핫패스는 `url` 과 `match_key` 인덱스만 사용한다. `host` 단일 인덱스와 +`(host, match_key)` 복합 인덱스는 생성만 되고 참조되는 쿼리가 없어 쓰기 오버헤드만 +발생하므로 제거한다. +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: str | None = "818a3f82f172" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.drop_index("ix_urlhaus_entries_host_match_key") + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_host")) + + +def downgrade() -> None: + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.create_index(batch_op.f("ix_urlhaus_entries_host"), ["host"], unique=False) + batch_op.create_index( + "ix_urlhaus_entries_host_match_key", ["host", "match_key"], unique=False + ) diff --git a/app/api/deps.py b/app/api/deps.py index ac94185..2b50252 100644 --- a/app/api/deps.py +++ b/app/api/deps.py @@ -2,9 +2,23 @@ from typing import Annotated -from fastapi import Depends +from fastapi import Depends, Header, HTTPException, status from sqlalchemy.ext.asyncio import AsyncSession +from app.core.config import settings from app.db.session import get_db DBSession = Annotated[AsyncSession, Depends(get_db)] + + +async def verify_internal_api_key( + x_internal_api_key: Annotated[str | None, Header()] = None, +) -> None: + if x_internal_api_key != settings.internal_api_key: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing X-Internal-Api-Key", + ) + + +InternalApiKey = Annotated[None, Depends(verify_internal_api_key)] diff --git a/app/api/error_handlers.py b/app/api/error_handlers.py index 951b12e..210c920 100644 --- a/app/api/error_handlers.py +++ b/app/api/error_handlers.py @@ -30,9 +30,7 @@ async def app_error_handler(request: Request, exc: AppError) -> JSONResponse: ) @app.exception_handler(StarletteHTTPException) - async def http_exception_handler( - request: Request, exc: StarletteHTTPException - ) -> JSONResponse: + async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: return JSONResponse( status_code=exc.status_code, content=ErrorResponse( diff --git a/app/api/v1/endpoints/analyze.py b/app/api/v1/endpoints/analyze.py new file mode 100644 index 0000000..2186c49 --- /dev/null +++ b/app/api/v1/endpoints/analyze.py @@ -0,0 +1,121 @@ +"""POST /api/v1/analyze 계열 — Spring 으로부터 URL 분석 위임을 받는 엔드포인트. + +- POST /analyze : 비동기 접수 (202). BackgroundTasks 로 파이프라인 실행 후 + 결과를 Spring 에 callback. +- POST /analyze/sync : 동기 실행 (200). 단건 호출 시 1~4단계 + verdict 를 즉시 반환. + 디버그/QA 용. + +두 엔드포인트 모두 X-Internal-Api-Key 헤더 인증을 거친다. +""" + +from __future__ import annotations + +import time +import uuid +from datetime import UTC, datetime + +from fastapi import APIRouter, BackgroundTasks, Request, status +from pydantic import BaseModel + +from app.api.deps import DBSession, InternalApiKey +from app.db.session import SessionLocal +from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineFailure, + DbIndependentPipelineSuccess, +) +from app.schemas.pipeline import PipelineFailure, PipelineSuccess +from app.services.analysis_callback import post_analysis_callback +from app.services.db_independent_pipeline import run_db_independent_pipeline +from app.services.pipeline import run_pipeline + +router = APIRouter() + + +async def _run_pipeline_task(analysis_id: str, original_url: str, request_id: str) -> None: + """백그라운드 태스크 전용 래퍼. 세션을 독립적으로 열고 닫는다.""" + started = time.perf_counter() + async with SessionLocal() as session: + result = await run_pipeline( + analysis_id=analysis_id, + original_url=original_url, + session=session, + ) + elapsed_ms = int((time.perf_counter() - started) * 1000) + await post_analysis_callback( + result, + request_id=request_id, + elapsed_ms=elapsed_ms, + analyzed_at=datetime.now(tz=UTC), + ) + + +@router.post( + "/analyze", + status_code=status.HTTP_202_ACCEPTED, + response_model=AnalyzeAccepted, + response_model_by_alias=True, + summary="URL 분석 위임 접수", +) +async def analyze( + body: AnalyzeRequest, + background_tasks: BackgroundTasks, + request: Request, + _: InternalApiKey, +) -> AnalyzeAccepted: + request_id = str(request.scope.get("request_id") or uuid.uuid4()) + background_tasks.add_task( + _run_pipeline_task, + analysis_id=body.analysis_id, + original_url=body.url, + request_id=request_id, + ) + return AnalyzeAccepted(analysis_id=body.analysis_id) + + +class AnalyzeSyncRequest(BaseModel): + url: str + + +@router.post( + "/analyze/sync", + response_model=PipelineSuccess | PipelineFailure, + summary="전체 파이프라인 — 동기 결과 반환", + description=( + "run_pipeline 을 동기로 실행해 1~4단계 + verdict 를 즉시 반환합니다. " + "단건 디버그/QA 용 — 본 호출은 RDAP·콘텐츠 페치·AI 추론을 모두 포함하므로 " + "지연이 길 수 있습니다. 운영 트래픽은 비동기 /analyze 를 사용하세요." + ), +) +async def analyze_sync( + body: AnalyzeSyncRequest, + session: DBSession, + _: InternalApiKey, +) -> PipelineSuccess | PipelineFailure: + analysis_id = str(uuid.uuid4()) + return await run_pipeline( + analysis_id=analysis_id, + original_url=body.url, + session=session, + ) + + +@router.post( + "/analyze/db-independent/sync", + response_model=DbIndependentPipelineSuccess | DbIndependentPipelineFailure, + summary="DB 비의존 파이프라인 — 동기 결과 반환", + description=( + "GSB, URLhaus 등 외부 threat DB 조회 없이 URL 정규화, 리다이렉트 체인, " + "도메인 휴리스틱, 콘텐츠 정적 분석 결과만으로 verdict/score 를 산출합니다. " + "외부 DB 의존도를 제거한 실험·QA 용 경로입니다." + ), +) +async def analyze_db_independent_sync( + body: AnalyzeSyncRequest, + _: InternalApiKey, +) -> DbIndependentPipelineSuccess | DbIndependentPipelineFailure: + analysis_id = str(uuid.uuid4()) + return await run_db_independent_pipeline( + analysis_id=analysis_id, + original_url=body.url, + ) diff --git a/app/api/v1/endpoints/stages.py b/app/api/v1/endpoints/stages.py new file mode 100644 index 0000000..3d26229 --- /dev/null +++ b/app/api/v1/endpoints/stages.py @@ -0,0 +1,211 @@ +"""파이프라인 단계 단독 실행 엔드포인트. + +운영 라우터에 등록되며 모든 호출은 X-Internal-Api-Key 헤더 인증을 거친다. +각 단계를 단독으로 실행해 응답을 그대로 돌려준다 — Spring 측에서 단계별 결과를 +수집하거나 운영 디버깅 시 분기 동작을 확인할 때 쓴다. + +| Path | Stage | 호출 함수 | +|-----------------------------------|------------|---------------------------------| +| POST /normalize | Stage 1 | normalize_url + unchain_url | +| POST /threat-db | Stage 2 | check_threat_db | +| POST /domain-heuristic | Stage 3 | check_domain_heuristic | +| POST /content-analysis | Stage 4 | analyze_content | +| POST /content/fetch-extract | (보조) | fetch_page + extract_features | + +전체 파이프라인 동기 실행은 `analyze.py` 의 `POST /analyze/sync`. 비동기 접수는 `POST /analyze`. +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException, status +from pydantic import BaseModel, Field + +from app.api.deps import DBSession, InternalApiKey +from app.core.exceptions import NormalizationError +from app.schemas.content_analysis import ( + ContentAnalysisResult, + ExtractedFeaturesView, + FetchExtractResponse, + FetchStatusView, +) +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.normalize import NormalizeResult +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult +from app.services.content_analyzer import analyze_content +from app.services.content_analyzer.extract import extract_features_async +from app.services.content_analyzer.fetch import fetch_page +from app.services.domain_heuristic import check_domain_heuristic +from app.services.normalizer import normalize_url +from app.services.threat_db import check_threat_db +from app.services.unchainer import unchain_url + +router = APIRouter() + +# HTML 미리보기 상한 — 에디터/브라우저 렌더 부담 없이 원본을 훑기 좋은 크기. +# 슬라이싱은 codepoint 단위라 UTF-8 바이트 길이는 더 클 수 있다. +_HTML_PREVIEW_CHARS = 2048 + + +def _normalize_or_400(raw_url: str) -> str: + """단계 단독 호출용 — 스킴/IPv6/포트 검증을 normalize_url 로 위임. + + 파이프라인은 1단계에서 이걸 거치지만 단독 단계 호출은 raw URL 을 직접 받기 때문에 + 여기서 동일한 검증을 통과시켜 file:// 같은 비허용 스킴을 fetch 로 흘리지 않는다. + SSRF 의 사설 IP 차단은 fetch.py 1선에서 한 번 더 막는다. + """ + try: + return normalize_url(raw_url).normalized_url + except NormalizationError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"invalid url: {exc.message}", + ) from exc + + +class StageUrlRequest(BaseModel): + url: str = Field(description="raw URL — 내부에서 normalize_url 을 통과시킨다.") + + +class StageNormalizeResponse(BaseModel): + """1단계 응답 — normalize 결과와 unchain 결과를 함께 반환.""" + + normalize: NormalizeResult + unchain: UnchainResult + + +# --------------------------------------------------------------------------- +# 단계별 API — Stage 1 ~ Stage 4 +# --------------------------------------------------------------------------- + + +@router.post( + "/normalize", + response_model=StageNormalizeResponse, + summary="1단계 — URL 정규화 + 단축 URL 언체이닝", + description=( + "normalize_url 로 canonical form 을 만든 뒤 unchain_url 로 리다이렉트 체인을 끝까지 따라가 " + "최종 URL 을 확정합니다. 2~4단계의 입력이 되는 final_url 을 별도로 확인할 때 사용합니다." + ), +) +async def stage_normalize( + body: StageUrlRequest, _: InternalApiKey +) -> StageNormalizeResponse: + try: + normalize_result = normalize_url(body.url) + except NormalizationError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"invalid url: {exc.message}", + ) from exc + unchain_result = await unchain_url( + normalize_result.normalized_url, + prefer_https_when_schemeless=normalize_result.scheme_was_added, + ) + return StageNormalizeResponse(normalize=normalize_result, unchain=unchain_result) + + +@router.post( + "/threat-db", + response_model=ThreatDbResult, + summary="2단계 — 외부 위협 DB 대조 (GSB + URLhaus)", + description=( + "check_threat_db 를 직접 호출합니다. 입력 URL 은 normalize_url 만 거친 상태로 전달되며 " + "단축 URL 을 풀고 싶다면 먼저 /normalize 로 final_url 을 얻어 그 값을 넣어야 합니다." + ), +) +async def stage_threat_db( + body: StageUrlRequest, session: DBSession, _: InternalApiKey +) -> ThreatDbResult: + target = _normalize_or_400(body.url) + return await check_threat_db(session, target) + + +@router.post( + "/domain-heuristic", + response_model=DomainHeuristicResult, + summary="3단계 — 도메인 휴리스틱 분석", + description=( + "check_domain_heuristic 를 직접 호출합니다. RDAP 캐시(24h) 가 적용되며 캐시 미스 시 " + "최대 약 5초 대기할 수 있습니다. 단축 URL 을 풀고 싶다면 /normalize 를 먼저 호출하세요." + ), +) +async def stage_domain_heuristic( + body: StageUrlRequest, _: InternalApiKey +) -> DomainHeuristicResult: + target = _normalize_or_400(body.url) + return await check_domain_heuristic(target) + + +@router.post( + "/content-analysis", + response_model=ContentAnalysisResult, + summary="4단계 — 페이지 콘텐츠 정적 분석 (fetch + extract + signals + AI)", + description=( + "analyze_content 를 직접 호출해 규칙 점수 · AI 추론이 합쳐진 ContentAnalysisResult 를 " + "그대로 반환합니다. 파이프라인 skip 로직은 거치지 않으므로 선행 단계 점수와 무관하게 항상 " + "실제 분석을 수행합니다. AI 프로바이더는 AI_PROVIDER 설정(기본 auto)을 따릅니다. " + "이 엔드포인트는 unchainer 를 거치지 않으므로 단축 URL 을 직접 넣으면 " + "unexpected_redirect 로 떨어집니다 — 단축 URL 은 /normalize 로 먼저 풀어야 합니다. " + "선행 단계 신호 없이 호출되므로 AI 프롬프트는 페이지 피처만 보고 판정합니다." + ), +) +async def stage_content_analysis( + body: StageUrlRequest, _: InternalApiKey +) -> ContentAnalysisResult: + return await analyze_content(_normalize_or_400(body.url)) + + +# --------------------------------------------------------------------------- +# 보조 — 4단계 내부 fetch+extract 만 분리해서 확인 +# --------------------------------------------------------------------------- + + +@router.post( + "/content/fetch-extract", + response_model=FetchExtractResponse, + summary="(보조) 크롤링 + HTML 파싱 결과 확인", + description=( + "fetch_page() + extract_features() 만 수행해서 원본 HTML preview 와 추출된 " + "피처(title, password 유무, meta refresh, 외부 링크 비율, 이미지 alt)를 반환합니다. " + "규칙 점수 · AI 추론은 돌리지 않습니다. 4단계 디버깅용 보조 엔드포인트." + ), +) +async def stage_content_fetch_extract( + body: StageUrlRequest, _: InternalApiKey +) -> FetchExtractResponse: + target = _normalize_or_400(body.url) + fetch_result = await fetch_page(target) + fetch_view = FetchStatusView( + ok=fetch_result.ok, + status_code=fetch_result.status_code, + html_length=len(fetch_result.html), + error=fetch_result.error, + ) + + if not fetch_result.ok: + return FetchExtractResponse(url=target, fetch=fetch_view) + + features = await extract_features_async(fetch_result.html, base_url=target) + return FetchExtractResponse( + url=target, + fetch=fetch_view, + features=ExtractedFeaturesView( + title=features.title, + has_password_field=features.has_password_field, + has_meta_refresh=features.has_meta_refresh, + has_password_form_external_action=features.has_password_form_external_action, + has_external_meta_refresh=features.has_external_meta_refresh, + external_link_ratio=features.external_link_ratio, + image_alts=features.image_alts, + is_spa_shell=features.is_spa_shell, + body_text_snippets=features.body_text_snippets, + form_field_summaries=features.form_field_summaries, + cta_texts=features.cta_texts, + download_links=features.download_links, + sensitive_field_types=features.sensitive_field_types, + korean_lure_keywords=features.korean_lure_keywords, + public_agency_keywords=features.public_agency_keywords, + ), + html_preview=fetch_result.html[:_HTML_PREVIEW_CHARS], + ) diff --git a/app/api/v1/router.py b/app/api/v1/router.py index ed8ad49..2bb39c4 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -1,6 +1,8 @@ from fastapi import APIRouter -from app.api.v1.endpoints import health +from app.api.v1.endpoints import analyze, health, stages api_router = APIRouter() api_router.include_router(health.router, tags=["health"]) +api_router.include_router(analyze.router, tags=["analyze"]) +api_router.include_router(stages.router, tags=["stages"]) diff --git a/app/core/config.py b/app/core/config.py index ed5ae8c..878eca9 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -16,24 +16,23 @@ class Settings(BaseSettings): case_sensitive=False, ) - # ---- App --------------------------------------------------------------- + # 앱 기본 app_name: str = "LinClean Open API" app_version: str = "0.1.0" environment: Literal["local", "dev", "staging", "production"] = "local" debug: bool = False api_v1_prefix: str = "/api/v1" - # ---- Server ------------------------------------------------------------ + # 서버 host: str = "0.0.0.0" # noqa: S104 port: int = 8000 log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" log_json: bool = False - # ---- CORS -------------------------------------------------------------- + # CORS cors_origins: list[str] = Field(default_factory=lambda: ["http://localhost:3000"]) - # ---- Database (SQLite) ------------------------------------------------- - # Path is resolved relative to the project root unless absolute. + # DB — 절대 경로 아니면 프로젝트 루트 기준으로 해석됨. sqlite_path: str = "data/linclean.db" db_echo: bool = False @@ -51,36 +50,205 @@ def database_url(self) -> str: @computed_field # type: ignore[prop-decorator] @property def alembic_database_url(self) -> str: - # Alembic runs migrations synchronously — use the stdlib sqlite driver. + # Alembic은 동기 실행이라 stdlib sqlite 드라이버 필요. return f"sqlite:///{self.sqlite_file}" - # ---- External: Google Safe Browsing ------------------------------------ + # Google Safe Browsing gsb_api_key: str | None = None - gsb_api_url: str = ( - "https://safebrowsing.googleapis.com/v4/threatMatches:find" - ) + gsb_api_url: str = "https://safebrowsing.googleapis.com/v4/threatMatches:find" gsb_client_id: str = "linclean" gsb_client_version: str = "0.1.0" gsb_timeout_seconds: float = 5.0 - # ---- External: URLhaus ------------------------------------------------- + # URLhaus urlhaus_recent_csv_url: str = "https://urlhaus.abuse.ch/downloads/csv_recent/" urlhaus_online_csv_url: str = "https://urlhaus.abuse.ch/downloads/csv_online/" urlhaus_refresh_interval_seconds: int = 60 * 60 # 60 minutes urlhaus_download_timeout_seconds: float = 60.0 + # URLhaus 매칭 키: 다중 테넌트 호스트는 host+path-prefix(N 세그먼트)까지 키로 사용. + urlhaus_multitenant_hosts: dict[str, int] = Field( + default_factory=lambda: { + "github.com": 2, + "raw.githubusercontent.com": 2, + "gist.github.com": 2, + "gitlab.com": 2, + "bitbucket.org": 2, + "sites.google.com": 2, + "dropbox.com": 2, + "www.dropbox.com": 2, + "dropboxusercontent.com": 2, + "dl.dropboxusercontent.com": 2, + "www.dropboxusercontent.com": 2, + } + ) + + # Scheduler / URLhaus 동기화 + scheduler_enabled: bool = True + urlhaus_sync_on_startup: bool = True - # ---- External: RDAP ---------------------------------------------------- + # RDAP rdap_bootstrap_url: str = "https://rdap.org/domain/" - rdap_timeout_seconds: float = 5.0 - rdap_cache_ttl_seconds: int = 60 * 60 * 24 # 24h + rdap_timeout_seconds: float = 3.0 + rdap_cache_ttl_seconds: int = 60 * 60 * 24 * 7 # 7d + # TTL 동안 누적될 수 있는 도메인 엔트리 상한. 무작위 도메인 트래픽이 들어와도 + # 메모리가 무한 성장하지 않도록 LRU 로 끊는다. 일 100만 URL 기준 도메인 수 5만 이하 가정. + rdap_cache_max_entries: int = 50_000 rdap_new_domain_threshold_days: int = 30 - # ---- Scoring ----------------------------------------------------------- + # 정규화 + normalizer_max_url_length: int = 1024 + + # DNS 해석 캐시 — fetch / unchain 양쪽이 같은 호스트를 반복 해석하는 비용 제거. + # SSRF 방어선이 의존하는 결과라 TTL 은 짧게 (30초) — 공격자가 캐시 hit 동안 IP 를 + # 사설 대역으로 재해석할 가능성을 최소화. 보안 위협보다 성능 보강이 주 목적. + dns_cache_ttl_seconds: int = 30 + dns_cache_max_entries: int = 10_000 + + # 언체이닝 + unchain_max_hops: int = 5 + unchain_timeout_seconds: float = 5.0 + unchain_connect_timeout_seconds: float = 3.0 + unchain_chain_timeout_seconds: float = 6.0 + schemeless_https_probe_timeout_seconds: float = 1.0 + unchain_user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ) + + # 점수 산정 score_weight_gsb: int = 50 score_weight_urlhaus: int = 50 score_weight_rdap: int = 20 score_malicious_threshold: int = 50 + # 도메인 휴리스틱 점수 + score_weight_ip_direct: int = 40 + score_weight_typo_domain: int = 30 + score_weight_punycode_idn: int = 35 + score_weight_no_https: int = 20 + score_weight_new_domain: int = 25 + score_weight_subdomain_overuse: int = 20 + score_weight_open_redirect_param: int = 31 + score_weight_hyphen_overuse: int = 20 + score_weight_suspicious_tld: int = 25 + score_weight_dga_like: int = 31 + score_weight_redirect_cross_origin: int = 15 + score_weight_hosting_platform: int = 20 + score_weight_url_userinfo: int = 45 + score_weight_brand_in_url: int = 30 + score_weight_free_hosting_lure: int = 25 + score_weight_sensitive_path: int = 20 + score_weight_url_shortener: int = 25 + + # 휴리스틱 점수 캡 — GSB/URLhaus(각 50) high-confidence 시그널을 합산이 압도하지 않도록 클램프 + domain_heuristic_score_cap: int = 80 + + # DGA 탐지 임계값 — 0.7이면 netflix/spotify/snapchat 등 긴 영어 브랜드 오탐 + dga_entropy_threshold: float = 3.5 + dga_consonant_ratio_threshold: float = 0.8 + + # 서브도메인 레이블 임계값 + subdomain_label_threshold: int = 4 + + # 하이픈/라벨 길이 임계값 — 20자 미만이면 standardchartered(17), samsungelectronics(18) 등 오탐 + hyphen_count_threshold: int = 3 + domain_label_length_threshold: int = 20 + + # 페이지 콘텐츠 정적 분석 (4단계) + content_fetch_timeout_seconds: float = 3.5 + content_fetch_connect_timeout_seconds: float = 2.0 + content_fetch_max_bytes: int = 2 * 1024 * 1024 # 2MiB 이상이면 끊고 분석 + # DNS rebinding 잔여 위험은 앱 레벨 사전 해석만으로 완전히 닫을 수 없다. 운영에서 분석 전용 + # egress 프록시를 두는 경우 이 값으로 fetch 트래픽을 강제 경유시킨다. + content_fetch_proxy_url: str | None = None + content_user_agents: list[str] = Field( + default_factory=lambda: [ + # 분석 엔진이 동일 UA를 반복 노출하면 쉽게 블랙리스트되므로 풀에서 라운드로빈 + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + "Version/17.0 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/130.0.0.0 Safari/537.36" + ), + ] + ) + content_external_link_ratio_threshold: float = 0.8 + # BS4 파싱은 입력 본문 대비 ~10배 메모리를 점유한다. FastAPI 동시성에 곱셈으로 폭주하는 + # 걸 막기 위해 추출 단계에 글로벌 세마포어를 둔다 — 동시성 N 이 와도 피크는 이 값 * per-page. + content_extract_concurrency: int = 8 + # 정확도 우선 slow path. 정적 HTML 로 판정이 애매하거나 SPA 셸인 경우에만 브라우저 + # 렌더링 DOM 을 추가 분석한다. Playwright 미설치 환경에서는 degraded no-op. + content_precision_enabled: bool = True + content_precision_min_score: int = 20 + content_render_timeout_seconds: float = 4.0 + content_render_settle_ms: int = 500 + content_render_concurrency: int = 2 + + # 콘텐츠 분석 점수 + score_weight_brand_impersonation: int = 50 + score_weight_logo_alt_impersonation: int = 10 + score_weight_credential_form_external: int = 45 + score_weight_pii_collection_form: int = 20 + score_weight_sensitive_id_field: int = 30 + score_weight_financial_field: int = 25 + score_weight_risky_download_link: int = 20 + score_weight_public_agency_lure: int = 20 + score_weight_korean_lure: int = 15 + score_weight_meta_refresh: int = 20 + score_weight_external_meta_refresh: int = 25 + score_weight_external_link_overuse: int = 5 + # 도달 실패(timeout/connect_error/HTTP 5xx 등)에 대한 보수적 가산. + # not_html(이미지·PDF), too_large(대용량 정상 페이지), unexpected_redirect(unchainer 누락)는 + # 정상 컨텐츠 또는 파이프라인 정합성 문제로 보고 점수 가산 없이 시그널만 남긴다. + score_weight_content_fetch_failed: int = 15 + score_weight_ai_phishing: int = 45 + score_weight_ai_suspicious: int = 31 + # 4단계 단독 캡 — 컨텐츠 분석 단계 안에서만 적용된다. 전 단계 합산은 별도로 score_total_cap 에서 + # 다시 100 으로 클램프되므로, 여기를 낮춰도 합산 상한이 자동으로 같이 낮아지는 게 아니다. + content_analysis_score_cap: int = 100 + + # 1~3단계 합산이 이 임계 이상이면 콘텐츠 정적 분석(네트워크·AI 비용)을 건너뛴다. + # README 의 danger(61+) 구간과 정합. + score_danger_threshold: int = 61 + # caution(31~60) 구간 진입 임계. 이 값 미만이면 safe. + score_caution_threshold: int = 31 + # 단계별 점수 합산이 100 을 넘지 않도록 종합 점수에서 캡 적용. + score_total_cap: int = 100 + + # 동기 분석 SLA. 단계별 작업은 이 총 예산 안에서만 실행되어 단일 요청이 20초를 넘기지 + # 않도록 한다. 각 stage budget 은 외부 네트워크 timeout 이 순차 누적되는 것을 막기 위한 상한. + pipeline_total_timeout_seconds: float = 20.0 + pipeline_unchain_timeout_seconds: float = 4.0 + pipeline_reputation_timeout_seconds: float = 5.0 + pipeline_domain_timeout_seconds: float = 4.0 + pipeline_content_timeout_seconds: float = 8.0 + + # 4단계 AI 프로바이더 선택. + # auto : OPENAI_API_KEY 있으면 OpenAIProvider, 없으면 NullAIProvider + # openai : 강제 OpenAIProvider (키 없으면 경고 + NullAIProvider 폴백) + # null : 비활성 (규칙 점수만 사용) + ai_provider: Literal["auto", "openai", "null"] = "auto" + + # OpenAI — 모델 교체는 OPENAI_MODEL 한 줄로 끝난다 (gpt-4o-mini / gpt-4o / gpt-4.1-mini). + openai_api_key: str | None = None + openai_model: str = "gpt-4o-mini" + openai_timeout_seconds: float = 5.0 + openai_max_output_tokens: int = 120 + + # Spring 통신 + internal_api_key: str + spring_internal_url: str = "http://localhost:8080" + @property def is_production(self) -> bool: return self.environment == "production" diff --git a/app/core/dns_cache.py b/app/core/dns_cache.py new file mode 100644 index 0000000..5fc4f36 --- /dev/null +++ b/app/core/dns_cache.py @@ -0,0 +1,51 @@ +"""호스트네임 → getaddrinfo 결과 짧은 TTL 캐시. + +fetch / unchain 양쪽이 같은 호스트를 반복 해석하던 비용을 제거한다. TTL 은 30초로 짧게 +잡아 캐시 hit 동안 공격자가 DNS 를 사설 대역으로 재해석하는 윈도우를 줄였다. +완전한 DNS rebind 방어는 호출 단(fetch / unchain) 의 IP 검증 + 배포 단의 egress 방화벽 +조합이 본선이며, 이 캐시는 어디까지나 성능 보강이다. + +캐시 키는 hostname 만이다 — port/service 는 getaddrinfo 의 IP 결과에 영향을 주지 않으므로 +호출 측에서 별도 처리하면 된다(sockaddr 의 port 필드는 service 인자로만 채워진다). + +실패(getaddrinfo 예외) 는 캐시하지 않는다. DNS 일시 장애가 TTL 동안 모든 후속 요청을 +막아버리는 동작을 피한다. +""" + +from __future__ import annotations + +import asyncio +import socket +from typing import Any + +from cachetools import TTLCache + +from app.core.config import settings + +# (family, type, proto, canonname, sockaddr) 튜플의 순서를 그대로 보존. +# socket.getaddrinfo 의 반환 형식과 동일. +AddrInfoTuple = tuple[int, int, int, str, tuple[Any, ...]] + +_cache: TTLCache[str, tuple[AddrInfoTuple, ...]] = TTLCache( + maxsize=settings.dns_cache_max_entries, + ttl=settings.dns_cache_ttl_seconds, +) + + +async def resolve_host_addrs(hostname: str) -> tuple[AddrInfoTuple, ...]: + """hostname 의 getaddrinfo 결과를 캐시. 실패 시 OSError 그대로 raise.""" + try: + return _cache[hostname] + except KeyError: + pass + + loop = asyncio.get_running_loop() + infos = await loop.getaddrinfo(hostname, None, type=socket.SOCK_STREAM) + result = tuple(infos) + _cache[hostname] = result + return result + + +def clear_cache() -> None: + """테스트에서 호출. 운영에서는 직접 부르지 않는다.""" + _cache.clear() diff --git a/app/core/exceptions.py b/app/core/exceptions.py index b52fea4..7bdf0b9 100644 --- a/app/core/exceptions.py +++ b/app/core/exceptions.py @@ -53,3 +53,9 @@ class ValidationError(AppError): status_code = 422 code = "validation_error" message = "Invalid input." + + +class NormalizationError(AppError): + status_code = 422 + code = "normalization_error" + message = "URL normalization failed." diff --git a/app/core/logging.py b/app/core/logging.py index 7891d59..45918fd 100644 --- a/app/core/logging.py +++ b/app/core/logging.py @@ -2,7 +2,7 @@ import logging import sys -from typing import Any +from typing import Any, cast import structlog from structlog.types import EventDict, Processor @@ -16,11 +16,7 @@ def _drop_color_message_key(_: Any, __: str, event_dict: EventDict) -> EventDict def configure_logging() -> None: - """Configure structlog + stdlib logging. - - All logs (including from uvicorn, sqlalchemy, etc.) are routed through - structlog so we get consistent JSON or pretty output everywhere. - """ + """structlog를 모든 로거의 단일 출력 경로로 설정.""" timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True) shared_processors: list[Processor] = [ @@ -64,7 +60,7 @@ def configure_logging() -> None: root.handlers = [handler] root.setLevel(settings.log_level) - # Quiet noisy libs; route through root handler + # 로그 시끄러운 라이브러리 억제, root 핸들러로 통합 for name in ("uvicorn", "uvicorn.error", "uvicorn.access"): lg = logging.getLogger(name) lg.handlers = [] @@ -76,4 +72,4 @@ def configure_logging() -> None: def get_logger(name: str | None = None) -> structlog.stdlib.BoundLogger: - return structlog.get_logger(name) + return cast(structlog.stdlib.BoundLogger, structlog.get_logger(name)) diff --git a/app/core/scheduler.py b/app/core/scheduler.py new file mode 100644 index 0000000..c94cc0d --- /dev/null +++ b/app/core/scheduler.py @@ -0,0 +1,66 @@ +"""APScheduler 싱글톤 — URLhaus 주기 동기화 스케줄링.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from app.core.config import settings +from app.core.logging import get_logger + +if TYPE_CHECKING: + from apscheduler.schedulers.asyncio import AsyncIOScheduler # type: ignore[import-untyped] + +logger = get_logger(__name__) + +_scheduler: AsyncIOScheduler | None = None + + +def get_scheduler() -> AsyncIOScheduler: + from apscheduler.schedulers.asyncio import AsyncIOScheduler + + global _scheduler + if _scheduler is None: + _scheduler = AsyncIOScheduler(timezone="UTC") + return _scheduler + + +def start_scheduler() -> None: + """URLhaus 동기화 job 을 등록하고 스케줄러 기동. + + `scheduler_enabled=False` 면 no-op (테스트 환경 용). + """ + if not settings.scheduler_enabled: + logger.info("scheduler.disabled") + return + + # import 시점에 순환 참조 방지차 지연 로드. + from apscheduler.triggers.interval import IntervalTrigger # type: ignore[import-untyped] + + from app.services.threat_db.urlhaus_sync import sync_urlhaus + + scheduler = get_scheduler() + if scheduler.running: + return + + scheduler.add_job( + sync_urlhaus, + trigger=IntervalTrigger(seconds=settings.urlhaus_refresh_interval_seconds), + id="urlhaus_sync", + coalesce=True, + max_instances=1, + misfire_grace_time=settings.urlhaus_refresh_interval_seconds, + replace_existing=True, + ) + scheduler.start() + logger.info( + "scheduler.started", + interval_seconds=settings.urlhaus_refresh_interval_seconds, + ) + + +def shutdown_scheduler(wait: bool = False) -> None: + global _scheduler + if _scheduler is None or not _scheduler.running: + return + _scheduler.shutdown(wait=wait) + logger.info("scheduler.stopped") diff --git a/app/core/tld.py b/app/core/tld.py new file mode 100644 index 0000000..c3d7186 --- /dev/null +++ b/app/core/tld.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse + +_MULTI_LABEL_SUFFIXES = frozenset( + { + "ac.kr", + "co.in", + "co.jp", + "co.kr", + "co.uk", + "com.au", + "com.br", + "com.cn", + "com.hk", + "com.mx", + "com.my", + "com.sg", + "com.tr", + "com.tw", + "edu.au", + "go.kr", + "gov.au", + "gov.uk", + "ne.kr", + "net.cn", + "or.kr", + "org.cn", + "re.kr", + # Private hosting suffixes that phishing kits commonly abuse. + "amazonaws.com", + "azurewebsites.net", + "backblazeb2.com", + "duckdns.org", + "firebaseapp.com", + "fly.dev", + "free.nf", + "github.io", + "gitlab.io", + "glitch.me", + "godaddysites.com", + "herokuapp.com", + "my.id", + "netlify.app", + "pages.dev", + "railway.app", + "render.com", + "replit.app", + "surge.sh", + "vercel.app", + "wasmer.app", + "web.app", + "workers.dev", + } +) + + +@dataclass(frozen=True) +class ExtractResult: + subdomain: str + domain: str + suffix: str + + @property + def top_domain_under_public_suffix(self) -> str: + if not self.domain or not self.suffix: + return "" + return f"{self.domain}.{self.suffix}" + + +def _hostname(url: str) -> str: + parsed = urlparse(url if "://" in url else f"https://{url}") + return (parsed.hostname or "").strip(".").lower() + + +def extract_url_parts(url: str) -> ExtractResult: + host = _hostname(url) + if not host: + return ExtractResult(subdomain="", domain="", suffix="") + + labels = [label for label in host.split(".") if label] + if len(labels) == 1: + return ExtractResult(subdomain="", domain=labels[0], suffix="") + + suffix_len = 1 + for size in range(min(3, len(labels) - 1), 1, -1): + candidate = ".".join(labels[-size:]) + if candidate in _MULTI_LABEL_SUFFIXES: + suffix_len = size + break + + suffix = ".".join(labels[-suffix_len:]) + domain_index = len(labels) - suffix_len - 1 + if domain_index < 0: + return ExtractResult(subdomain="", domain="", suffix=suffix) + + domain = labels[domain_index] + subdomain = ".".join(labels[:domain_index]) + return ExtractResult(subdomain=subdomain, domain=domain, suffix=suffix) diff --git a/app/db/base.py b/app/db/base.py index c0a1213..6f93fcd 100644 --- a/app/db/base.py +++ b/app/db/base.py @@ -1,11 +1,11 @@ from datetime import datetime -from typing import Any +from typing import Any, ClassVar from sqlalchemy import MetaData from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.sql import func -# Naming convention for constraints — keeps Alembic autogenerate stable. +# 네이밍 컨벤션 명시 — Alembic autogenerate 결과 일관성 유지용. NAMING_CONVENTION = { "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", @@ -18,7 +18,7 @@ class Base(DeclarativeBase): metadata = MetaData(naming_convention=NAMING_CONVENTION) - type_annotation_map: dict[Any, Any] = {} + type_annotation_map: ClassVar[dict[Any, Any]] = {} class TimestampMixin: diff --git a/app/db/session.py b/app/db/session.py index 7c359b4..0c3c600 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -12,13 +12,8 @@ from app.core.config import settings -# SQLite-specific engine. Notes: -# - check_same_thread=False is required because aiosqlite hands the -# connection across threads internally. -# - We don't tune pool size: SQLite is single-writer, and the default -# pool is fine for an async app of this scale. -# - We ensure the parent directory exists at import time so the very -# first connection doesn't fail in a fresh checkout. +# aiosqlite는 스레드 간 커넥션 전달하므로 check_same_thread=False 필수. +# 데이터 디렉토리 없으면 첫 연결에서 터지니까 미리 생성. settings.sqlite_file.parent.mkdir(parents=True, exist_ok=True) engine: AsyncEngine = create_async_engine( @@ -31,12 +26,7 @@ @event.listens_for(Engine, "connect") def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: - """Apply sane defaults for every SQLite connection. - - - WAL gives concurrent readers while a writer is active. - - foreign_keys=ON is off by default in SQLite (!) and must be re-enabled. - - NORMAL synchronous trades a tiny crash-recovery window for big writes. - """ + """WAL 모드로 동시 읽기 지원, foreign_keys=ON (SQLite 기본값은 OFF).""" cursor = dbapi_connection.cursor() try: cursor.execute("PRAGMA journal_mode=WAL") @@ -55,7 +45,7 @@ def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: async def get_db() -> AsyncIterator[AsyncSession]: - """FastAPI dependency yielding an AsyncSession with rollback-on-error.""" + """AsyncSession 제공. 미처리 예외 시 롤백.""" async with SessionLocal() as session: try: yield session diff --git a/app/main.py b/app/main.py index 2b62a34..41a28d0 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ +import asyncio from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -8,12 +9,31 @@ from app.api.v1.router import api_router as api_v1_router from app.core.config import settings from app.core.logging import configure_logging, get_logger +from app.core.scheduler import shutdown_scheduler, start_scheduler from app.db.session import engine from app.middleware.request_context import RequestContextMiddleware +from app.services.content_analyzer.ai import AIProvider, set_ai_provider +from app.services.content_analyzer.fetch import aclose_client as aclose_fetch_client +from app.services.domain_heuristic.rdap import aclose_client as aclose_rdap_client +from app.services.unchainer.unchain import aclose_client as aclose_unchain_client logger = get_logger(__name__) +def _select_ai_provider() -> AIProvider: + """OpenAIProvider 를 고정 사용한다. SDK import 는 provider 내부에서 infer 시점까지 지연한다.""" + from app.services.content_analyzer.ai_openai import OpenAIProvider + + return OpenAIProvider() + + +async def _aclose_provider(provider: AIProvider) -> None: + # AIProvider Protocol 에는 aclose 가 없다 — 구현체에만 있으면 호출. + aclose = getattr(provider, "aclose", None) + if aclose is not None: + await aclose() + + @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: configure_logging() @@ -22,10 +42,55 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: environment=settings.environment, version=settings.app_version, ) + start_scheduler() + + # AI 프로바이더 부트스트랩 — 키가 없으면 NullAIProvider (4단계 AI 추론 비활성). + # 로컬/CI 에서도 이 분기로 안전하게 동작하므로 별도 조건이 필요 없다. + ai_provider = _select_ai_provider() + set_ai_provider(ai_provider) + logger.info( + "app.ai_provider", + provider=type(ai_provider).__name__, + choice=settings.ai_provider, + ) + startup_task: asyncio.Task[dict[str, int]] | None = None + if settings.scheduler_enabled and settings.urlhaus_sync_on_startup: + # 최초 부트 시 즉시 1회 동기화 (백그라운드, 앱 기동은 블로킹하지 않음). + from app.services.threat_db.urlhaus_sync import sync_urlhaus + + startup_task = asyncio.create_task(sync_urlhaus()) + + def _log_startup_sync_result(t: asyncio.Task[dict[str, int]]) -> None: + # 백그라운드 task 의 예외가 사일런트하게 사라지지 않도록 로깅. + if t.cancelled(): + return + exc = t.exception() + if exc is not None: + logger.warning( + "urlhaus_sync.startup_failed", + error=str(exc), + error_type=type(exc).__name__, + ) + + startup_task.add_done_callback(_log_startup_sync_result) + try: yield finally: logger.info("app.shutdown") + if startup_task is not None and not startup_task.done(): + startup_task.cancel() + # cancel 후 짧게 대기해 "Task was destroyed but it is pending" 경고 방지. + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for( + asyncio.shield(asyncio.gather(startup_task, return_exceptions=True)), + timeout=2.0, + ) + shutdown_scheduler(wait=False) + await aclose_rdap_client() + await aclose_fetch_client() + await aclose_unchain_client() + await _aclose_provider(ai_provider) await engine.dispose() diff --git a/app/middleware/request_context.py b/app/middleware/request_context.py index 40b6e96..0e89afd 100644 --- a/app/middleware/request_context.py +++ b/app/middleware/request_context.py @@ -1,47 +1,60 @@ +from __future__ import annotations + import time import uuid import structlog -from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from starlette.requests import Request -from starlette.responses import Response +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send logger = structlog.get_logger(__name__) REQUEST_ID_HEADER = "X-Request-ID" -class RequestContextMiddleware(BaseHTTPMiddleware): - """Bind a request_id + access log entry to every request. +class RequestContextMiddleware: + """요청마다 X-Request-ID를 생성·전파하고 structlog에 바인딩한다. - - Reuses an inbound `X-Request-ID` if present, else generates one. - - Binds it into structlog contextvars so all logs in the request carry it. - - Emits a single structured access log line per request. + BaseHTTPMiddleware 대신 순수 ASGI로 구현해 응답 버퍼링 오버헤드를 제거한다. """ - async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: - request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + headers = dict(scope.get("headers", [])) + raw_id = headers.get(REQUEST_ID_HEADER.lower().encode()) + request_id = raw_id.decode() if raw_id else str(uuid.uuid4()) + scope["request_id"] = request_id structlog.contextvars.clear_contextvars() structlog.contextvars.bind_contextvars( request_id=request_id, - method=request.method, - path=request.url.path, + method=scope.get("method", ""), + path=scope.get("path", ""), ) start = time.perf_counter() + status_code = 0 + + async def send_wrapper(message: Message) -> None: + nonlocal status_code + if message["type"] == "http.response.start": + status_code = message["status"] + mutable = MutableHeaders(scope=message) + mutable.append(REQUEST_ID_HEADER, request_id) + await send(message) + try: - response = await call_next(request) + await self.app(scope, receive, send_wrapper) except Exception: duration_ms = (time.perf_counter() - start) * 1000 logger.exception("request.failed", duration_ms=round(duration_ms, 2)) raise duration_ms = (time.perf_counter() - start) * 1000 - response.headers[REQUEST_ID_HEADER] = request_id - logger.info( - "request.completed", - status_code=response.status_code, - duration_ms=round(duration_ms, 2), - ) - return response + logger.info("request.completed", status_code=status_code, duration_ms=round(duration_ms, 2)) diff --git a/app/models/__init__.py b/app/models/__init__.py index 7015561..c0b7a2e 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,5 +4,6 @@ """ from app.db.base import Base +from app.models.urlhaus_entry import URLhausEntry -__all__ = ["Base"] +__all__ = ["Base", "URLhausEntry"] diff --git a/app/models/urlhaus_entry.py b/app/models/urlhaus_entry.py new file mode 100644 index 0000000..c9efb58 --- /dev/null +++ b/app/models/urlhaus_entry.py @@ -0,0 +1,32 @@ +"""URLhaus 로컬 스냅샷 엔트리.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class URLhausEntry(Base): + """URLhaus CSV 한 행을 upsert 한 레코드. + + `match_key` 는 host 또는 host+path-prefix 로, 조회 시 빠른 exact match 용도. + """ + + __tablename__ = "urlhaus_entries" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False) + url: Mapped[str] = mapped_column(String, nullable=False, unique=True, index=True) + host: Mapped[str] = mapped_column(String, nullable=False) + match_key: Mapped[str] = mapped_column(String, nullable=False, index=True) + threat: Mapped[str | None] = mapped_column(String, nullable=True) + tags: Mapped[str | None] = mapped_column(String, nullable=True) + url_status: Mapped[str | None] = mapped_column(String, nullable=True) + date_added: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + last_online: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + urlhaus_link: Mapped[str | None] = mapped_column(String, nullable=True) + reporter: Mapped[str | None] = mapped_column(String, nullable=True) + synced_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py index e69de29..3f490a0 100644 --- a/app/schemas/__init__.py +++ b/app/schemas/__init__.py @@ -0,0 +1,13 @@ +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineFailure, + DbIndependentPipelineResult, + DbIndependentPipelineStages, + DbIndependentPipelineSuccess, +) + +__all__ = [ + "DbIndependentPipelineFailure", + "DbIndependentPipelineResult", + "DbIndependentPipelineStages", + "DbIndependentPipelineSuccess", +] diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py new file mode 100644 index 0000000..abf8554 --- /dev/null +++ b/app/schemas/analysis.py @@ -0,0 +1,61 @@ +# 하위 호환 re-export. 신규 코드는 각 모듈에서 직접 import할 것. +from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.schemas.content_analysis import ( + AIVerdict, + ContentAnalysisResult, + ContentSignal, + ExtractedFeaturesView, + FetchExtractResponse, + FetchStatusView, +) +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineFailure, + DbIndependentPipelineResult, + DbIndependentPipelineStages, + DbIndependentPipelineSuccess, +) +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import ( + PipelineFailure, + PipelineResult, + PipelineStage, + PipelineStages, + PipelineStageTimings, + PipelineSuccess, + PipelineTimings, +) +from app.schemas.threat_db import GSBMatch, GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import HopRecord, UnchainResult + +__all__ = [ + "AIVerdict", + "AnalyzeAccepted", + "AnalyzeRequest", + "ContentAnalysisResult", + "ContentSignal", + "DbIndependentPipelineFailure", + "DbIndependentPipelineResult", + "DbIndependentPipelineStages", + "DbIndependentPipelineSuccess", + "DomainHeuristicResult", + "DomainHeuristicSignal", + "ExtractedFeaturesView", + "FetchExtractResponse", + "FetchStatusView", + "GSBMatch", + "GSBResult", + "HopRecord", + "NormalizeResult", + "PipelineFailure", + "PipelineResult", + "PipelineStage", + "PipelineStageTimings", + "PipelineStages", + "PipelineSuccess", + "PipelineTimings", + "RdapInfo", + "ThreatDbResult", + "URLhausResult", + "UnchainResult", +] diff --git a/app/schemas/analyze.py b/app/schemas/analyze.py new file mode 100644 index 0000000..81dd1a2 --- /dev/null +++ b/app/schemas/analyze.py @@ -0,0 +1,17 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class AnalyzeRequest(BaseModel): + model_config = {"populate_by_name": True} + + analysis_id: str = Field(alias="analysisId") + url: str + + +class AnalyzeAccepted(BaseModel): + model_config = {"populate_by_name": True} + + analysis_id: str = Field(alias="analysisId") + status: Literal["queued"] = "queued" diff --git a/app/schemas/content_analysis.py b/app/schemas/content_analysis.py new file mode 100644 index 0000000..a633b82 --- /dev/null +++ b/app/schemas/content_analysis.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class ContentSignal(StrEnum): + BRAND_IMPERSONATION_FORM = "BRAND_IMPERSONATION_FORM" + LOGO_ALT_IMPERSONATION = "LOGO_ALT_IMPERSONATION" + CREDENTIAL_FORM_EXTERNAL = "CREDENTIAL_FORM_EXTERNAL" + PII_COLLECTION_FORM = "PII_COLLECTION_FORM" + SENSITIVE_ID_FIELD = "SENSITIVE_ID_FIELD" + FINANCIAL_FIELD = "FINANCIAL_FIELD" + RISKY_DOWNLOAD_LINK = "RISKY_DOWNLOAD_LINK" + PUBLIC_AGENCY_LURE = "PUBLIC_AGENCY_LURE" + KOREAN_LURE_TEXT = "KOREAN_LURE_TEXT" + RENDERED_DOM_ANALYZED = "RENDERED_DOM_ANALYZED" + META_REFRESH = "META_REFRESH" + EXTERNAL_META_REFRESH = "EXTERNAL_META_REFRESH" + EXTERNAL_LINK_OVERUSE = "EXTERNAL_LINK_OVERUSE" + # 초기 HTML 이 JS 마운트 셸만 담고 있어 정적 추출로 input/form 판정이 불가한 상태. + # 점수 가산 없이 판정 불가 플래그로만 사용 — AI 프롬프트에도 힌트로 전달된다. + SPA_SHELL = "SPA_SHELL" + FETCH_FAILED = "FETCH_FAILED" + SKIPPED_ALREADY_DANGER = "SKIPPED_ALREADY_DANGER" + + +class AIVerdict(StrEnum): + PHISHING = "phishing" + SUSPICIOUS = "suspicious" + BENIGN = "benign" + + +class TokenUsage(BaseModel): + """AI 호출 1건의 토큰 사용량. 모델 비용 · 프롬프트 튜닝 관측에 쓴다.""" + + prompt_tokens: int = Field(ge=0) + completion_tokens: int = Field(ge=0) + total_tokens: int = Field(ge=0) + + +class ContentAnalysisResult(BaseModel): + final_url: str + fetched: bool + status_code: int | None = Field( + default=None, + description="HTTP status code observed when fetching final_url for content analysis", + ) + score: int = 0 + signals: list[ContentSignal] = Field( + default_factory=list, + description=( + "분석 결과 신호 코드. fetch 실패는 FETCH_FAILED, 선행 단계 danger 로 4단계가 " + "건너뛰어진 경우는 SKIPPED_ALREADY_DANGER 가 들어간다. 분석 결과 차원의 진실원." + ), + ) + + title: str | None = None + has_password_field: bool = False + has_password_form_external_action: bool = False + has_meta_refresh: bool = False + has_external_meta_refresh: bool = False + external_link_ratio: float | None = None + brand_impersonation: bool = False + logo_alt_impersonation: bool = False + # JS 마운트 셸로 판단돼 정적 추출이 결정적이지 않은 상태. 여기서 has_password_field=False 는 + # "폼 없음" 이 아니라 "판정 불가" 로 읽어야 한다. + is_spa_shell: bool = False + body_text_snippets: list[str] = Field(default_factory=list) + form_field_summaries: list[str] = Field(default_factory=list) + cta_texts: list[str] = Field(default_factory=list) + download_links: list[str] = Field(default_factory=list) + sensitive_field_types: list[str] = Field(default_factory=list) + korean_lure_keywords: list[str] = Field(default_factory=list) + public_agency_keywords: list[str] = Field(default_factory=list) + precision_analysis_used: bool = False + precision_analysis_error: str | None = None + + ai_verdict: AIVerdict | None = None + ai_reason: str | None = None + reason: str | None = Field( + default=None, + description=( + "AI 여부와 무관한 사용자 표시용 설명. 예: http_error_404 는 " + "'페이지를 찾을 수 없습니다.' 로 내려간다." + ), + ) + ai_error: str | None = Field( + default=None, + description=( + "AI 호출 실패/비활성 사유. " + "ai_unavailable=API 호출 실패, provider_misconfigured=설정 강제 openai 인데 키 누락, " + "None=정상 호출 또는 NullProvider 정상 동작." + ), + ) + # 실제로 응답한 모델 id — 설정 변경/폴백 상황에서도 추적 가능하도록 응답에 싣는다. + ai_model: str | None = None + ai_token_usage: TokenUsage | None = None + + error: str | None = Field( + default=None, + description=( + "페치/스킵 사유 코드 — fetch 레이어의 진실원. " + "timeout/connect_error/http_error_*/not_html/too_large/unexpected_redirect/" + "blocked_host/unexpected/skipped_already_danger 중 하나. " + "signals 와 동시에 채워질 수 있으나 의미는 다르다 (signals=결과 차원, error=원인 코드)." + ), + ) + + +class FetchStatusView(BaseModel): + ok: bool + status_code: int | None = None + html_length: int = 0 + error: str | None = None + + +class ExtractedFeaturesView(BaseModel): + title: str | None = None + has_password_field: bool = False + has_password_form_external_action: bool = False + has_meta_refresh: bool = False + has_external_meta_refresh: bool = False + external_link_ratio: float | None = None + image_alts: list[str] = Field(default_factory=list) + is_spa_shell: bool = False + body_text_snippets: list[str] = Field(default_factory=list) + form_field_summaries: list[str] = Field(default_factory=list) + cta_texts: list[str] = Field(default_factory=list) + download_links: list[str] = Field(default_factory=list) + sensitive_field_types: list[str] = Field(default_factory=list) + korean_lure_keywords: list[str] = Field(default_factory=list) + public_agency_keywords: list[str] = Field(default_factory=list) + + +class FetchExtractResponse(BaseModel): + """fetch 단계 상태 + 파싱 결과 + HTML preview.""" + + url: str + fetch: FetchStatusView + features: ExtractedFeaturesView | None = None + html_preview: str | None = Field( + default=None, + description="HTML 본문 앞부분(최대 2KB) — 파싱 원본 확인용", + ) diff --git a/app/schemas/db_independent_pipeline.py b/app/schemas/db_independent_pipeline.py new file mode 100644 index 0000000..62da8f0 --- /dev/null +++ b/app/schemas/db_independent_pipeline.py @@ -0,0 +1,46 @@ +from typing import Literal + +from pydantic import BaseModel, Field + +from app.schemas.content_analysis import ContentAnalysisResult +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import PipelineStage, PipelineTimings, Verdict +from app.schemas.unchain import UnchainResult + + +class DbIndependentPipelineStages(BaseModel): + """외부 threat DB를 제외한 독립 파이프라인 단계 결과.""" + + normalize: NormalizeResult + unchain: UnchainResult + domain_heuristic: DomainHeuristicResult + content_analysis: ContentAnalysisResult + + +class DbIndependentPipelineSuccess(BaseModel): + """DB 비의존 파이프라인 성공 응답.""" + + status: Literal["success"] = "success" + analysis_id: str + original_url: str + final_url: str + verdict: Verdict + score: int = Field(ge=0, le=100) + timings: PipelineTimings | None = None + stages: DbIndependentPipelineStages + + +class DbIndependentPipelineFailure(BaseModel): + status: Literal["failed"] = "failed" + analysis_id: str + original_url: str + final_url: str | None = None + failed_at_stage: PipelineStage + error: str + error_code: str | None = None + status_code: int | None = None + timings: PipelineTimings | None = None + + +DbIndependentPipelineResult = DbIndependentPipelineSuccess | DbIndependentPipelineFailure diff --git a/app/schemas/domain_heuristic.py b/app/schemas/domain_heuristic.py new file mode 100644 index 0000000..54b96c8 --- /dev/null +++ b/app/schemas/domain_heuristic.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from pydantic import BaseModel + + +class RdapInfo(BaseModel): + domain: str + registrar: str | None + created_date: datetime | None + expiry_date: datetime | None + domain_age_days: int | None + is_new_domain: bool + + +class DomainHeuristicSignal(StrEnum): + IP_DIRECT = "IP_DIRECT" + TYPO_DOMAIN = "TYPO_DOMAIN" + PUNYCODE_IDN = "PUNYCODE_IDN" + NEW_DOMAIN = "NEW_DOMAIN" + SUBDOMAIN_OVERUSE = "SUBDOMAIN_OVERUSE" + NO_HTTPS = "NO_HTTPS" + OPEN_REDIRECT_PARAM = "OPEN_REDIRECT_PARAM" + HYPHEN_OVERUSE = "HYPHEN_OVERUSE" + SUSPICIOUS_TLD = "SUSPICIOUS_TLD" + DGA_LIKE = "DGA_LIKE" + HOSTING_PLATFORM = "HOSTING_PLATFORM" + URL_USERINFO = "URL_USERINFO" + BRAND_IN_URL = "BRAND_IN_URL" + FREE_HOSTING_LURE = "FREE_HOSTING_LURE" + SENSITIVE_PATH = "SENSITIVE_PATH" + URL_SHORTENER = "URL_SHORTENER" + REDIRECT_CROSS_ORIGIN = "REDIRECT_CROSS_ORIGIN" + + +class DomainHeuristicSkippedReason(StrEnum): + THREAT_MATCHED = "threat_matched" + + +class DomainHeuristicResult(BaseModel): + domain: str + score: int + signals: list[DomainHeuristicSignal] + rdap: RdapInfo | None = None + rdap_error: str | None = None + skipped_reason: DomainHeuristicSkippedReason | None = None diff --git a/app/schemas/normalize.py b/app/schemas/normalize.py new file mode 100644 index 0000000..9f93b97 --- /dev/null +++ b/app/schemas/normalize.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, Field + + +class NormalizeResult(BaseModel): + original_url: str = Field(description="Original URL after trimming") + normalized_url: str = Field(description="Canonicalized URL") + scheme_was_added: bool = Field( + default=False, + description="True when the request URL had no explicit scheme and normalizer added one", + ) diff --git a/app/schemas/pipeline.py b/app/schemas/pipeline.py new file mode 100644 index 0000000..0d2ff7b --- /dev/null +++ b/app/schemas/pipeline.py @@ -0,0 +1,87 @@ +from enum import StrEnum +from typing import Literal + +from pydantic import BaseModel, Field + +from app.schemas.content_analysis import ContentAnalysisResult +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.normalize import NormalizeResult +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult + + +class PipelineStage(StrEnum): + NORMALIZE = "normalize" + UNCHAIN = "unchain" + THREAT_DB = "threat_db" + DOMAIN_HEURISTIC = "domain_heuristic" + CONTENT_ANALYSIS = "content_analysis" + + +class Verdict(StrEnum): + """종합 판정 — UI 의 색상/카피 분기 기준. + + score 만으로 매핑하되, 외부 위협 DB 매치(`threat_db.is_malicious=True`) 면 + 점수와 무관하게 강제로 DANGER 가 된다. blacklist 매치 = 알려진 악성 URL 이라 + 점수 합산이 임계 미달이어도 verdict 자체는 danger 가 맞다. + """ + + SAFE = "safe" + CAUTION = "caution" + DANGER = "danger" + + +class PipelineStages(BaseModel): + """성공한 파이프라인의 각 단계 원시 결과. 모든 필드가 반드시 채워진다.""" + + normalize: NormalizeResult + unchain: UnchainResult + threat_db: ThreatDbResult + domain_heuristic: DomainHeuristicResult + content_analysis: ContentAnalysisResult + + +class PipelineStageTimings(BaseModel): + """단계별 소요 시간(초). 실행되지 않은 단계는 None 으로 남긴다.""" + + normalize: float | None = Field(default=None, ge=0) + unchain: float | None = Field(default=None, ge=0) + threat_db: float | None = Field(default=None, ge=0) + domain_heuristic: float | None = Field(default=None, ge=0) + content_analysis: float | None = Field(default=None, ge=0) + + +class PipelineTimings(BaseModel): + """파이프라인 전체 wall-clock 시간과 단계별 소요 시간(초).""" + + total_seconds: float = Field(ge=0) + stages: PipelineStageTimings + + +class PipelineSuccess(BaseModel): + """파이프라인 성공 응답. verdict/score 는 stages 합산의 결론이라 응답 상단에 노출 — + 클라이언트가 stages 트리를 파싱하지 않고도 곧장 사용자에게 보여줄 수 있게 한다.""" + + status: Literal["success"] = "success" + analysis_id: str + original_url: str + final_url: str + verdict: Verdict + score: int = Field(ge=0, le=100) + timings: PipelineTimings | None = None + stages: PipelineStages + + +class PipelineFailure(BaseModel): + status: Literal["failed"] = "failed" + analysis_id: str + original_url: str + final_url: str | None = None + failed_at_stage: PipelineStage + error: str + error_code: str | None = None + status_code: int | None = None + timings: PipelineTimings | None = None + + +PipelineResult = PipelineSuccess | PipelineFailure diff --git a/app/schemas/threat_db.py b/app/schemas/threat_db.py new file mode 100644 index 0000000..ee9a960 --- /dev/null +++ b/app/schemas/threat_db.py @@ -0,0 +1,36 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class GSBMatch(BaseModel): + threat_type: str = Field(description="MALWARE / SOCIAL_ENGINEERING 등") + platform_type: str | None = None + cache_duration: str | None = None + + +class GSBResult(BaseModel): + checked: bool = Field(description="API 호출 성공 여부") + is_threat: bool = False + matches: list[GSBMatch] = Field(default_factory=list) + error: str | None = None + + +class URLhausResult(BaseModel): + checked: bool + is_threat: bool = False + match_type: Literal["url", "host", "host_path"] | None = None + matched_key: str | None = None + threat: str | None = None + tags: list[str] = Field(default_factory=list) + urlhaus_link: str | None = None + error: str | None = None + + +class ThreatDbResult(BaseModel): + final_url: str + is_malicious: bool + sources_checked: int = Field(description="성공적으로 조회된 외부 소스 수 (0~2)") + gsb: GSBResult + urlhaus: URLhausResult + threat_types: list[str] = Field(default_factory=list) diff --git a/app/schemas/unchain.py b/app/schemas/unchain.py new file mode 100644 index 0000000..75f0e14 --- /dev/null +++ b/app/schemas/unchain.py @@ -0,0 +1,19 @@ +from pydantic import BaseModel, Field + + +class HopRecord(BaseModel): + url: str = Field(description="이 hop에서 요청한 URL") + status_code: int = Field(description="HTTP 응답 상태 코드") + raw_location: str | None = Field(default=None, description="Location 헤더 원본 값") + location: str | None = Field(default=None, description="절대경로로 해석된 Location 값") + method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") + + +class UnchainResult(BaseModel): + input_url: str = Field(description="언체이닝 시작 URL") + final_url: str = Field(description="최종 도달 URL") + hops: list[HopRecord] = Field(default_factory=list) + hop_count: int = Field(default=0) + timed_out: bool = Field(default=False) + error: str | None = Field(default=None) + signals: list[str] = Field(default_factory=list) diff --git a/app/services/analysis_callback.py b/app/services/analysis_callback.py new file mode 100644 index 0000000..41e7956 --- /dev/null +++ b/app/services/analysis_callback.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlparse + +import httpx + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.pipeline import PipelineFailure, PipelineStage, PipelineSuccess, Verdict + +logger = get_logger(__name__) + +_CALLBACK_PATH = "/internal/analysis-result" +_MAX_ATTEMPTS = 3 +_KNOWN_MALICIOUS_SUMMARY = "악성으로 알려진 페이지 입니다." + +_STAGE_NUMBERS: dict[PipelineStage, int] = { + PipelineStage.NORMALIZE: 1, + PipelineStage.UNCHAIN: 1, + PipelineStage.THREAT_DB: 2, + PipelineStage.DOMAIN_HEURISTIC: 3, + PipelineStage.CONTENT_ANALYSIS: 4, +} + +_DOMAIN_REASON_MESSAGES: dict[str, str] = { + "IP_DIRECT": "IP 주소로 직접 접근하는 URL입니다.", + "TYPO_DOMAIN": "유명 서비스와 유사한 도메인입니다.", + "PUNYCODE_IDN": "문자 혼동을 유도할 수 있는 국제화 도메인입니다.", + "NEW_DOMAIN": "최근 등록된 도메인입니다.", + "SUBDOMAIN_OVERUSE": "하위 도메인을 과도하게 사용합니다.", + "NO_HTTPS": "HTTPS가 아닌 연결을 사용합니다.", + "OPEN_REDIRECT_PARAM": "오픈 리다이렉트에 악용될 수 있는 파라미터가 있습니다.", + "HYPHEN_OVERUSE": "하이픈을 과도하게 사용한 도메인입니다.", + "SUSPICIOUS_TLD": "피싱에 자주 악용되는 최상위 도메인입니다.", + "DGA_LIKE": "자동 생성된 것으로 의심되는 도메인입니다.", + "HOSTING_PLATFORM": "무료 호스팅 플랫폼을 사용합니다.", + "URL_USERINFO": "URL에 사용자 정보 형식이 포함되어 있습니다.", + "BRAND_IN_URL": "URL에 브랜드명을 포함해 신뢰를 유도합니다.", + "FREE_HOSTING_LURE": "무료 호스팅 주소에서 신뢰를 유도하는 문구를 사용합니다.", + "SENSITIVE_PATH": "로그인 또는 인증 관련 경로를 사용합니다.", + "URL_SHORTENER": "단축 URL 서비스를 사용합니다.", + "REDIRECT_CROSS_ORIGIN": "입력 URL이 다른 사이트로 이동합니다.", +} + +_CONTENT_REASON_MESSAGES: dict[str, str] = { + "BRAND_IMPERSONATION_FORM": "브랜드를 사칭하는 로그인 폼이 확인되었습니다.", + "LOGO_ALT_IMPERSONATION": "브랜드 로고 또는 이미지 설명을 사칭합니다.", + "CREDENTIAL_FORM_EXTERNAL": "로그인 정보가 외부 주소로 전송될 수 있습니다.", + "PII_COLLECTION_FORM": "개인정보 입력을 유도하는 폼이 있습니다.", + "SENSITIVE_ID_FIELD": "민감 식별정보 입력을 요구합니다.", + "FINANCIAL_FIELD": "금융정보 입력을 요구합니다.", + "RISKY_DOWNLOAD_LINK": "위험한 파일 다운로드를 유도합니다.", + "PUBLIC_AGENCY_LURE": "공공기관을 사칭하는 문구가 있습니다.", + "KOREAN_LURE_TEXT": "국내 사용자를 노린 유인 문구가 있습니다.", + "META_REFRESH": "자동 이동 태그가 포함되어 있습니다.", + "EXTERNAL_META_REFRESH": "외부 사이트로 자동 이동할 수 있습니다.", + "EXTERNAL_LINK_OVERUSE": "외부 링크 비율이 높습니다.", + "SPA_SHELL": "정적 분석만으로는 페이지 내용을 확인하기 어렵습니다.", + "FETCH_FAILED": "페이지 내용을 가져오지 못했습니다.", + "SKIPPED_ALREADY_DANGER": "선행 단계에서 위험성이 확인되어 콘텐츠 분석을 생략했습니다.", +} + + +def _iso_z(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def _callback_url() -> str: + return f"{settings.spring_internal_url.rstrip('/')}{_CALLBACK_PATH}" + + +def _enum_value(value: Any) -> str: + return value.value if hasattr(value, "value") else str(value) + + +def _host(url: str) -> str | None: + return urlparse(url).hostname + + +def _signal_weight(code: str) -> int: + domain_weights = { + "IP_DIRECT": settings.score_weight_ip_direct, + "TYPO_DOMAIN": settings.score_weight_typo_domain, + "PUNYCODE_IDN": settings.score_weight_punycode_idn, + "NEW_DOMAIN": settings.score_weight_new_domain, + "SUBDOMAIN_OVERUSE": settings.score_weight_subdomain_overuse, + "NO_HTTPS": settings.score_weight_no_https, + "OPEN_REDIRECT_PARAM": settings.score_weight_open_redirect_param, + "HYPHEN_OVERUSE": settings.score_weight_hyphen_overuse, + "SUSPICIOUS_TLD": settings.score_weight_suspicious_tld, + "DGA_LIKE": settings.score_weight_dga_like, + "HOSTING_PLATFORM": settings.score_weight_hosting_platform, + "URL_USERINFO": settings.score_weight_url_userinfo, + "BRAND_IN_URL": settings.score_weight_brand_in_url, + "FREE_HOSTING_LURE": settings.score_weight_free_hosting_lure, + "SENSITIVE_PATH": settings.score_weight_sensitive_path, + "URL_SHORTENER": settings.score_weight_url_shortener, + "REDIRECT_CROSS_ORIGIN": settings.score_weight_redirect_cross_origin, + } + content_weights = { + "BRAND_IMPERSONATION_FORM": settings.score_weight_brand_impersonation, + "LOGO_ALT_IMPERSONATION": settings.score_weight_logo_alt_impersonation, + "CREDENTIAL_FORM_EXTERNAL": settings.score_weight_credential_form_external, + "PII_COLLECTION_FORM": settings.score_weight_pii_collection_form, + "SENSITIVE_ID_FIELD": settings.score_weight_sensitive_id_field, + "FINANCIAL_FIELD": settings.score_weight_financial_field, + "RISKY_DOWNLOAD_LINK": settings.score_weight_risky_download_link, + "PUBLIC_AGENCY_LURE": settings.score_weight_public_agency_lure, + "KOREAN_LURE_TEXT": settings.score_weight_korean_lure, + "META_REFRESH": settings.score_weight_meta_refresh, + "EXTERNAL_META_REFRESH": settings.score_weight_external_meta_refresh, + "EXTERNAL_LINK_OVERUSE": settings.score_weight_external_link_overuse, + "FETCH_FAILED": settings.score_weight_content_fetch_failed, + } + return domain_weights.get(code) or content_weights.get(code) or 0 + + +def _add_reason( + reasons: list[dict[str, Any]], + *, + code: str, + stage: int, + weight: int, + message: str, +) -> None: + if weight <= 0 and code not in {"GSB_MATCH", "URLHAUS_MATCH"}: + return + reasons.append( + { + "code": code, + "stage": stage, + "weight": weight, + "message": message, + } + ) + + +def _callback_reasons(result: PipelineSuccess) -> list[dict[str, Any]]: + stages = result.stages + reasons: list[dict[str, Any]] = [] + + if stages.threat_db.gsb.is_threat: + _add_reason( + reasons, + code="GSB_MATCH", + stage=2, + weight=settings.score_weight_gsb, + message="Google Safe Browsing에 악성 URL로 등록되어 있습니다.", + ) + if stages.threat_db.urlhaus.is_threat: + _add_reason( + reasons, + code="URLHAUS_MATCH", + stage=2, + weight=settings.score_weight_urlhaus, + message="URLhaus에 악성 URL로 등록되어 있습니다.", + ) + + for signal in stages.domain_heuristic.signals: + code = _enum_value(signal) + _add_reason( + reasons, + code=code, + stage=3, + weight=_signal_weight(code), + message=_DOMAIN_REASON_MESSAGES.get(code, "도메인 휴리스틱 위험 신호가 있습니다."), + ) + + for signal in stages.content_analysis.signals: + code = _enum_value(signal) + _add_reason( + reasons, + code=code, + stage=4, + weight=_signal_weight(code), + message=_CONTENT_REASON_MESSAGES.get(code, "콘텐츠 분석 위험 신호가 있습니다."), + ) + + ai_verdict = stages.content_analysis.ai_verdict + if ai_verdict is not None: + verdict = _enum_value(ai_verdict) + if verdict == "phishing": + _add_reason( + reasons, + code="AI_PHISHING", + stage=4, + weight=settings.score_weight_ai_phishing, + message=( + stages.content_analysis.ai_reason + or "AI가 피싱 가능성이 높다고 판단했습니다." + ), + ) + elif verdict == "suspicious": + _add_reason( + reasons, + code="AI_SUSPICIOUS", + stage=4, + weight=settings.score_weight_ai_suspicious, + message=( + stages.content_analysis.ai_reason + or "AI가 의심스러운 페이지로 판단했습니다." + ), + ) + + return reasons + + +def _callback_summary(result: PipelineSuccess) -> str: + if result.stages.threat_db.is_malicious: + return _KNOWN_MALICIOUS_SUMMARY + content = result.stages.content_analysis + if content.ai_reason: + return content.ai_reason + if content.reason: + return content.reason + if result.verdict == Verdict.DANGER: + return "위험성이 높은 페이지입니다. 접속과 정보 입력을 피하세요." + if result.verdict == Verdict.CAUTION: + return "주의가 필요한 페이지입니다. 링크와 입력 정보를 한 번 더 확인하세요." + return "현재 분석 기준에서 뚜렷한 위험 신호가 확인되지 않았습니다." + + +def _spring_stages(result: PipelineSuccess) -> dict[str, Any]: + stages = result.stages + gsb_matches = [ + match.threat_type + for match in stages.threat_db.gsb.matches + if match.threat_type + ] + + chain = [hop.url for hop in stages.unchain.hops] + if not chain: + chain.append(stages.unchain.input_url) + if stages.unchain.final_url not in chain: + chain.append(stages.unchain.final_url) + + rdap = stages.domain_heuristic.rdap + rdap_payload = None + if rdap is not None: + rdap_payload = { + "domain": rdap.domain, + "registrar": rdap.registrar, + "createdDate": _iso_z(rdap.created_date) if rdap.created_date else None, + "domainAgeDays": rdap.domain_age_days, + "isNewDomain": rdap.is_new_domain, + } + + content = stages.content_analysis + return { + "externalDb": { + "gsb": { + "isThreat": stages.threat_db.gsb.is_threat, + "matchedTypes": gsb_matches, + }, + "urlhaus": { + "isThreat": stages.threat_db.urlhaus.is_threat, + "host": _host(stages.threat_db.final_url), + }, + }, + "unchain": { + "hops": stages.unchain.hop_count, + "chain": chain, + }, + "domainHeuristic": { + "rdap": rdap_payload, + "signals": [_enum_value(signal) for signal in stages.domain_heuristic.signals], + }, + "contentAnalysis": { + "fetched": content.fetched, + "hasPasswordField": content.has_password_field, + "aiVerdict": _enum_value(content.ai_verdict) if content.ai_verdict else None, + "aiReason": content.ai_reason, + }, + } + + +def _error_stage(stage: PipelineStage) -> int: + return _STAGE_NUMBERS[stage] + + +def _success_payload( + result: PipelineSuccess, + *, + request_id: str, + elapsed_ms: int, + analyzed_at: datetime, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "analysisId": result.analysis_id, + "requestId": request_id, + "status": "succeeded", + "originalUrl": result.original_url, + "finalUrl": result.final_url, + "verdict": result.verdict.value, + "score": result.score, + "reasons": _callback_reasons(result), + "stages": _spring_stages(result), + "summary": _callback_summary(result), + "engineVersion": settings.app_version, + "analyzedAt": _iso_z(analyzed_at), + "elapsedMs": elapsed_ms, + } + return payload + + +def _failure_payload( + result: PipelineFailure, + *, + request_id: str, + elapsed_ms: int, + analyzed_at: datetime, +) -> dict[str, Any]: + error: dict[str, Any] = { + "code": result.error_code or f"{result.failed_at_stage.value.upper()}_FAILED", + "stage": _error_stage(result.failed_at_stage), + "message": result.error, + } + if result.status_code is not None: + error["statusCode"] = result.status_code + + payload: dict[str, Any] = { + "analysisId": result.analysis_id, + "requestId": request_id, + "status": "failed", + "originalUrl": result.original_url, + "error": error, + "engineVersion": settings.app_version, + "analyzedAt": _iso_z(analyzed_at), + "elapsedMs": elapsed_ms, + } + if result.final_url is not None: + payload["finalUrl"] = result.final_url + return payload + + +def build_analysis_callback_payload( + result: PipelineSuccess | PipelineFailure, + *, + request_id: str, + elapsed_ms: int, + analyzed_at: datetime, +) -> dict[str, Any]: + if isinstance(result, PipelineSuccess): + return _success_payload( + result, + request_id=request_id, + elapsed_ms=elapsed_ms, + analyzed_at=analyzed_at, + ) + return _failure_payload( + result, + request_id=request_id, + elapsed_ms=elapsed_ms, + analyzed_at=analyzed_at, + ) + + +async def post_analysis_callback( + result: PipelineSuccess | PipelineFailure, + *, + request_id: str, + elapsed_ms: int, + analyzed_at: datetime, +) -> bool: + payload = build_analysis_callback_payload( + result, + request_id=request_id, + elapsed_ms=elapsed_ms, + analyzed_at=analyzed_at, + ) + headers = { + "Content-Type": "application/json", + "X-Internal-Api-Key": settings.internal_api_key, + "X-Request-ID": request_id, + } + url = _callback_url() + + async with httpx.AsyncClient(timeout=10.0) as client: + for attempt in range(1, _MAX_ATTEMPTS + 1): + try: + resp = await client.post(url, json=payload, headers=headers) + except httpx.HTTPError as exc: + logger.warning( + "analysis_callback.request_failed", + analysis_id=result.analysis_id, + attempt=attempt, + error=str(exc), + ) + else: + if 200 <= resp.status_code < 300: + logger.info( + "analysis_callback.delivered", + analysis_id=result.analysis_id, + attempt=attempt, + ) + return True + logger.warning( + "analysis_callback.bad_status", + analysis_id=result.analysis_id, + attempt=attempt, + status_code=resp.status_code, + ) + + if attempt < _MAX_ATTEMPTS: + await asyncio.sleep(0.5 * (2 ** (attempt - 1))) + + logger.error( + "analysis_callback.dead_letter", + analysis_id=result.analysis_id, + callback_url=url, + attempts=_MAX_ATTEMPTS, + payload=payload, + ) + return False diff --git a/app/services/content_analyzer/__init__.py b/app/services/content_analyzer/__init__.py new file mode 100644 index 0000000..51ad6d9 --- /dev/null +++ b/app/services/content_analyzer/__init__.py @@ -0,0 +1,3 @@ +from app.services.content_analyzer.analyze import analyze_content, skipped_already_danger + +__all__ = ["analyze_content", "skipped_already_danger"] diff --git a/app/services/content_analyzer/ai.py b/app/services/content_analyzer/ai.py new file mode 100644 index 0000000..33726c0 --- /dev/null +++ b/app/services/content_analyzer/ai.py @@ -0,0 +1,83 @@ +"""AI 피싱 추론 어댑터 — 모델을 나중에 갈아끼울 수 있게 Protocol 로 분리. + +모델 미정 상태에서는 NullAIProvider 로 동작. 실제 모델 연동은 +set_ai_provider() 로 부팅 시 런타임에 교체한다 (Spring 측과의 호환성 유지용). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from app.schemas.content_analysis import AIVerdict, TokenUsage + + +@dataclass(frozen=True) +class AIPromptContext: + final_url: str + title: str | None + has_password_field: bool + has_meta_refresh: bool + # frozen 데이터클래스는 deep-freeze 가 아니므로 mutable list 를 두면 type 계약과 어긋난다. + # tuple 로 받아 호출 측이 의도치 않게 mutate 하는 경로를 차단. + image_alts: tuple[str, ...] + external_link_ratio: float | None + # True 면 정적 추출이 결정적이지 않음 — has_password_field=False 를 + # "폼 없음" 으로 단정하면 안 된다. + is_spa_shell: bool = False + has_password_form_external_action: bool = False + has_external_meta_refresh: bool = False + # 상위 단계(도메인 휴리스틱·threat_db)에서 이미 잡힌 시그널 코드 목록. + # AI 가 페이지 피처 단독으로는 결론 내기 약한 케이스에 사전 정보로 활용한다. + # 예: TYPO_DOMAIN + 브랜드 title + 비밀번호 폼이면 단독 페이지 분석보다 강한 phishing 신호. + upstream_signals: tuple[str, ...] = () + # HTML 원문 대신 비용이 낮은 구조화 요약만 전달한다. + body_text_snippets: tuple[str, ...] = () + form_field_summaries: tuple[str, ...] = () + cta_texts: tuple[str, ...] = () + download_links: tuple[str, ...] = () + sensitive_field_types: tuple[str, ...] = () + korean_lure_keywords: tuple[str, ...] = () + public_agency_keywords: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AIInference: + verdict: AIVerdict + reason: str + # 어떤 모델이 응답했는지 + 토큰 사용량. 응답에 그대로 실어 비용/성능 관측에 쓴다. + # 과거 호환 & 테스트 편의를 위해 옵셔널로 둔다 — 실제 프로바이더는 채워서 돌려준다. + model: str | None = None + token_usage: TokenUsage | None = None + + +class AIProvider(Protocol): + async def infer(self, ctx: AIPromptContext) -> AIInference | None: ... + + +class NullAIProvider: + """모델이 설정되지 않았을 때의 기본 구현. 추론을 시도하지 않고 None 반환. + + `fallback_reason` 이 None 이 아니면 "정상 NullProvider" 가 아니라 다른 프로바이더가 + 부팅 시 폴백된 상태라는 뜻이다. analyze.py 가 이 값을 읽어 응답의 `ai_error` 로 + 노출해 운영자가 NullProvider 정상 동작과 misconfigured 폴백을 구분할 수 있게 한다. + """ + + def __init__(self, *, fallback_reason: str | None = None) -> None: + self.fallback_reason = fallback_reason + + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + return None + + +_provider: AIProvider = NullAIProvider() + + +def get_ai_provider() -> AIProvider: + return _provider + + +def set_ai_provider(provider: AIProvider) -> None: + """앱 부팅 시점에만 호출 — 파이프라인 실행 중 교체하면 race 위험.""" + global _provider + _provider = provider diff --git a/app/services/content_analyzer/ai_openai.py b/app/services/content_analyzer/ai_openai.py new file mode 100644 index 0000000..90502a6 --- /dev/null +++ b/app/services/content_analyzer/ai_openai.py @@ -0,0 +1,272 @@ +"""OpenAI Chat Completions 기반 피싱 추론 어댑터. + +Structured Outputs (`response_format=json_schema`, `strict=True`) 로 JSON 포맷을 +모델 측에서 강제한다. 클라이언트 측 파싱은 최소한으로 유지하고, 스키마 위반· +API 오류는 전부 None 으로 떨어뜨려 상위 analyze 에서 `ai_error` 로 기록한다. +asyncio.CancelledError 만 예외적으로 re-raise — 취소 신호를 삼키지 않는다. + +모델 교체는 생성자 파라미터 또는 `settings.openai_model` 환경변수로 한다. +OpenAI 채팅 모델(gpt-4o-mini / gpt-4o / gpt-4.1-mini 등)은 전부 같은 코드로 돌아간다. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import httpx + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.content_analysis import AIVerdict, TokenUsage +from app.services.content_analyzer.ai import AIInference, AIPromptContext + +logger = get_logger(__name__) + +_VERDICT_SCHEMA: dict[str, Any] = { + "name": "phishing_verdict", + "strict": True, + "schema": { + "type": "object", + "properties": { + "verdict": { + "type": "string", + "enum": [v.value for v in AIVerdict], + }, + "reason": { + "type": "string", + "description": "Korean analysis reason plus action guidance, within 100 chars.", + }, + }, + "required": ["verdict", "reason"], + "additionalProperties": False, + }, +} + +_RESPONSE_FORMAT: dict[str, Any] = { + "type": "json_schema", + "json_schema": _VERDICT_SCHEMA, +} + +_SYSTEM_PROMPT = ( + "당신은 웹 페이지 피싱 여부를 판정하는 보안 엔진이다. " + "입력은 이미 크롤러가 뽑아놓은 구조화된 피처(title, 비밀번호 필드 유무, " + "meta refresh 유무, 외부 링크 비율, 이미지 alt 텍스트, 본문 요약, 폼 필드 요약, " + "CTA 문구, 다운로드 링크, 민감 입력 필드 후보, 한국형 루어 키워드, " + "최종 URL, spa_shell 플래그)와 " + "선행 단계(도메인 휴리스틱·위협 DB)에서 이미 잡힌 시그널 코드 배열 upstream_signals 다. " + "규칙 기반 점수는 이미 상위 레이어에서 계산되고 있으므로, 당신은 " + "'이 페이지가 특정 브랜드를 사칭하거나 자격증명을 탈취하려 하는지' 를 " + "뉘앙스 기반으로만 판정한다. " + "upstream_signals 는 도메인/네트워크 레이어의 사전 정보다 — 단독 시그널이 아니라 " + "페이지 피처와 결합해 판정을 강화하는 데 쓴다. 예: TYPO_DOMAIN + 브랜드 title + " + "비밀번호 폼이면 단독 페이지 분석보다 강한 phishing 신호다. NEW_DOMAIN/IP_DIRECT 도 " + "동일한 맥락에서 가중하되, 페이지 측 증거 없이 upstream_signals 만으로 phishing 을 " + "확정하지는 않는다. " + "spa_shell=true 이면 초기 HTML 이 JS 마운트 셸뿐이라 has_password_field=false 는 " + "'폼이 없다' 가 아니라 '정적 추출로는 판정 불가' 를 의미한다. 이 경우 남은 단서" + "(title/URL/이미지 alt/upstream_signals 등)만으로 단정하지 말고 suspicious 또는 benign 을 " + "보수적으로 선택한다. " + "verdict 는 phishing / suspicious / benign 중 하나. reason 은 보안 전문가처럼 " + "근거 중심으로 쓰되, IT와 보안을 모르는 사람도 이해할 수 있는 쉬운 한국어 100자 이내 " + "1문장으로 작성한다. reason 에는 별도 필드 없이 분석 근거와 사용자 행동 가이드를 함께 " + "담아라. 예: 비밀번호 입력에 주의하세요, 결제 수단을 등록하지 마세요, 첨부 파일을 " + "내려받지 마세요. " + "확증이 없으면 benign 또는 suspicious 를 쓰고 phishing 은 보수적으로만 사용한다." +) + + +def _build_user_prompt(ctx: AIPromptContext) -> str: + # 토큰 절약 차원에서 짧은 JSON 으로 직렬화. 중요 필드만 추려서 모델이 노이즈에 끌리지 않게 한다. + payload = { + "final_url": ctx.final_url, + "title": ctx.title, + "has_password_field": ctx.has_password_field, + "has_password_form_external_action": ctx.has_password_form_external_action, + "has_meta_refresh": ctx.has_meta_refresh, + "has_external_meta_refresh": ctx.has_external_meta_refresh, + "external_link_ratio": ctx.external_link_ratio, + "spa_shell": ctx.is_spa_shell, + # 과도하게 긴 alt 리스트는 비용만 늘리고 판정에 도움 안 됨 + "image_alts": list(ctx.image_alts[:10]), + "body_text": list(ctx.body_text_snippets[:10]), + "form_fields": list(ctx.form_field_summaries[:20]), + "cta_texts": list(ctx.cta_texts[:10]), + "download_links": list(ctx.download_links[:10]), + "sensitive_field_types": list(ctx.sensitive_field_types), + "korean_lure_keywords": list(ctx.korean_lure_keywords), + "public_agency_keywords": list(ctx.public_agency_keywords), + # 선행 단계 시그널 — 빈 배열이면 단독 페이지 분석과 동일. + "upstream_signals": list(ctx.upstream_signals), + } + return json.dumps(payload, ensure_ascii=False) + + +def _extract_token_usage(completion: object) -> TokenUsage | None: + # HTTP 응답 dict 또는 SDK 호환 객체를 모두 받는다. 스트리밍/에러 응답 등에선 비어있을 수 있다. + if isinstance(completion, dict): + usage = completion.get("usage") + else: + usage = getattr(completion, "usage", None) + if usage is None: + return None + if isinstance(usage, dict): + prompt = usage.get("prompt_tokens") + completion_tokens = usage.get("completion_tokens") + total = usage.get("total_tokens") + else: + prompt = getattr(usage, "prompt_tokens", None) + completion_tokens = getattr(usage, "completion_tokens", None) + total = getattr(usage, "total_tokens", None) + if prompt is None or completion_tokens is None or total is None: + return None + return TokenUsage( + prompt_tokens=int(prompt), + completion_tokens=int(completion_tokens), + total_tokens=int(total), + ) + + +def _parse_verdict( + raw: str | None, + *, + model: str, + token_usage: TokenUsage | None, +) -> AIInference | None: + if not raw: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + logger.warning("openai_ai.parse_error", raw=raw[:200]) + return None + + verdict_str = data.get("verdict") + reason = data.get("reason") + if not isinstance(verdict_str, str) or not isinstance(reason, str): + return None + reason = reason.strip() + if len(reason) > 100: + reason = reason[:100] + + try: + verdict = AIVerdict(verdict_str) + except ValueError: + logger.warning("openai_ai.unknown_verdict", verdict=verdict_str) + return None + + return AIInference( + verdict=verdict, + reason=reason, + model=model, + token_usage=token_usage, + ) + + +class OpenAIProvider: + """OpenAI Chat Completions 기반 AIProvider 구현. + + - 모델/타임아웃/최대 출력 토큰은 생성자에서 오버라이드 가능. 미지정 시 settings 값을 사용. + 같은 클래스로 gpt-4o-mini, gpt-4o, gpt-4.1 등 OpenAI 계열 모델을 바로 스왑할 수 있다. + - API 키가 비어있으면 infer() 는 즉시 None (NullAIProvider 와 동일 동작). + - 요청 실패·파싱 실패는 전부 None 으로 흡수 → 파이프라인은 계속 돈다. + - `asyncio.CancelledError` 는 re-raise. + """ + + def __init__( + self, + *, + model: str | None = None, + timeout_seconds: float | None = None, + max_output_tokens: int | None = None, + ) -> None: + # 생성자 인자가 우선, 없으면 settings 기본값. 인스턴스 생성 후에는 불변으로 본다. + self._model = model if model is not None else settings.openai_model + self._timeout = ( + timeout_seconds if timeout_seconds is not None else settings.openai_timeout_seconds + ) + self._max_output_tokens = ( + max_output_tokens + if max_output_tokens is not None + else settings.openai_max_output_tokens + ) + # 클라이언트는 호출 시점에 지연 생성 — 키가 없으면 아예 만들지 않는다. + self._client: httpx.AsyncClient | None = None + # aclose() 후 재사용 방지. 더블 콜은 no-op, 이후 infer() 는 None 반환. + self._closed: bool = False + + @property + def model(self) -> str: + return self._model + + def _get_client(self) -> httpx.AsyncClient | None: + if self._closed: + # 정상 시나리오에선 lifespan 종료 후 호출이 없어야 한다. 호출이 들리면 + # provider 재바인딩이 누락된 신호이므로 silent 폴백 대신 한 번 경고로 남긴다. + logger.warning("openai_ai.called_after_close", model=self._model) + return None + if not settings.openai_api_key: + return None + if self._client is None: + self._client = httpx.AsyncClient( + base_url="https://api.openai.com/v1", + timeout=self._timeout, + headers={ + "Authorization": f"Bearer {settings.openai_api_key}", + "Content-Type": "application/json", + }, + trust_env=False, + ) + return self._client + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + if self._client is not None: + await self._client.aclose() + self._client = None + + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + client = self._get_client() + if client is None: + return None + + messages: list[dict[str, str]] = [ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": _build_user_prompt(ctx)}, + ] + payload = { + "model": self._model, + "messages": messages, + "response_format": _RESPONSE_FORMAT, + "max_tokens": self._max_output_tokens, + "temperature": 0, + } + try: + response = await asyncio.wait_for( + client.post("/chat/completions", json=payload), + timeout=self._timeout, + ) + response.raise_for_status() + completion = response.json() + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning( + "openai_ai.request_failed", + error=str(exc), + error_type=type(exc).__name__, + model=self._model, + ) + return None + + try: + raw = completion["choices"][0]["message"]["content"] + except (KeyError, TypeError, IndexError) as exc: + logger.warning("openai_ai.unexpected_shape", error=str(exc)) + return None + + token_usage = _extract_token_usage(completion) + return _parse_verdict(raw, model=self._model, token_usage=token_usage) diff --git a/app/services/content_analyzer/analyze.py b/app/services/content_analyzer/analyze.py new file mode 100644 index 0000000..f4ca63a --- /dev/null +++ b/app/services/content_analyzer/analyze.py @@ -0,0 +1,363 @@ +"""콘텐츠 정적 분석 오케스트레이터 — 4단계 진입점. + +fetch · extract · signals · AI 를 순차 호출하고 ContentAnalysisResult 로 병합한다. +외부 의존성(네트워크·AI) 실패는 degraded 결과로 흡수하되, +asyncio.CancelledError 는 반드시 re-raise 해서 상위 shutdown/timeout 신호가 살아있게 한다. +""" + +from __future__ import annotations + +import asyncio +import inspect +from collections.abc import Awaitable, Iterable + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.content_analysis import ( + AIVerdict, + ContentAnalysisResult, + ContentSignal, + TokenUsage, +) +from app.services.content_analyzer.ai import AIPromptContext, AIProvider, get_ai_provider +from app.services.content_analyzer.extract import ExtractedFeatures, extract_features_async +from app.services.content_analyzer.fetch import fetch_page +from app.services.content_analyzer.render import render_page +from app.services.content_analyzer.signals import ContentScoring, score_content +from app.services.domain_heuristic.patterns import is_trusted_registered_domain + +logger = get_logger(__name__) + + +def _build_prompt_context( + final_url: str, + features: ExtractedFeatures, + upstream_signals: tuple[str, ...], +) -> AIPromptContext: + return AIPromptContext( + final_url=final_url, + title=features.title, + has_password_field=features.has_password_field, + has_meta_refresh=features.has_meta_refresh, + image_alts=tuple(features.image_alts), + external_link_ratio=features.external_link_ratio, + is_spa_shell=features.is_spa_shell, + has_password_form_external_action=features.has_password_form_external_action, + has_external_meta_refresh=features.has_external_meta_refresh, + upstream_signals=upstream_signals, + body_text_snippets=tuple(features.body_text_snippets), + form_field_summaries=tuple(features.form_field_summaries), + cta_texts=tuple(features.cta_texts), + download_links=tuple(features.download_links), + sensitive_field_types=tuple(features.sensitive_field_types), + korean_lure_keywords=tuple(features.korean_lure_keywords), + public_agency_keywords=tuple(features.public_agency_keywords), + ) + + +def _ai_score_weight(verdict: AIVerdict) -> int: + # benign 은 규칙 점수를 감점하지 않는다 — AI 오탐이 정상 점수까지 깎을 수 있기 때문 + if verdict == AIVerdict.PHISHING: + return settings.score_weight_ai_phishing + if verdict == AIVerdict.SUSPICIOUS: + return settings.score_weight_ai_suspicious + return 0 + + +_HIGH_RISK_CONTENT_SIGNALS: frozenset[str] = frozenset( + { + ContentSignal.BRAND_IMPERSONATION_FORM.value, + ContentSignal.CREDENTIAL_FORM_EXTERNAL.value, + ContentSignal.SENSITIVE_ID_FIELD.value, + ContentSignal.FINANCIAL_FIELD.value, + ContentSignal.RISKY_DOWNLOAD_LINK.value, + ContentSignal.EXTERNAL_META_REFRESH.value, + } +) +_HIGH_RISK_UPSTREAM_SIGNALS: frozenset[str] = frozenset( + { + "URL_USERINFO", + "OPEN_REDIRECT_PARAM", + "BRAND_IN_URL", + "FREE_HOSTING_LURE", + "SENSITIVE_PATH", + "SUSPICIOUS_TLD", + "PUNYCODE_IDN", + "TYPO_DOMAIN", + } +) + + +def _should_apply_ai_suspicious_score( + final_url: str, + scoring: ContentScoring, + upstream_signals: tuple[str, ...], +) -> bool: + if not is_trusted_registered_domain(final_url): + return True + content_signal_values = {signal.value for signal in scoring.signals} + if content_signal_values & _HIGH_RISK_CONTENT_SIGNALS: + return True + return bool(set(upstream_signals) & _HIGH_RISK_UPSTREAM_SIGNALS) + + +def _unique_merge(left: list[str], right: list[str]) -> list[str]: + merged = list(left) + for item in right: + if item not in merged: + merged.append(item) + return merged + + +def _merge_features(static: ExtractedFeatures, rendered: ExtractedFeatures) -> ExtractedFeatures: + return ExtractedFeatures( + title=rendered.title or static.title, + has_password_field=static.has_password_field or rendered.has_password_field, + has_password_form_external_action=( + static.has_password_form_external_action + or rendered.has_password_form_external_action + ), + has_meta_refresh=static.has_meta_refresh or rendered.has_meta_refresh, + has_external_meta_refresh=( + static.has_external_meta_refresh or rendered.has_external_meta_refresh + ), + external_link_ratio=( + rendered.external_link_ratio + if rendered.external_link_ratio is not None + else static.external_link_ratio + ), + image_alts=_unique_merge(static.image_alts, rendered.image_alts), + is_spa_shell=rendered.is_spa_shell, + body_text_snippets=_unique_merge( + static.body_text_snippets, + rendered.body_text_snippets, + ), + form_field_summaries=_unique_merge( + static.form_field_summaries, + rendered.form_field_summaries, + ), + cta_texts=_unique_merge(static.cta_texts, rendered.cta_texts), + download_links=_unique_merge(static.download_links, rendered.download_links), + sensitive_field_types=_unique_merge( + static.sensitive_field_types, + rendered.sensitive_field_types, + ), + korean_lure_keywords=_unique_merge( + static.korean_lure_keywords, + rendered.korean_lure_keywords, + ), + public_agency_keywords=_unique_merge( + static.public_agency_keywords, + rendered.public_agency_keywords, + ), + ) + + +def _should_run_precision_analysis( + features: ExtractedFeatures, + scoring: ContentScoring, +) -> bool: + if not settings.content_precision_enabled: + return False + if scoring.score >= settings.score_danger_threshold: + return False + if features.is_spa_shell: + return True + if scoring.score >= settings.content_precision_min_score: + return True + return bool( + features.download_links + or features.sensitive_field_types + or features.korean_lure_keywords + or features.public_agency_keywords + ) + + +# 정상 컨텐츠(이미지·PDF·대용량 정적 페이지) 또는 파이프라인 정합성 이슈(unchainer 가 놓친 3xx)는 +# 악성 신호로 보지 않는다 — FETCH_FAILED 시그널만 남기고 점수는 가산하지 않는다. +# blocked_host 는 우리가 SSRF 1·2선에서 차단한 결과지 페이지의 악성 신호가 아니라 동일하게 0 가산. +# 그 외(timeout/connect_error/http_error 등)는 보수적으로 settings 의 가중치를 가산. +_FETCH_ERROR_NO_SCORE: frozenset[str] = frozenset( + {"not_html", "too_large", "unexpected_redirect", "blocked_host"} +) +_SKIPPED_ALREADY_DANGER_REASON = "위험성이 확인된 URL입니다. 페이지를 열지 않는 것이 좋습니다." + + +def _fetch_failed_score(error: str | None) -> int: + if error in _FETCH_ERROR_NO_SCORE: + return 0 + return settings.score_weight_content_fetch_failed + + +def _fetch_failed_reason(error: str | None) -> str: + if error == "http_error_404": + return "페이지를 찾을 수 없습니다." + if error and error.startswith("http_error_4"): + return "페이지 요청이 거부되었거나 찾을 수 없습니다." + if error and error.startswith("http_error_5"): + return "대상 서버 오류로 페이지를 확인할 수 없습니다." + if error == "timeout": + return "페이지 응답 시간이 초과되었습니다." + if error == "connect_error": + return "페이지에 연결할 수 없습니다." + if error == "dns_failure": + return "도메인 주소를 확인할 수 없습니다." + if error == "not_html": + return "분석 가능한 HTML 페이지가 아닙니다." + if error == "too_large": + return "페이지가 너무 커서 분석하지 않았습니다." + if error == "blocked_host": + return "내부망 또는 차단된 호스트라 분석하지 않았습니다." + if error == "unexpected_redirect": + return "예상하지 못한 리다이렉트 응답으로 페이지를 분석하지 못했습니다." + return "페이지를 가져오지 못했습니다." + + +def _fetch_failed_result(final_url: str, error: str | None) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=None, + score=_fetch_failed_score(error), + signals=[ContentSignal.FETCH_FAILED], + reason=_fetch_failed_reason(error), + error=error, + ) + + +def skipped_already_danger(final_url: str) -> ContentAnalysisResult: + """선행 단계에서 이미 danger 판정 — 네트워크·AI 비용을 아끼기 위해 분석을 건너뛴 상태. + + 점수는 0으로 두어 상위 합산에 왜곡을 주지 않는다. 이미 다른 신호로 danger 가 + 확정돼 있으므로 여기서 추가 가산할 필요가 없다. + """ + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + score=0, + signals=[ContentSignal.SKIPPED_ALREADY_DANGER], + reason=_SKIPPED_ALREADY_DANGER_REASON, + error="skipped_already_danger", + ) + + +async def analyze_content( + final_url: str, + *, + provider: AIProvider | None = None, + upstream_signals: Iterable[str] | Awaitable[Iterable[str]] = (), +) -> ContentAnalysisResult: + """콘텐츠 정적 분석 진입점. + + provider 를 명시하면 전역 프로바이더 대신 그것을 사용한다 — 모델 비교/디버그용. + None(기본)이면 set_ai_provider() 로 설정된 전역 값을 쓴다. + upstream_signals 는 도메인 휴리스틱·threat_db 등 선행 단계에서 잡힌 시그널 코드. + AI 프롬프트에 사전 정보로 실어 보내 단독 페이지 피처보다 강한 판정이 가능하게 한다. + """ + fetch_result = await fetch_page(final_url) + if not fetch_result.ok: + logger.info( + "content_analysis.fetch_failed", + url=final_url, + error=fetch_result.error, + status=fetch_result.status_code, + ) + result = _fetch_failed_result(final_url, fetch_result.error) + result.status_code = fetch_result.status_code + return result + + features = await extract_features_async(fetch_result.html, base_url=final_url) + scoring: ContentScoring = score_content(features, final_url) + precision_analysis_used = False + precision_analysis_error: str | None = None + + if _should_run_precision_analysis(features, scoring): + precision_analysis_used = True + render_result = await render_page(final_url) + if render_result.ok: + rendered_features = await extract_features_async(render_result.html, base_url=final_url) + features = _merge_features(features, rendered_features) + scoring = score_content(features, final_url) + if ContentSignal.RENDERED_DOM_ANALYZED not in scoring.signals: + scoring.signals.append(ContentSignal.RENDERED_DOM_ANALYZED) + else: + precision_analysis_error = render_result.error + + ai_verdict: AIVerdict | None = None + ai_reason: str | None = None + ai_error: str | None = None + ai_model: str | None = None + ai_token_usage: TokenUsage | None = None + + active_provider = provider if provider is not None else get_ai_provider() + if inspect.isawaitable(upstream_signals): + upstream_tuple = tuple(await upstream_signals) + else: + upstream_tuple = tuple(upstream_signals) + try: + inference = await active_provider.infer( + _build_prompt_context(final_url, features, upstream_tuple) + ) + except asyncio.CancelledError: + # shutdown / 요청 타임아웃 신호는 degraded 결과로 흡수하지 않는다 + raise + except Exception as exc: + logger.warning( + "content_analysis.ai_error", + url=final_url, + error=str(exc), + error_type=type(exc).__name__, + ) + ai_error = "ai_unavailable" + inference = None + + score = scoring.score + if inference is not None: + ai_verdict = inference.verdict + ai_reason = inference.reason + ai_model = inference.model + ai_token_usage = inference.token_usage + if inference.verdict != AIVerdict.SUSPICIOUS or _should_apply_ai_suspicious_score( + final_url, + scoring, + upstream_tuple, + ): + score += _ai_score_weight(inference.verdict) + elif ai_error is None: + # 추론이 None 인데 호출 단계 예외도 없었다면 NullAIProvider 동작. + # 부팅 시 misconfiguration 으로 폴백된 NullProvider 면 fallback_reason 을 응답에 노출. + # 정상 NullProvider 는 fallback_reason=None 이라 ai_error 가 None 으로 유지된다. + ai_error = getattr(active_provider, "fallback_reason", None) + + score = min(score, settings.content_analysis_score_cap) + + return ContentAnalysisResult( + final_url=final_url, + fetched=True, + status_code=fetch_result.status_code, + score=score, + signals=list(scoring.signals), + title=features.title, + has_password_field=features.has_password_field, + has_password_form_external_action=features.has_password_form_external_action, + has_meta_refresh=features.has_meta_refresh, + has_external_meta_refresh=features.has_external_meta_refresh, + external_link_ratio=features.external_link_ratio, + brand_impersonation=scoring.brand_impersonation, + logo_alt_impersonation=scoring.logo_alt_impersonation, + is_spa_shell=features.is_spa_shell, + body_text_snippets=features.body_text_snippets, + form_field_summaries=features.form_field_summaries, + cta_texts=features.cta_texts, + download_links=features.download_links, + sensitive_field_types=features.sensitive_field_types, + korean_lure_keywords=features.korean_lure_keywords, + public_agency_keywords=features.public_agency_keywords, + precision_analysis_used=precision_analysis_used, + precision_analysis_error=precision_analysis_error, + ai_verdict=ai_verdict, + ai_reason=ai_reason, + ai_error=ai_error, + ai_model=ai_model, + ai_token_usage=ai_token_usage, + ) diff --git a/app/services/content_analyzer/extract.py b/app/services/content_analyzer/extract.py new file mode 100644 index 0000000..f02694b --- /dev/null +++ b/app/services/content_analyzer/extract.py @@ -0,0 +1,460 @@ +"""HTML 페이지에서 피싱 신호 후보 필드를 추출한다. + +여기서는 "위험하다"는 판단을 하지 않는다. 단순 추출만 담당하고, +점수화·브랜드 비교는 signals.py 가 맡는다. + +파서는 lxml. 순수 파이썬 html.parser 대비 속도·메모리 모두 유리해서 단건 비용을 줄인다. +대신 BS4 가 모든 노드를 Tag 래퍼로 감싸는 비용은 그대로라 동시성 N 에 곱셈으로 폭주할 수 있어, +extract_features_async() 에 글로벌 세마포어 + to_thread 를 둬 피크 메모리에 천장을 박았다. +""" + +from __future__ import annotations + +import asyncio +import re +from dataclasses import dataclass, field +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup +from bs4.element import Tag + +from app.core.config import settings + +# href 가 상대 경로(빈 scheme)이면 base_url 과 urljoin 후 http(s) 로 정규화되므로 +# 여기서 검사 시점엔 이 둘만 보면 충분하다. +_NAV_SCHEMES: frozenset[str] = frozenset({"http", "https"}) + +# 주류 프레임워크의 마운트 컨테이너. 번들이 로드돼야 실제 DOM 이 생긴다. +# - root : React (CRA, Vite React) +# - app : Vue 2/3 CLI 기본 +# - __next : Next.js (SSR 없이 빌드된 경우) +# - __nuxt : Nuxt +# - __layout : SvelteKit 구버전 +# - svelte : Svelte/SvelteKit +# - q-app : Quasar/Qwik +_SPA_MOUNT_IDS: frozenset[str] = frozenset( + {"root", "app", "__next", "__nuxt", "__layout", "svelte", "q-app"} +) + +# Angular 는 id 가 아니라 커스텀 요소 태그로 마운트한다. +_SPA_MOUNT_TAGS: frozenset[str] = frozenset({"app-root", "ng-app"}) + +# alt 무제한 수집 시 alt 수만 개를 박은 페이지에서 signals 의 브랜드 매칭 비용이 선형으로 증폭. +# AI 프롬프트도 어차피 [:10] 으로 자르니 추출 단계에서 상한으로 막아둔다. +_MAX_IMAGE_ALTS = 200 + +# anchor / input / meta / div 등 일반 태그 순회 상한. fetch 단의 max_bytes(2MiB) 로 파싱 +# 입력은 이미 캡 됐지만, 그 안에서도 노드를 압축적으로 쌓은 페이지(예: 만 50만 개)에서 +# find_all 의 매칭 비용이 폭주하지 않도록 두 번째 방어선을 둔다. +_MAX_NODES_PER_TAG = 1000 +_MAX_TEXT_SNIPPETS = 40 +_MAX_TEXT_CHARS = 180 +_MAX_FORM_FIELD_SUMMARIES = 80 +_MAX_CTA_TEXTS = 40 +_MAX_DOWNLOAD_LINKS = 40 +_RISKY_DOWNLOAD_EXTENSIONS: frozenset[str] = frozenset( + {".apk", ".ipa", ".exe", ".msi", ".dmg", ".scr", ".bat", ".cmd", ".vbs"} +) +_RISKY_DOWNLOAD_ATTR_EXTENSIONS: frozenset[str] = frozenset({".js"}) + +_KOREAN_LURE_KEYWORDS: tuple[str, ...] = ( + "지원금", + "환급금", + "택배", + "부고", + "청첩장", + "과태료", + "건강보험", + "정부기관", + "카카오톡", + "인증번호", + "otp", +) + +_PUBLIC_AGENCY_KEYWORDS: tuple[str, ...] = ( + "국민건강보험", + "건강보험공단", + "건강보험", + "정부24", + "정부", + "공단", + "국세청", + "관세청", + "경찰청", + "검찰청", + "고용노동부", + "보건복지부", + "행정안전부", +) + +_SENSITIVE_FIELD_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("resident_registration_number", ("resident_registration", "rrn", "주민등록", "주민번호")), + ("phone", ("mobile", "phone", "tel", "휴대폰", "전화번호")), + ("card", ("card_number", "credit_card", "creditcard", "카드번호", "카드")), + ("cvc", ("cvc", "cvv")), + ("account", ("account", "bank", "계좌", "은행")), + ("otp", ("otp", "auth_code", "verification_code", "인증번호", "인증")), +) + + +@dataclass +class ExtractedFeatures: + title: str | None = None + has_password_field: bool = False + has_password_form_external_action: bool = False + has_meta_refresh: bool = False + has_external_meta_refresh: bool = False + external_link_ratio: float | None = None + image_alts: list[str] = field(default_factory=list) + is_spa_shell: bool = False + body_text_snippets: list[str] = field(default_factory=list) + form_field_summaries: list[str] = field(default_factory=list) + cta_texts: list[str] = field(default_factory=list) + download_links: list[str] = field(default_factory=list) + sensitive_field_types: list[str] = field(default_factory=list) + korean_lure_keywords: list[str] = field(default_factory=list) + public_agency_keywords: list[str] = field(default_factory=list) + + +def _normalize_host(host: str | None) -> str: + if not host: + return "" + host = host.lower() + return host.removeprefix("www.") + + +def _is_external_nav_url(raw_url: str, base_url: str) -> bool: + joined = urljoin(base_url, raw_url.strip()) + parsed = urlparse(joined) + if parsed.scheme not in _NAV_SCHEMES: + return False + base_host = _normalize_host(urlparse(base_url).hostname) + return _normalize_host(parsed.hostname) != base_host + + +def _compute_external_link_ratio(soup: BeautifulSoup, base_url: str) -> float | None: + base_host = _normalize_host(urlparse(base_url).hostname) + total = 0 + external = 0 + for a in soup.find_all("a", limit=_MAX_NODES_PER_TAG): + href = a.get("href") + if not href or not isinstance(href, str): + continue + joined = urljoin(base_url, href.strip()) + parsed = urlparse(joined) + if parsed.scheme not in _NAV_SCHEMES: + # mailto: / tel: / javascript: 등 네비게이션이 아닌 링크는 비율 분모에서 제외 + continue + total += 1 + if _normalize_host(parsed.hostname) != base_host: + external += 1 + + if total == 0: + return None + return external / total + + +def _extract_title(soup: BeautifulSoup) -> str | None: + if soup.title is None or soup.title.string is None: + return None + text = soup.title.string.strip() + return text or None + + +def _has_password_field(soup: BeautifulSoup) -> bool: + return soup.find("input", attrs={"type": _is_password_type}) is not None + + +def _is_password_type(value: object) -> bool: + return isinstance(value, str) and value.lower() == "password" + + +def _has_password_form_external_action(soup: BeautifulSoup, base_url: str) -> bool: + for form in soup.find_all("form"): + if form.find("input", attrs={"type": _is_password_type}) is None: + continue + action = form.get("action") + if isinstance(action, str) and action.strip(): + return _is_external_nav_url(action, base_url) + return False + + +_META_REFRESH_URL_RE = re.compile(r"(?:^|;)\s*url\s*=\s*['\"]?([^'\";]+)", re.IGNORECASE) + + +def _meta_refresh_target(content: str) -> str | None: + match = _META_REFRESH_URL_RE.search(content) + if match is None: + return None + target = match.group(1).strip() + return target or None + + +def _meta_refresh_info(soup: BeautifulSoup, base_url: str) -> tuple[bool, bool]: + # http-equiv=refresh 만 보고 단정하면 헤더만 무의미하게 박힌 페이지가 잘못 매치된다. + # 실제 자동 리다이렉트 의도가 있으려면 content 도 비어있지 않아야 한다. + has_refresh = False + has_external = False + for meta in soup.find_all("meta", limit=_MAX_NODES_PER_TAG): + http_equiv = meta.get("http-equiv") + if not (isinstance(http_equiv, str) and http_equiv.lower() == "refresh"): + continue + content = meta.get("content") + if not (isinstance(content, str) and content.strip()): + continue + has_refresh = True + target = _meta_refresh_target(content) + if target is not None and _is_external_nav_url(target, base_url): + has_external = True + return has_refresh, has_external + + +_SPA_MOUNT_ID_SELECTOR: str = ", ".join(f'div[id="{i}"]' for i in _SPA_MOUNT_IDS) + + +def _detect_spa_shell(soup: BeautifulSoup) -> bool: + """초기 HTML 이 JS 마운트 셸만 담고 있는지 여부. + form/input 이 이미 있으면 SSR 됐거나 완전 정적 페이지이므로 판정 불가 대상 아님. + 그 상태에서 주요 프레임워크 마운트 컨테이너(id/태그)가 있으면 SPA 셸로 본다. + + div 마운트 검사는 CSS 셀렉터로 한 번에 — 모든 div 를 순회하지 않는다. + HTML5 의 id 매칭이 case-sensitive 라 셀렉터로는 정확 매치만 잡고, 대문자 ID 도 + 수용하던 종전 동작은 fallback 루프로 유지한다. + """ + if soup.find("form") is not None or soup.find("input") is not None: + return False + if soup.select_one(_SPA_MOUNT_ID_SELECTOR) is not None: + return True + for div in soup.find_all("div", limit=_MAX_NODES_PER_TAG): + div_id = div.get("id") + if isinstance(div_id, str) and div_id.lower() in _SPA_MOUNT_IDS: + return True + return any(soup.find(tag_name) is not None for tag_name in _SPA_MOUNT_TAGS) + + +def _collect_image_alts(soup: BeautifulSoup) -> list[str]: + alts: list[str] = [] + # find_all 자체에 limit 을 박아 매칭 비용까지 끊어둔다 — 백만 개 페이지에서 + # _MAX_IMAGE_ALTS 만큼 쌓일 때까지 alts 길이만 검사하던 종전 동작은 매칭 비용이 살아 있었다. + for img in soup.find_all("img", limit=_MAX_IMAGE_ALTS): + alt = img.get("alt") + if not isinstance(alt, str): + continue + alt = alt.strip() + if alt: + alts.append(alt) + return alts + + +def _trim_text(text: str, *, max_chars: int = _MAX_TEXT_CHARS) -> str: + text = " ".join(text.split()) + if len(text) <= max_chars: + return text + return text[:max_chars].rstrip() + + +def _append_unique(items: list[str], value: str, *, limit: int) -> None: + value = _trim_text(value) + if value and value not in items and len(items) < limit: + items.append(value) + + +def _collect_body_text_snippets(soup: BeautifulSoup) -> list[str]: + snippets: list[str] = [] + body = soup.body or soup + for text in body.stripped_strings: + _append_unique(snippets, text, limit=_MAX_TEXT_SNIPPETS) + if len(snippets) >= _MAX_TEXT_SNIPPETS: + break + return snippets + + +def _label_texts_by_for(soup: BeautifulSoup) -> dict[str, str]: + labels: dict[str, str] = {} + for label in soup.find_all("label", limit=_MAX_NODES_PER_TAG): + if not isinstance(label, Tag): + continue + target = label.get("for") + if isinstance(target, str) and target.strip(): + text = label.get_text(" ", strip=True) + if text: + labels[target.strip()] = _trim_text(text) + return labels + + +def _attr_text(tag: Tag, name: str) -> str | None: + value = tag.get(name) + if isinstance(value, str): + value = _trim_text(value) + return value or None + return None + + +def _field_label(control: Tag, labels_by_for: dict[str, str]) -> str | None: + control_id = _attr_text(control, "id") + if control_id and control_id in labels_by_for: + return labels_by_for[control_id] + parent_label = control.find_parent("label") + if isinstance(parent_label, Tag): + text = parent_label.get_text(" ", strip=True) + return _trim_text(text) if text else None + return None + + +def _collect_form_field_summaries(soup: BeautifulSoup) -> list[str]: + labels_by_for = _label_texts_by_for(soup) + summaries: list[str] = [] + for control in soup.find_all(["input", "textarea", "select"], limit=_MAX_NODES_PER_TAG): + if not isinstance(control, Tag): + continue + parts: list[str] = [] + label = _field_label(control, labels_by_for) + if label: + parts.append(f"label={label}") + for attr in ("name", "type", "placeholder", "id", "title", "aria-label"): + value = _attr_text(control, attr) + if value: + parts.append(f"{attr}={value}") + if parts: + _append_unique(summaries, " ".join(parts), limit=_MAX_FORM_FIELD_SUMMARIES) + if len(summaries) >= _MAX_FORM_FIELD_SUMMARIES: + break + return summaries + + +def _collect_cta_texts(soup: BeautifulSoup) -> list[str]: + texts: list[str] = [] + for tag in soup.find_all(["button", "a", "input"], limit=_MAX_NODES_PER_TAG): + if not isinstance(tag, Tag): + continue + text: str | None = None + if tag.name == "input": + input_type = (_attr_text(tag, "type") or "").lower() + if input_type not in {"submit", "button", "reset"}: + continue + text = _attr_text(tag, "value") + else: + text = tag.get_text(" ", strip=True) + if text: + _append_unique(texts, text, limit=_MAX_CTA_TEXTS) + if len(texts) >= _MAX_CTA_TEXTS: + break + return texts + + +def _anchor_has_download_attr(anchor: Tag) -> bool: + return anchor.has_attr("download") + + +def _is_risky_download_url(raw_url: str, base_url: str, *, has_download_attr: bool) -> str | None: + joined = urljoin(base_url, raw_url.strip()) + parsed = urlparse(joined) + if parsed.scheme not in _NAV_SCHEMES: + return None + path = parsed.path.lower() + if any(path.endswith(ext) for ext in _RISKY_DOWNLOAD_EXTENSIONS): + return joined + if has_download_attr and any( + path.endswith(ext) for ext in _RISKY_DOWNLOAD_ATTR_EXTENSIONS + ): + return joined + return None + + +def _collect_download_links(soup: BeautifulSoup, base_url: str) -> list[str]: + links: list[str] = [] + for anchor in soup.find_all("a", limit=_MAX_NODES_PER_TAG): + href = anchor.get("href") + if not isinstance(href, str): + continue + resolved = _is_risky_download_url( + href, + base_url, + has_download_attr=_anchor_has_download_attr(anchor), + ) + if resolved is not None: + _append_unique(links, resolved, limit=_MAX_DOWNLOAD_LINKS) + if len(links) >= _MAX_DOWNLOAD_LINKS: + break + return links + + +def _collect_sensitive_field_types(form_field_summaries: list[str]) -> list[str]: + found: list[str] = [] + haystack = "\n".join(form_field_summaries).lower() + for field_type, keywords in _SENSITIVE_FIELD_KEYWORDS: + if any(keyword.lower() in haystack for keyword in keywords): + found.append(field_type) + return found + + +def _collect_keywords(texts: list[str], keywords: tuple[str, ...]) -> list[str]: + found: list[str] = [] + haystack = "\n".join(texts).lower() + for keyword in keywords: + if keyword.lower() in haystack and keyword not in found: + found.append(keyword) + return found + + +def extract_features(html: str, base_url: str) -> ExtractedFeatures: + soup = BeautifulSoup(html or "", "lxml") + has_meta_refresh, has_external_meta_refresh = _meta_refresh_info(soup, base_url) + body_text_snippets = _collect_body_text_snippets(soup) + form_field_summaries = _collect_form_field_summaries(soup) + cta_texts = _collect_cta_texts(soup) + image_alts = _collect_image_alts(soup) + download_links = _collect_download_links(soup, base_url) + keyword_texts = [ + item + for item in [ + _extract_title(soup) or "", + *body_text_snippets, + *form_field_summaries, + *cta_texts, + *image_alts, + ] + if item + ] + return ExtractedFeatures( + title=_extract_title(soup), + has_password_field=_has_password_field(soup), + has_password_form_external_action=_has_password_form_external_action(soup, base_url), + has_meta_refresh=has_meta_refresh, + has_external_meta_refresh=has_external_meta_refresh, + external_link_ratio=_compute_external_link_ratio(soup, base_url), + image_alts=image_alts, + is_spa_shell=_detect_spa_shell(soup), + body_text_snippets=body_text_snippets, + form_field_summaries=form_field_summaries, + cta_texts=cta_texts, + download_links=download_links, + sensitive_field_types=_collect_sensitive_field_types(form_field_summaries), + korean_lure_keywords=_collect_keywords(keyword_texts, _KOREAN_LURE_KEYWORDS), + public_agency_keywords=_collect_keywords(keyword_texts, _PUBLIC_AGENCY_KEYWORDS), + ) + + +# 모듈 로드 시점이 아닌 첫 호출 때 만든다 — settings 가 환경변수로 동적으로 결정되는 경로를 +# 막지 않기 위함. 3.10+ 부터 Semaphore 는 특정 루프에 묶이지 않으므로 이대로 안전하다. +_extract_semaphore: asyncio.Semaphore | None = None + + +def _get_extract_semaphore() -> asyncio.Semaphore: + global _extract_semaphore + if _extract_semaphore is None: + _extract_semaphore = asyncio.Semaphore(settings.content_extract_concurrency) + return _extract_semaphore + + +async def extract_features_async(html: str, base_url: str) -> ExtractedFeatures: + """비동기 파이프라인용 진입점. + + BS4 파싱은 동기 CPU 작업이라 이벤트 루프를 블록한다. to_thread 로 오프로드해 + fetch·AI 호출 같은 IO 가 그동안 진행될 수 있게 한다. 동시에, BS4 트리는 본문 대비 ~10배로 + 부풀므로 incoming concurrency 가 폭주하면 메모리도 같이 폭주한다 — 글로벌 세마포어로 + 피크에 천장을 박아 운영 메모리 안에 가둔다. + """ + sem = _get_extract_semaphore() + async with sem: + return await asyncio.to_thread(extract_features, html, base_url) diff --git a/app/services/content_analyzer/fetch.py b/app/services/content_analyzer/fetch.py new file mode 100644 index 0000000..165cf0c --- /dev/null +++ b/app/services/content_analyzer/fetch.py @@ -0,0 +1,305 @@ +"""최종 URL의 HTML 본문을 취득한다. + +정적 분석 파이프라인 특성상 리다이렉트는 이미 2단계에서 해소됐다고 가정하고 +follow_redirects=False 로 굳힌다. 3xx 를 되받으면 unchainer 가 놓친 케이스이므로 +에러로 처리해 후속 단계에서 파이프라인 정합성을 따질 수 있게 한다. + +httpx.AsyncClient 는 모듈 레벨 싱글턴으로 재사용한다 — 매 요청마다 TCP/TLS 핸드셰이크를 +새로 여는 비용을 피하기 위함. lifespan 에서 aclose_client() 로 명시 종료한다. + +SSRF 방어 — 호스트가 IP 리터럴이면 사설/loopback/link-local 등이 아닌지 검사하고, +잘 알려진 내부 호스트네임(localhost, 클라우드 메타데이터 서비스)은 차단한다. +호스트네임은 추가로 실제 DNS 해석한 결과 IP 들도 동일 룰로 검증한다 — `evil.com → 127.0.0.1` +처럼 lexical 검사를 통과하는 케이스 차단. 단, getaddrinfo 시점과 connect 시점 사이의 +재해석(완전한 DNS rebind)은 본 1선에서 막지 못하므로 운영 환경에서는 egress 방화벽 +또는 분석 전용 프록시로 잔여 위험을 차단해야 한다. +""" + +from __future__ import annotations + +import ipaddress +import random +import socket +from dataclasses import dataclass +from urllib.parse import urlparse + +import httpx + +from app.core.config import settings +from app.core.dns_cache import resolve_host_addrs +from app.core.logging import get_logger + +logger = get_logger(__name__) + +# 분석 대상이 HTML이 아닌 바이너리(PNG, PDF, ZIP 등)면 정적 분석 자체가 의미 없으므로 조기 컷. +# application/xhtml+xml 은 의도적으로 제외 — BS4+lxml HTML 파서를 그대로 쓰지만, XHTML/XML +# 경로는 외부 DTD/엔티티 해석 가능성이 (lxml 버전·옵션에 따라) 회색지대라 입력에서 잘라낸다. +# 정상 트래픽 영향은 거의 없다 — 대부분의 사이트는 text/html 로 응답하고, XHTML 만 응답하는 +# 사이트는 분석 누락으로 두는 트레이드오프. +_HTML_CT_KEYWORDS: tuple[str, ...] = ("text/html",) + +# SSRF 방어 — 사용자가 정상 도메인을 위장해 보내도 호스트네임 자체가 내부 자원을 가리키면 차단. +# 클라우드 메타데이터 엔드포인트는 IP 가 169.254.169.254 라 IP 검사로도 대부분 잡히지만, +# DNS alias 만 노출된 케이스 대비 호스트네임도 함께 거부한다 (GCE/AWS/Alibaba 모두 커버). +_BLOCKED_HOSTNAMES: frozenset[str] = frozenset( + { + "localhost", + "ip6-localhost", + "ip6-loopback", + "metadata", + "metadata.google.internal", + "instance-data", + "instance-data.ec2.internal", + } +) + + +@dataclass +class FetchResult: + ok: bool + url: str + status_code: int | None = None + html: str = "" + error: str | None = None + + +# 모듈 레벨 싱글턴 — RDAP client 와 동일 패턴. 첫 요청 때 lazy init, lifespan 에서 aclose. +_client: httpx.AsyncClient | None = None + + +def _build_client() -> httpx.AsyncClient: + timeout = httpx.Timeout( + settings.content_fetch_timeout_seconds, + connect=settings.content_fetch_connect_timeout_seconds, + ) + if settings.content_fetch_proxy_url: + return httpx.AsyncClient( + timeout=timeout, + follow_redirects=False, + verify=True, + cookies=None, + trust_env=False, + proxy=settings.content_fetch_proxy_url, + ) + return httpx.AsyncClient( + timeout=timeout, + follow_redirects=False, + verify=True, + cookies=None, + trust_env=False, + ) + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None: + _client = _build_client() + return _client + + +async def aclose_client() -> None: + """앱 셧다운 훅에서 호출. 테스트에서도 client 상태 초기화에 쓴다.""" + global _client + if _client is not None: + await _client.aclose() + _client = None + + +def _pick_user_agent() -> str: + pool = settings.content_user_agents + # 풀이 비면 misconfiguration 이라 침묵 폴백 대신 raise — 매번 같은 UA 노출되는 + # 봇 탐지 회피 의도를 운영자가 알지 못한 채 흘려보내지 않게 한다. + if not pool: + raise RuntimeError("content_user_agents pool is empty — set CONTENT_USER_AGENTS") + # UA 로테이션은 보안 속성이 아니라 봇 탐지 회피 — random 으로 충분하다. + return random.choice(pool) # noqa: S311 + + +def _is_ip_literal(host: str) -> bool: + try: + ipaddress.ip_address(host) + except ValueError: + return False + return True + + +def _is_blocked_ip_literal(host: str) -> bool: + """호스트가 IP 리터럴이고 사설/loopback/link-local 등이면 True. IP 아니면 False.""" + try: + ip_obj = ipaddress.ip_address(host) + except ValueError: + return False + # is_reserved 는 DNS64/NAT64(64:ff9b::/96)처럼 공개 도달 가능한 대역도 True 로 + # 표시한다. 외부 접근 가능성은 is_global 을 기준으로 보고, multicast 는 별도 차단한다. + return not ip_obj.is_global or ip_obj.is_multicast + + +def _is_blocked_host(host: str | None) -> bool: + if not host: + return True + # urlparse().hostname 은 이미 IPv6 brackets 를 벗기지만, 다른 경로로 들어온 호스트 + # (테스트·내부 헬퍼)도 동일 룰을 통과시키도록 strip("[]") 를 한 번 더 둔다. + h = host.strip("[]").lower() + if h in _BLOCKED_HOSTNAMES: + return True + return _is_blocked_ip_literal(h) + + +async def _resolved_addrs_blocked(hostname: str) -> bool: + """hostname 의 모든 A/AAAA 응답을 _is_blocked_ip_literal 로 검증. + + 하나라도 사설/loopback 범위면 True. 해석 결과는 dns_cache 의 짧은 TTL 캐시로 재사용 — + 같은 호스트 반복 트래픽에서 매번 DNS 비용이 들어가지 않게 한다. + 완전한 DNS rebind(검증 후 connect 시점에 다른 IP 로 재해석) 는 본 1선에서 막을 수 없다 — + 배포 단의 egress 방화벽으로 보강해야 한다. + """ + infos = await resolve_host_addrs(hostname) + seen: set[str] = set() + for family, _type, _proto, _canon, sockaddr in infos: + if family not in (socket.AF_INET, socket.AF_INET6): + continue + addr = sockaddr[0] + if not isinstance(addr, str): + continue + # %zone 같은 IPv6 scope id 가 붙어 들어오면 제거 — ip_address() 가 거부한다. + # zone 이 다른 동일 IP 도 같은 정책을 적용하면 되므로 dedup 키에서 zone 은 의도적으로 뺀다. + # link-local(fe80::*) 은 zone 유무와 무관하게 _is_link_local 로 잡혀 차단된다. + ip = addr.split("%", 1)[0] + if ip in seen: + continue + seen.add(ip) + if _is_blocked_ip_literal(ip): + return True + return False + + +def _is_html_content_type(headers: httpx.Headers) -> bool: + ct = headers.get("content-type", "").lower() + return any(kw in ct for kw in _HTML_CT_KEYWORDS) + + +def _content_length_over_cap(headers: httpx.Headers, max_bytes: int) -> bool: + raw = headers.get("content-length") + if raw is None: + return False + try: + return int(raw) > max_bytes + except ValueError: + return False + + +async def _read_capped(resp: httpx.Response, max_bytes: int) -> tuple[bytes, bool]: + """응답 본문을 max_bytes 까지만 수집. 초과 시 (부분 본문, True) 반환.""" + buf = bytearray() + async for chunk in resp.aiter_bytes(): + if len(buf) + len(chunk) > max_bytes: + return bytes(buf), True + buf.extend(chunk) + return bytes(buf), False + + +def _decode(body: bytes, resp: httpx.Response) -> str: + encoding = resp.charset_encoding or "utf-8" + try: + return body.decode(encoding, errors="replace") + except LookupError: + # 서버가 이상한 encoding 을 선언했을 때의 안전망 + return body.decode("utf-8", errors="replace") + + +async def fetch_page(url: str) -> FetchResult: + """단일 GET 요청으로 HTML 본문 취득. 실패해도 raise 하지 않고 FetchResult 로 반환.""" + # SSRF 1선 방어 — 내부망 IP/호스트로 떨어지는 요청을 connect 전에 차단. + parsed = urlparse(url) + if _is_blocked_host(parsed.hostname): + logger.info("content_fetch.blocked_host", url=url, host=parsed.hostname) + return FetchResult(ok=False, url=url, error="blocked_host") + + # SSRF 2선 — 호스트네임이 공인 도메인처럼 보여도 실제로 사설 IP 로 풀릴 수 있다 + # (ex. `evil.com → 127.0.0.1`). 사전 DNS 해석 후 모든 응답 IP 를 동일 룰로 검증. + # IP 리터럴은 1선에서 이미 검증돼 통과된 것 — 같은 IP 를 다시 풀 필요 없다. + if parsed.hostname and not _is_ip_literal(parsed.hostname.strip("[]")): + try: + blocked = await _resolved_addrs_blocked(parsed.hostname) + except OSError: + # 해석 실패는 connect 단계 ConnectError 로 떨어지게 그대로 둔다 — + # 여기서 별도 코드 만들지 않는다. + blocked = False + if blocked: + logger.info("content_fetch.blocked_resolved_ip", url=url, host=parsed.hostname) + return FetchResult(ok=False, url=url, error="blocked_host") + + headers = { + "User-Agent": _pick_user_agent(), + # XHTML/XML 응답은 이후 _is_html_content_type 에서 거절되므로 Accept 에서도 빼둔다. + "Accept": "text/html,*/*;q=0.8", + "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7", + } + max_bytes = settings.content_fetch_max_bytes + + try: + client = _get_client() + req = client.build_request("GET", url, headers=headers) + resp = await client.send(req, stream=True) + try: + # 3xx 는 unchainer 가 이미 해소했어야 하는 신호 — 여기서는 예외로 취급 + if 300 <= resp.status_code < 400: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="unexpected_redirect", + ) + + if resp.status_code >= 400: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error=f"http_error_{resp.status_code}", + ) + + if not _is_html_content_type(resp.headers): + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="not_html", + ) + + if _content_length_over_cap(resp.headers, max_bytes): + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="too_large", + ) + + body, over = await _read_capped(resp, max_bytes) + if over: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="too_large", + ) + + html = _decode(body, resp) + return FetchResult(ok=True, url=url, status_code=resp.status_code, html=html) + finally: + await resp.aclose() + except httpx.TimeoutException: + return FetchResult(ok=False, url=url, error="timeout") + except httpx.ConnectError: + return FetchResult(ok=False, url=url, error="connect_error") + except httpx.HTTPError as exc: + logger.warning("content_fetch.http_error", url=url, error=str(exc)) + return FetchResult(ok=False, url=url, error="http_error") + except Exception as exc: + logger.warning( + "content_fetch.unexpected", + url=url, + error=str(exc), + error_type=type(exc).__name__, + ) + return FetchResult(ok=False, url=url, error="unexpected") diff --git a/app/services/content_analyzer/render.py b/app/services/content_analyzer/render.py new file mode 100644 index 0000000..374c2ba --- /dev/null +++ b/app/services/content_analyzer/render.py @@ -0,0 +1,117 @@ +"""선택적 Playwright 기반 렌더링 분석. + +정적 HTML 만으로 판정하기 어려운 SPA/동적 페이지에 한해 호출된다. +Playwright 가 설치되지 않았거나 브라우저 실행에 실패하면 degraded 결과를 반환하고, +기존 정적 분석 경로는 그대로 유지한다. +""" + +from __future__ import annotations + +import asyncio +import weakref +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + +from app.core.config import settings +from app.core.logging import get_logger +from app.services.content_analyzer.fetch import ( + _is_blocked_host, + _is_ip_literal, + _pick_user_agent, + _resolved_addrs_blocked, +) + +logger = get_logger(__name__) +_BLOCKED_RESOURCE_TYPES: frozenset[str] = frozenset({"image", "media", "font"}) + + +@dataclass(frozen=True) +class RenderResult: + ok: bool + url: str + html: str = "" + error: str | None = None + + +_render_semaphore: asyncio.Semaphore | None = None +_render_semaphore_loop_ref: weakref.ReferenceType[asyncio.AbstractEventLoop] | None = None + + +def _get_render_semaphore() -> asyncio.Semaphore: + global _render_semaphore, _render_semaphore_loop_ref + current_loop = asyncio.get_running_loop() + stored_loop = _render_semaphore_loop_ref() if _render_semaphore_loop_ref else None + if _render_semaphore is None or stored_loop is not current_loop: + _render_semaphore = asyncio.Semaphore(settings.content_render_concurrency) + _render_semaphore_loop_ref = weakref.ref(current_loop) + return _render_semaphore + + +async def _target_blocked(url: str) -> bool: + parsed = urlparse(url) + if _is_blocked_host(parsed.hostname): + return True + if parsed.hostname and not _is_ip_literal(parsed.hostname.strip("[]")): + try: + return await _resolved_addrs_blocked(parsed.hostname) + except OSError: + return False + return False + + +async def render_page(url: str) -> RenderResult: + """브라우저 렌더링 후 DOM HTML 을 반환한다. + + 내부망 SSRF 방어는 fetch 와 같은 1·2선 검사 함수를 재사용한다. DNS rebind 잔여 위험은 + fetch 와 동일하게 배포 단 egress 정책으로 닫아야 한다. + """ + if not settings.content_precision_enabled: + return RenderResult(ok=False, url=url, error="precision_disabled") + + if await _target_blocked(url): + logger.info("content_render.blocked_host", url=url) + return RenderResult(ok=False, url=url, error="blocked_host") + + try: + from playwright.async_api import TimeoutError as PlaywrightTimeoutError + from playwright.async_api import async_playwright + except ImportError: + return RenderResult(ok=False, url=url, error="playwright_unavailable") + + timeout_ms = int(settings.content_render_timeout_seconds * 1000) + sem = _get_render_semaphore() + async with sem: + try: + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + try: + page = await browser.new_page( + user_agent=_pick_user_agent(), + locale="ko-KR", + ) + + async def _route_handler(route: Any) -> None: + request = route.request + if request.resource_type in _BLOCKED_RESOURCE_TYPES: + await route.abort() + else: + await route.continue_() + + await page.route("**/*", _route_handler) + await page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms) + await page.wait_for_timeout(settings.content_render_settle_ms) + html = await page.content() + return RenderResult(ok=True, url=url, html=html) + finally: + await browser.close() + except PlaywrightTimeoutError: + return RenderResult(ok=False, url=url, error="timeout") + except Exception as exc: + logger.warning( + "content_render.failed", + url=url, + error=str(exc), + error_type=type(exc).__name__, + ) + return RenderResult(ok=False, url=url, error="render_failed") diff --git a/app/services/content_analyzer/signals.py b/app/services/content_analyzer/signals.py new file mode 100644 index 0000000..f51e7a8 --- /dev/null +++ b/app/services/content_analyzer/signals.py @@ -0,0 +1,217 @@ +"""추출된 피처 + 도메인을 바탕으로 규칙 기반 점수와 신호 코드를 산출한다. + +AI 추론 가산은 analyze.py 에서 합산한다 — 여기서는 순수 규칙 레이어만 담당. + +브랜드 매칭 전략 — ASCII 라벨은 단어 경계(\b) 매칭, 비-ASCII(한국어 등)는 substring fallback. +영문은 토큰 경계가 명확해 'pineapple' 이 'apple', 'naverstore' 가 'naver' 로 잡히는 +substring false-positive 를 경계 매칭으로 차단한다. 형태소 경계가 어려운 한국어/한자 +라벨은 substring 으로 받도록 코드 경로가 준비돼 있다. 다만 현재 brands.txt 는 ASCII +라벨만 들어 있어 substring 분기는 dead path 다 — 한국어 라벨 추가 시 자동 활성화된다. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path + +from app.core.config import settings +from app.core.tld import extract_url_parts +from app.schemas.content_analysis import ContentSignal +from app.services.content_analyzer.extract import ExtractedFeatures + +# typosquatting.py 와 동일한 brands.txt 를 재사용. 경로 변경 시 양쪽을 함께 수정해야 한다. +_BRANDS_FILE = Path(__file__).resolve().parents[1] / "domain_heuristic" / "brands.txt" + +# 짧은 브랜드 라벨(kb, sk 등)은 일반 영단어로 오탐되므로 4자 이상만 매칭에 사용. +_MIN_BRAND_LEN = 4 + + +def _read_brand_labels() -> frozenset[str]: + # BRAND_IMPERSONATION 은 4단계 최고 가중치 시그널 — 라벨이 비면 silent 하게 + # 무력화돼 안전 분석이 약해진다. 여기서 강하게 raise 해 부팅 실패로 만든다. + if not _BRANDS_FILE.exists(): + raise RuntimeError( + f"brand labels file missing: {_BRANDS_FILE}. " + "BRAND_IMPERSONATION/LOGO_ALT_IMPERSONATION 시그널이 무력화된다." + ) + labels: set[str] = set() + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + ext = extract_url_parts(f"https://{line}") + if ext.domain and len(ext.domain) >= _MIN_BRAND_LEN: + labels.add(ext.domain.lower()) + if not labels: + raise RuntimeError( + f"brand labels file produced zero labels: {_BRANDS_FILE}" + ) + return frozenset(labels) + + +@lru_cache(maxsize=1) +def _brand_label_index() -> tuple[frozenset[str], frozenset[str], re.Pattern[str] | None]: + """라벨을 lazy 하게 1회만 로드해 (ASCII set, non-ASCII set, ASCII alternation 패턴) 으로 반환. + + 예전엔 모듈 임포트 시점에 즉시 raise 하는 방식이라 라벨 파일 경로 변경/스왑 시 + 테스트 픽스처 단계에서 import 자체가 죽는 문제가 있었다. lru_cache 로 첫 호출 + 시점까지 미루고, 테스트는 `_brand_label_index.cache_clear()` 로 리셋하면 된다. + + 운영 정책 — brands.txt 갱신은 프로세스 재시작으로 반영. 핫 리로드는 지원하지 않는다. + 런타임 갱신이 필요해지면 SIGHUP 핸들러나 admin endpoint 로 cache_clear() 를 호출하는 + 경로를 별도로 도입해야 한다. + """ + labels = _read_brand_labels() + ascii_labels = frozenset(label for label in labels if label.isascii()) + non_ascii_labels = labels - ascii_labels + pattern = ( + re.compile(r"\b(" + "|".join(re.escape(label) for label in ascii_labels) + r")\b") + if ascii_labels + else None + ) + return ascii_labels, non_ascii_labels, pattern + + +@dataclass +class ContentScoring: + score: int = 0 + signals: list[ContentSignal] = field(default_factory=list) + brand_impersonation: bool = False + logo_alt_impersonation: bool = False + + +_SENSITIVE_ID_TYPES: frozenset[str] = frozenset({"resident_registration_number", "otp"}) +_PII_FIELD_TYPES: frozenset[str] = frozenset( + {"resident_registration_number", "phone", "otp"} +) +_FINANCIAL_FIELD_TYPES: frozenset[str] = frozenset({"card", "cvc", "account"}) + + +def _add_signal(result: ContentScoring, signal: ContentSignal, score: int = 0) -> None: + if signal not in result.signals: + result.signals.append(signal) + result.score += score + + +def _url_brand_label(final_url: str) -> str: + ext = extract_url_parts(final_url) + return (ext.domain or "").lower() + + +def _brands_in_text(text: str | None) -> set[str]: + # 영문은 \b 경계로만 매치 — 'pineapple' → 'apple', 'naverstore' → 'naver' 같은 + # substring false-positive 차단. 한국어 등 비-ASCII 라벨은 본문에 분리 없이 박히므로 substring. + # 4자 미만 라벨은 로드 단계에서 이미 제외됐다. + if not text: + return set() + _ascii_labels, non_ascii_labels, ascii_pattern = _brand_label_index() + lowered = text.lower() + found: set[str] = set() + if ascii_pattern is not None: + found.update(ascii_pattern.findall(lowered)) + for label in non_ascii_labels: + if label in lowered: + found.add(label) + return found + + +def score_content(features: ExtractedFeatures, final_url: str) -> ContentScoring: + result = ContentScoring() + own_label = _url_brand_label(final_url) + + # SPA 셸은 "판정 불가" 표식일 뿐 — 점수는 주지 않는다. + # 정상 SPA 가 대부분이라 가산하면 오탐이 크다. + if features.is_spa_shell: + result.signals.append(ContentSignal.SPA_SHELL) + + if features.has_password_field: + # 자기 도메인 라벨이 매칭된 브랜드에 있으면 정상. + # 그 외 브랜드가 하나라도 섞이면 위장으로 판단한다. + title_brands = _brands_in_text(features.title) + if title_brands and own_label not in title_brands: + result.brand_impersonation = True + result.signals.append(ContentSignal.BRAND_IMPERSONATION_FORM) + result.score += settings.score_weight_brand_impersonation + + if features.has_password_form_external_action: + _add_signal( + result, + ContentSignal.CREDENTIAL_FORM_EXTERNAL, + settings.score_weight_credential_form_external, + ) + + sensitive_types = set(features.sensitive_field_types) + has_pii_fields = bool(sensitive_types & _PII_FIELD_TYPES) + has_sensitive_id_fields = bool(sensitive_types & _SENSITIVE_ID_TYPES) + has_financial_fields = bool(sensitive_types & _FINANCIAL_FIELD_TYPES) + has_lure_text = bool(features.korean_lure_keywords) + has_public_agency_lure = bool(features.public_agency_keywords) + + if has_lure_text: + _add_signal(result, ContentSignal.KOREAN_LURE_TEXT) + + if has_public_agency_lure: + _add_signal(result, ContentSignal.PUBLIC_AGENCY_LURE) + + if has_pii_fields: + _add_signal( + result, + ContentSignal.PII_COLLECTION_FORM, + settings.score_weight_pii_collection_form, + ) + + if has_sensitive_id_fields: + _add_signal( + result, + ContentSignal.SENSITIVE_ID_FIELD, + settings.score_weight_sensitive_id_field, + ) + + if has_financial_fields: + _add_signal( + result, + ContentSignal.FINANCIAL_FIELD, + settings.score_weight_financial_field, + ) + + if features.download_links: + _add_signal( + result, + ContentSignal.RISKY_DOWNLOAD_LINK, + settings.score_weight_risky_download_link, + ) + + # 루어/기관명은 단독 문구만으로 점수를 만들면 정상 안내 페이지 오탐이 커진다. + # 실제 입력 폼이나 위험 다운로드와 결합될 때만 추가 가중한다. + if has_lure_text and (has_pii_fields or has_financial_fields or features.download_links): + result.score += settings.score_weight_korean_lure + + if has_public_agency_lure and has_pii_fields: + result.score += settings.score_weight_public_agency_lure + + alt_brands: set[str] = set() + for alt in features.image_alts: + alt_brands |= _brands_in_text(alt) + if alt_brands and own_label not in alt_brands: + result.logo_alt_impersonation = True + result.signals.append(ContentSignal.LOGO_ALT_IMPERSONATION) + result.score += settings.score_weight_logo_alt_impersonation + + if features.has_meta_refresh: + result.signals.append(ContentSignal.META_REFRESH) + result.score += settings.score_weight_meta_refresh + if features.has_external_meta_refresh: + result.signals.append(ContentSignal.EXTERNAL_META_REFRESH) + result.score += settings.score_weight_external_meta_refresh + + if ( + features.external_link_ratio is not None + and features.external_link_ratio >= settings.content_external_link_ratio_threshold + ): + result.signals.append(ContentSignal.EXTERNAL_LINK_OVERUSE) + result.score += settings.score_weight_external_link_overuse + + result.score = min(result.score, settings.content_analysis_score_cap) + return result diff --git a/app/services/db_independent_pipeline.py b/app/services/db_independent_pipeline.py new file mode 100644 index 0000000..09fef3e --- /dev/null +++ b/app/services/db_independent_pipeline.py @@ -0,0 +1,387 @@ +"""외부 threat DB 비의존 URL 분석 파이프라인.""" + +from __future__ import annotations + +import time + +import structlog + +from app.core.config import settings +from app.core.exceptions import NormalizationError +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineFailure, + DbIndependentPipelineStages, + DbIndependentPipelineSuccess, +) +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.schemas.pipeline import ( + PipelineStage, + PipelineStageTimings, + PipelineTimings, + Verdict, +) +from app.schemas.unchain import UnchainResult +from app.services.content_analyzer import analyze_content +from app.services.domain_heuristic import check_domain_heuristic +from app.services.normalizer import normalize_url +from app.services.page_unavailability import ( + PAGE_UNAVAILABLE_CODE, + content_page_unavailable, + unchain_page_unavailable, +) +from app.services.pipeline_deadline import ( + PipelineDeadline, + PipelineStageTimeoutError, + timed_out_content_result, + timed_out_domain_result, + timed_out_unchain_result, +) +from app.services.unchainer import unchain_url + +logger = structlog.get_logger(__name__) + + +def _elapsed_seconds(started: float) -> float: + return round(time.perf_counter() - started, 6) + + +def _set_stage_timing( + timings: PipelineStageTimings, + stage: PipelineStage, + started: float, +) -> None: + setattr(timings, stage.value, _elapsed_seconds(started)) + + +def _build_timings(started: float, stage_timings: PipelineStageTimings) -> PipelineTimings: + return PipelineTimings( + total_seconds=_elapsed_seconds(started), + stages=stage_timings, + ) + + +def _decide_verdict(score: int) -> Verdict: + if score >= settings.score_danger_threshold: + return Verdict.DANGER + if score >= settings.score_caution_threshold: + return Verdict.CAUTION + return Verdict.SAFE + + +def _total_score( + heuristic: DomainHeuristicResult, + content: ContentAnalysisResult, +) -> int: + return min(heuristic.score + content.score, settings.score_total_cap) + + +def _redirect_signal_code(raw_signal: str) -> str | None: + if raw_signal.startswith("cross_origin:"): + return DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN.value + if raw_signal == "scheme_downgrade": + return "REDIRECT_SCHEME_DOWNGRADE" + if raw_signal == "redirect_loop": + return "REDIRECT_LOOP" + if raw_signal == "max_hops_reached": + return "REDIRECT_MAX_HOPS_REACHED" + if raw_signal.startswith("unsafe_scheme:"): + return "REDIRECT_UNSAFE_SCHEME" + if raw_signal == "ssrf_blocked": + return "REDIRECT_SSRF_BLOCKED" + return None + + +def _augment_heuristic_with_redirect_signals( + heuristic: DomainHeuristicResult, + unchain: UnchainResult, +) -> DomainHeuristicResult: + signals = list(heuristic.signals) + score = heuristic.score + if ( + any(signal.startswith("cross_origin:") for signal in unchain.signals) + and DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN not in signals + ): + signals.append(DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN) + score = min( + score + settings.score_weight_redirect_cross_origin, + settings.domain_heuristic_score_cap, + ) + if signals == heuristic.signals and score == heuristic.score: + return heuristic + return heuristic.model_copy(update={"signals": signals, "score": score}) + + +def _page_unavailable_content( + final_url: str, + *, + message: str, + status_code: int | None, +) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=status_code, + score=0, + signals=[ContentSignal.FETCH_FAILED], + reason=message, + error="page_unavailable", + ) + + +def _page_unavailable_failure( + *, + analysis_id: str, + original_url: str, + final_url: str, + failed_at_stage: PipelineStage, + error: str, + status_code: int | None, + started: float, + stage_timings: PipelineStageTimings, +) -> DbIndependentPipelineFailure: + return DbIndependentPipelineFailure( + analysis_id=analysis_id, + original_url=original_url, + final_url=final_url, + failed_at_stage=failed_at_stage, + error=error, + error_code=PAGE_UNAVAILABLE_CODE, + status_code=status_code, + timings=_build_timings(started, stage_timings), + ) + + +def _collect_db_independent_signals( + heuristic: DomainHeuristicResult, + unchain: UnchainResult, +) -> tuple[str, ...]: + codes: list[str] = [signal.value for signal in heuristic.signals] + for raw_signal in unchain.signals: + code = _redirect_signal_code(raw_signal) + if code is not None and code not in codes: + codes.append(code) + return tuple(codes) + + +async def run_db_independent_pipeline( + analysis_id: str, + original_url: str, +) -> DbIndependentPipelineSuccess | DbIndependentPipelineFailure: + """GSB/URLhaus 조회 없이 URL·리다이렉트·도메인 신호로 판정한다.""" + log = logger.bind(analysis_id=analysis_id, pipeline="db_independent") + log.info("db_independent_pipeline.start", url=original_url) + total_started = time.perf_counter() + deadline = PipelineDeadline() + stage_timings = PipelineStageTimings() + + stage_started = time.perf_counter() + try: + normalize = normalize_url(original_url) + except NormalizationError as exc: + _set_stage_timing(stage_timings, PipelineStage.NORMALIZE, stage_started) + log.warning("db_independent_pipeline.failed", stage=PipelineStage.NORMALIZE, error=str(exc)) + return DbIndependentPipelineFailure( + analysis_id=analysis_id, + original_url=original_url, + failed_at_stage=PipelineStage.NORMALIZE, + error=str(exc), + timings=_build_timings(total_started, stage_timings), + ) + _set_stage_timing(stage_timings, PipelineStage.NORMALIZE, stage_started) + + stage_started = time.perf_counter() + try: + unchain = await deadline.run( + PipelineStage.UNCHAIN.value, + unchain_url( + normalize.normalized_url, + prefer_https_when_schemeless=normalize.scheme_was_added, + ), + settings.pipeline_unchain_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning("db_independent_pipeline.stage_timeout", stage=PipelineStage.UNCHAIN) + unchain = timed_out_unchain_result(normalize.normalized_url) + except Exception as exc: + log.warning( + "db_independent_pipeline.stage_error", + stage=PipelineStage.UNCHAIN, + error=str(exc), + error_type=type(exc).__name__, + ) + unchain = timed_out_unchain_result(normalize.normalized_url, error="stage_error") + _set_stage_timing(stage_timings, PipelineStage.UNCHAIN, stage_started) + + if unavailable := unchain_page_unavailable(unchain): + message, status_code = unavailable + log.info( + "db_independent_pipeline.page_unavailable", + stage=PipelineStage.UNCHAIN, + final_url=unchain.final_url, + status_code=status_code, + error=unchain.error, + ) + stage_started = time.perf_counter() + try: + heuristic = await deadline.run( + PipelineStage.DOMAIN_HEURISTIC.value, + check_domain_heuristic(unchain.final_url), + settings.pipeline_domain_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning( + "db_independent_pipeline.stage_timeout", + stage=PipelineStage.DOMAIN_HEURISTIC, + ) + heuristic = timed_out_domain_result(unchain.final_url) + except Exception as exc: + log.warning( + "db_independent_pipeline.stage_error", + stage=PipelineStage.DOMAIN_HEURISTIC, + error=str(exc), + error_type=type(exc).__name__, + ) + heuristic = timed_out_domain_result(unchain.final_url) + _set_stage_timing(stage_timings, PipelineStage.DOMAIN_HEURISTIC, stage_started) + heuristic = _augment_heuristic_with_redirect_signals(heuristic, unchain) + content = _page_unavailable_content( + unchain.final_url, + message=message, + status_code=status_code, + ) + score = _total_score(heuristic, content) + if score >= settings.score_caution_threshold: + verdict = _decide_verdict(score) + return DbIndependentPipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=DbIndependentPipelineStages( + normalize=normalize, + unchain=unchain, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) + return _page_unavailable_failure( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + failed_at_stage=PipelineStage.UNCHAIN, + error=message, + status_code=status_code, + started=total_started, + stage_timings=stage_timings, + ) + + stage_started = time.perf_counter() + try: + heuristic = await deadline.run( + PipelineStage.DOMAIN_HEURISTIC.value, + check_domain_heuristic(unchain.final_url), + settings.pipeline_domain_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning( + "db_independent_pipeline.stage_timeout", + stage=PipelineStage.DOMAIN_HEURISTIC, + ) + heuristic = timed_out_domain_result(unchain.final_url) + except Exception as exc: + log.warning( + "db_independent_pipeline.stage_error", + stage=PipelineStage.DOMAIN_HEURISTIC, + error=str(exc), + error_type=type(exc).__name__, + ) + heuristic = timed_out_domain_result(unchain.final_url) + _set_stage_timing(stage_timings, PipelineStage.DOMAIN_HEURISTIC, stage_started) + heuristic = _augment_heuristic_with_redirect_signals(heuristic, unchain) + + upstream = _collect_db_independent_signals(heuristic, unchain) + stage_started = time.perf_counter() + try: + content = await deadline.run( + PipelineStage.CONTENT_ANALYSIS.value, + analyze_content(unchain.final_url, upstream_signals=upstream), + settings.pipeline_content_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning( + "db_independent_pipeline.stage_timeout", + stage=PipelineStage.CONTENT_ANALYSIS, + ) + content = timed_out_content_result(unchain.final_url) + except Exception as exc: + log.warning( + "db_independent_pipeline.stage_error", + stage=PipelineStage.CONTENT_ANALYSIS, + error=str(exc), + error_type=type(exc).__name__, + ) + content = timed_out_content_result(unchain.final_url) + _set_stage_timing(stage_timings, PipelineStage.CONTENT_ANALYSIS, stage_started) + + if unavailable := content_page_unavailable(content): + message, status_code = unavailable + log.info( + "db_independent_pipeline.page_unavailable", + stage=PipelineStage.CONTENT_ANALYSIS, + final_url=unchain.final_url, + status_code=status_code, + error=content.error, + ) + score = _total_score(heuristic, content) + if score >= settings.score_caution_threshold: + verdict = _decide_verdict(score) + return DbIndependentPipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=DbIndependentPipelineStages( + normalize=normalize, + unchain=unchain, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) + return _page_unavailable_failure( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + failed_at_stage=PipelineStage.CONTENT_ANALYSIS, + error=message, + status_code=status_code, + started=total_started, + stage_timings=stage_timings, + ) + + score = _total_score(heuristic, content) + verdict = _decide_verdict(score) + log.info( + "db_independent_pipeline.done", + final_url=unchain.final_url, + verdict=verdict.value, + score=score, + ) + return DbIndependentPipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=DbIndependentPipelineStages( + normalize=normalize, + unchain=unchain, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) diff --git a/app/services/domain_heuristic/__init__.py b/app/services/domain_heuristic/__init__.py new file mode 100644 index 0000000..e4821f9 --- /dev/null +++ b/app/services/domain_heuristic/__init__.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Any + +__all__ = ["check_domain_heuristic"] + + +def __getattr__(name: str) -> Any: + if name == "check_domain_heuristic": + from app.services.domain_heuristic.check import check_domain_heuristic + + return check_domain_heuristic + raise AttributeError(name) diff --git a/app/services/domain_heuristic/brands.txt b/app/services/domain_heuristic/brands.txt new file mode 100644 index 0000000..3e19252 --- /dev/null +++ b/app/services/domain_heuristic/brands.txt @@ -0,0 +1,612 @@ +# brands.txt — 피싱/타이포스쿼팅 탐지용 브랜드 도메인 목록 +# 형식: 한 줄에 도메인 하나, # 주석 허용 + +# ───────────────────────────────────────── +# 국내 포털 / SNS +# ───────────────────────────────────────── +naver.com +navercloud.com +navercorp.com +daum.net +daum.com +nate.com +kakao.com +kakaotalk.com +kakaocorp.com +kakaoenterprise.com +band.us +band.naver.com +cyworld.com +me2day.net +blog.naver.com +cafe.naver.com +map.naver.com +shopping.naver.com +news.naver.com +finance.naver.com +webtoon.naver.com + +# ───────────────────────────────────────── +# 국내 금융 — 은행 +# ───────────────────────────────────────── +kbstar.com +kbbank.com +kb.co.kr +shinhan.com +shinhanbank.com +wooribank.com +woori.com +ibk.co.kr +ibkbank.com +hanabank.com +hana.com +bnk.co.kr +nonghyup.com +nh.com +nhbank.com +kebhana.com +saemaeulgeumgo.or.kr +busan.bank +daegubank.com +gwangjubank.com +jeonbukbank.com +jejubank.com +scbank.co.kr +citybank.co.kr +citi.co.kr +kakaobank.com +kbank.co.kr +tossbank.com +suhyup.com +nfcf.co.kr + +# ───────────────────────────────────────── +# 국내 금융 — 카드 / 간편결제 / 핀테크 +# ───────────────────────────────────────── +kakaopay.com +toss.im +tossbank.com +payco.com +samsungcard.com +lottecard.com +hyundaicard.com +bccard.com +shinhancard.com +wooricard.com +kbcard.com +nonghyupcard.com +hanacard.com +lottefinance.com +ssgincome.com +zeropay.or.kr +naver.pay +ssgpay.com +lpay.com +chai.finance +ypay.co.kr + +# ───────────────────────────────────────── +# 국내 금융 — 증권 / 자산운용 / 보험 +# ───────────────────────────────────────── +kiwoom.com +miraeasset.com +samsungfund.com +hdsecurities.com +nhqv.com +daishin.com +hi-investment.com +eugenefn.com +sk.securities +dbfinancial.com +koreinvestment.com +meritz.co.kr +samsung.life +hanwhalife.com +kyobolife.com +linakorea.com +axa.co.kr +db.insurance +hyundai.insurance +lotte.insurance + +# ───────────────────────────────────────── +# 국내 증권 / 암호화폐 거래소 +# ───────────────────────────────────────── +upbit.com +bithumb.com +korbit.co.kr +coinone.co.kr +gopax.co.kr +huobi.co.kr +coinplug.com +bitsum.com +probit.com +flata.exchange +coredax.com + +# ───────────────────────────────────────── +# 국내 이커머스 +# ───────────────────────────────────────── +coupang.com +coupangplay.com +gmarket.co.kr +auction.co.kr +11st.co.kr +interpark.com +tmon.co.kr +wemakeprice.com +ssg.com +lotteon.com +lotteshopping.com +kurly.com +marketkurly.com +musinsa.com +zigzag.kr +brandi.co.kr +ably.kr +29cm.co.kr +hm.com +zara.com +shein.com +aboutyou.com +fashionplate.co.kr +nsmall.com +hyundaishopping.com +hmall.com +emart.com +emarttraders.com +lottemart.com +homeplus.co.kr +costco.co.kr +iherb.com +elevenstreet.co.kr + +# ───────────────────────────────────────── +# 국내 배달 / O2O / 라이프스타일 +# ───────────────────────────────────────── +baemin.com +yogiyo.co.kr +coupangeats.com +shuttledelivery.com +naver.place +kakaomap.com +kakaonaviapp.com +kakaomobility.com +socar.kr +greencar.com +lottaxi.com +kakao.taxi +ddangyo.com +yanolja.com +goodchoice.kr +airbnb.co.kr +syranos.com +myrealtrip.com +naver.hotel + +# ───────────────────────────────────────── +# 국내 통신 +# ───────────────────────────────────────── +kt.com +olleh.com +tworld.co.kr +skt.com +sktelecom.com +lguplus.com +uplus.co.kr +lg.com +lg.co.kr +skbroadband.com +ktcs.co.kr +ktwiz.co.kr +mvno.skt.com +mvno.kt.com + +# ───────────────────────────────────────── +# 국내 공공 / 정부 +# ───────────────────────────────────────── +gov.kr +nhis.or.kr +nps.or.kr +hometax.go.kr +nts.go.kr +fss.or.kr +kisa.or.kr +kcb.co.kr +nice.co.kr +mois.go.kr +mohw.go.kr +moef.go.kr +moj.go.kr +cdc.go.kr +police.go.kr +customs.go.kr +epts.go.kr +eps.go.kr +work.go.kr +koreajobworld.com +epeople.go.kr +wetax.go.kr +irs.go.kr +judicial.go.kr +scourt.go.kr +patent.go.kr +kipo.go.kr +mfds.go.kr +mida.re.kr +kofia.or.kr +krx.co.kr + +# ───────────────────────────────────────── +# 국내 미디어 / 콘텐츠 +# ───────────────────────────────────────── +webtoon.com +kakaopage.com +ridibooks.com +yes24.com +watcha.com +wavve.com +tving.com +melon.com +bugs.co.kr +flocloud.kr +genie.co.kr +vibe.naver.com +chosun.com +joongang.co.kr +donga.com +yonhapnews.co.kr +mbc.co.kr +kbs.co.kr +sbs.co.kr +jtbc.co.kr +tvn.com +ocn.com +ytn.co.kr +khan.co.kr +hani.co.kr +ohmynews.com +pressian.com +mediaus.co.kr +newsis.com +newspim.com +edaily.co.kr +mt.co.kr +ajunews.com +yna.co.kr + +# ───────────────────────────────────────── +# 국내 IT / 스타트업 +# ───────────────────────────────────────── +krafton.com +ncsoft.com +nexon.com +netmarble.com +smilegate.com +com2us.com +kakao.games +kakao.corp +line.me +naverwebtoon.com +ncloud.com +gabia.com +cafe24.com +hosting.kr +iwinv.com +whois.co.kr +godaddy.co.kr + +# ───────────────────────────────────────── +# 글로벌 빅테크 +# ───────────────────────────────────────── +google.com +gmail.com +youtube.com +google.co.kr +google.co.jp +google.co.uk +google.de +google.fr +google.com.au +googleads.com +googleanalytics.com +googlepay.com +apple.com +icloud.com +appleid.apple.com +itunes.com +appstore.com +microsoft.com +outlook.com +hotmail.com +live.com +office.com +office365.com +teams.microsoft.com +azure.com +windowsazure.com +microsoftonline.com +onedrive.com +skype.com +xbox.com +amazon.com +aws.amazon.com +amazonpay.com +primevideo.com +audible.com +twitch.tv +meta.com +facebook.com +instagram.com +whatsapp.com +messenger.com +threads.net +twitter.com +x.com +linkedin.com +pinterest.com +snapchat.com +tiktok.com +reddit.com +tumblr.com +quora.com +medium.com +substack.com + +# ───────────────────────────────────────── +# 글로벌 금융 +# ───────────────────────────────────────── +paypal.com +stripe.com +wise.com +revolut.com +mastercard.com +visa.com +americanexpress.com +amex.com +citibank.com +bankofamerica.com +chase.com +wellsfargo.com +hsbc.com +hsbc.co.uk +barclays.com +lloydsbank.com +natwest.com +santander.com +ubs.com +credit-suisse.com +deutschebank.com +bnpparibas.com +societegenale.com +inggroup.com +ing.com +commerzbank.com +rabobank.com +abnamro.com +coinbase.com +binance.com +kraken.com +bitfinex.com +gemini.com +ftx.com +crypto.com +bybit.com +okx.com +kucoin.com + +# ───────────────────────────────────────── +# 글로벌 이커머스 / 소매 +# ───────────────────────────────────────── +ebay.com +etsy.com +shopify.com +aliexpress.com +alibaba.com +walmart.com +target.com +bestbuy.com +homedepot.com +costco.com +wayfair.com +overstock.com +newegg.com +rakuten.com +lazada.com +shopee.com +tokopedia.com +bukalapak.com +flipkart.com +jd.com +taobao.com +tmall.com +pinduoduo.com + +# ───────────────────────────────────────── +# 글로벌 클라우드 / DevTools / 협업 +# ───────────────────────────────────────── +github.com +gitlab.com +bitbucket.org +stackoverflow.com +python.org +postgresql.org +typescriptlang.org +rust-lang.org +docker.com +dockerhub.com +cloudflare.com +digitalocean.com +notion.so +notion.com +atlassian.com +jira.atlassian.com +confluence.atlassian.com +slack.com +discord.com +zoom.us +figma.com +miro.com +trello.com +asana.com +monday.com +airtable.com +clickup.com +linear.app +vercel.com +netlify.com +heroku.com +render.com +railway.app +supabase.com +firebase.google.com +amplify.aws +terraform.io +ansible.com +kubernetes.io +helm.sh +grafana.com +datadog.com +newrelic.com +splunk.com +elastic.co +kibana.com +prometheus.io +circleci.com +travisci.com +jenkins.io +githubactions.com +argocd.io + +# ───────────────────────────────────────── +# 글로벌 보안 +# ───────────────────────────────────────── +auth0.com +okta.com +lastpass.com +1password.com +bitwarden.com +norton.com +mcafee.com +kaspersky.com +malwarebytes.com +crowdstrike.com +paloaltonetworks.com +sentinelone.com +darktrace.com +fortinet.com +checkpoint.com +sophos.com +trendmicro.com +avast.com +avg.com +eset.com +bitdefender.com +cylance.com +qualys.com +rapid7.com +tenable.com + +# ───────────────────────────────────────── +# 글로벌 여행 / 숙박 +# ───────────────────────────────────────── +booking.com +airbnb.com +expedia.com +tripadvisor.com +hotels.com +agoda.com +kayak.com +skyscanner.com +momondo.com +trivago.com +vrbo.com +marriott.com +hilton.com +hyatt.com +ihg.com +bestwestern.com +radissonhotels.com +accor.com + +# ───────────────────────────────────────── +# 글로벌 게임 +# ───────────────────────────────────────── +steam.com +steampowered.com +epicgames.com +roblox.com +ea.com +origin.com +ubisoft.com +blizzard.com +battlenet.com +playstation.com +xbox.com +nintendo.com +itch.io +gog.com +humblebundle.com +razer.com + +# ───────────────────────────────────────── +# 글로벌 미디어 / 스트리밍 / SaaS +# ───────────────────────────────────────── +netflix.com +spotify.com +ubuntu.com +apnews.com +ethz.ch +dropbox.com +wordpress.com +wix.com +godaddy.com +twilio.com +sendgrid.com +salesforce.com +zendesk.com +hubspot.com +mailchimp.com +constantcontact.com +marketo.com +pardot.com +servicenow.com +workday.com +sap.com +oracle.com +adobe.com +canva.com +squarespace.com +weebly.com +shopify.ca +shopify.co.uk +bigcommerce.com +magento.com +prestashop.com +opencart.com +woocommerce.com +hulu.com +disneyplus.com +hbomax.com +peacocktv.com +paramountplus.com +appletv.com +crunchyroll.com +funimation.com +vimeo.com +dailymotion.com +twitch.com +youtube.tv +soundcloud.com +tidal.com +deezer.com +pandora.com +iheartradio.com +audible.com +kindle.com +scribd.com +coursera.com +udemy.com +edx.org +khanacademy.org +duolingo.com diff --git a/app/services/domain_heuristic/check.py b/app/services/domain_heuristic/check.py new file mode 100644 index 0000000..ffb7785 --- /dev/null +++ b/app/services/domain_heuristic/check.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import asyncio +from functools import lru_cache +from urllib.parse import urlparse + +from app.core.config import settings +from app.core.logging import get_logger +from app.core.tld import extract_url_parts +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.services.domain_heuristic.dga import check_dga +from app.services.domain_heuristic.patterns import check_patterns +from app.services.domain_heuristic.rdap import lookup_rdap +from app.services.domain_heuristic.typosquatting import check_typosquatting + +logger = get_logger(__name__) + + +@lru_cache(maxsize=1) +def _signal_scores() -> dict[DomainHeuristicSignal, int]: + """시그널 → 점수 매핑을 lazy 하게 1회만 빌드. + + 예전엔 모듈 import 시점에 settings 값을 dict 로 캡처했는데, 그러면 테스트의 + monkeypatch 와 운영의 env 갱신이 다른 모듈(signals.py 등 lazy 경로)과 + 비대칭적으로 반영됐다. 호출 단계에서 캐시된 dict 를 받도록 옮겨 일관성 확보. + """ + return { + DomainHeuristicSignal.IP_DIRECT: settings.score_weight_ip_direct, + DomainHeuristicSignal.TYPO_DOMAIN: settings.score_weight_typo_domain, + DomainHeuristicSignal.PUNYCODE_IDN: settings.score_weight_punycode_idn, + DomainHeuristicSignal.NEW_DOMAIN: settings.score_weight_new_domain, + DomainHeuristicSignal.SUBDOMAIN_OVERUSE: settings.score_weight_subdomain_overuse, + DomainHeuristicSignal.NO_HTTPS: settings.score_weight_no_https, + DomainHeuristicSignal.OPEN_REDIRECT_PARAM: settings.score_weight_open_redirect_param, + DomainHeuristicSignal.HYPHEN_OVERUSE: settings.score_weight_hyphen_overuse, + DomainHeuristicSignal.SUSPICIOUS_TLD: settings.score_weight_suspicious_tld, + DomainHeuristicSignal.DGA_LIKE: settings.score_weight_dga_like, + DomainHeuristicSignal.HOSTING_PLATFORM: settings.score_weight_hosting_platform, + DomainHeuristicSignal.URL_USERINFO: settings.score_weight_url_userinfo, + DomainHeuristicSignal.BRAND_IN_URL: settings.score_weight_brand_in_url, + DomainHeuristicSignal.FREE_HOSTING_LURE: settings.score_weight_free_hosting_lure, + DomainHeuristicSignal.SENSITIVE_PATH: settings.score_weight_sensitive_path, + DomainHeuristicSignal.URL_SHORTENER: settings.score_weight_url_shortener, + DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN: settings.score_weight_redirect_cross_origin, + } + + +async def check_domain_heuristic(url: str) -> DomainHeuristicResult: + ext = extract_url_parts(url) + domain = ext.top_domain_under_public_suffix or (urlparse(url).hostname or "") + + signals: list[DomainHeuristicSignal] = [] + rdap_info = None + rdap_error = None + + try: + pattern_signals = check_patterns(url) + signals.extend(pattern_signals) + except Exception as exc: + logger.warning("domain_heuristic.patterns_error", error=str(exc)) + + # IP 직접 접근이면 도메인 기반 신호 분석 불필요 + if DomainHeuristicSignal.IP_DIRECT not in signals: + try: + dga_signal = check_dga(url) + if dga_signal: + signals.append(dga_signal) + except Exception as exc: + logger.warning("domain_heuristic.dga_error", error=str(exc)) + + # 호스팅 플랫폼(github.io, netlify.app 등)은 suffix가 브랜드의 suffix와 + # 다른 것이 정상 → 타이포스쿼팅 판정에서 제외 + if DomainHeuristicSignal.HOSTING_PLATFORM not in signals: + try: + typo_signal = check_typosquatting(url) + if typo_signal: + signals.append(typo_signal) + except Exception as exc: + logger.warning("domain_heuristic.typo_error", error=str(exc)) + + # RDAP은 보조 신호다. RDAP이 느려져도 앞에서 이미 잡은 URL 패턴/DGA/typo + # 신호를 잃지 않도록 내부 timeout으로 격리한다. + try: + timeout_seconds = min( + settings.rdap_timeout_seconds, + settings.pipeline_domain_timeout_seconds - 0.25, + ) + rdap_info, rdap_error = await asyncio.wait_for( + lookup_rdap(url), + timeout=timeout_seconds, + ) + except TimeoutError: + rdap_info, rdap_error = None, "timeout" + if rdap_info and rdap_info.is_new_domain: + signals.append(DomainHeuristicSignal.NEW_DOMAIN) + + # 휴리스틱은 보조 시그널이라 합산이 GSB/URLhaus를 압도하지 않도록 캡 적용 + score_map = _signal_scores() + raw_score = sum(score_map.get(s, 0) for s in signals) + score = min(raw_score, settings.domain_heuristic_score_cap) + + return DomainHeuristicResult( + domain=domain, + score=score, + signals=signals, + rdap=rdap_info, + rdap_error=rdap_error, + ) diff --git a/app/services/domain_heuristic/dga.py b/app/services/domain_heuristic/dga.py new file mode 100644 index 0000000..1c62334 --- /dev/null +++ b/app/services/domain_heuristic/dga.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import math + +from app.core.config import settings +from app.core.tld import extract_url_parts +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.patterns import is_trusted_registered_domain + +_CONSONANTS = frozenset("bcdfghjklmnpqrstvwxyz") + + +_MIN_DGA_LABEL_LEN = 8 # netflix(7), spotify(7) 같은 짧은 영어 브랜드 오탐 방지 + + +def _shannon_entropy(s: str) -> float: + if not s: + return 0.0 + freq: dict[str, int] = {} + for c in s: + freq[c] = freq.get(c, 0) + 1 + n = len(s) + total = sum((count / n) * math.log2(count / n) for count in freq.values()) + return -total + + +def _consonant_ratio(s: str) -> float: + letters = [c for c in s.lower() if c.isalpha()] + if not letters: + return 0.0 + return sum(1 for c in letters if c in _CONSONANTS) / len(letters) + + +def check_dga(url: str) -> DomainHeuristicSignal | None: + if is_trusted_registered_domain(url): + return None + + ext = extract_url_parts(url) + label = ext.domain # 등록 도메인 레이블 + if not label or len(label) < _MIN_DGA_LABEL_LEN: + # 짧은 레이블은 엔트로피/자음 비율 통계가 불안정 → 스킵 + return None + + entropy = _shannon_entropy(label) + consonant_ratio = _consonant_ratio(label) + + if ( + entropy >= settings.dga_entropy_threshold + or consonant_ratio >= settings.dga_consonant_ratio_threshold + ): + return DomainHeuristicSignal.DGA_LIKE + return None diff --git a/app/services/domain_heuristic/patterns.py b/app/services/domain_heuristic/patterns.py new file mode 100644 index 0000000..1e2e583 --- /dev/null +++ b/app/services/domain_heuristic/patterns.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import ipaddress +import re +from functools import lru_cache +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +from app.core.config import settings +from app.core.tld import extract_url_parts +from app.schemas.domain_heuristic import DomainHeuristicSignal + +_BRANDS_FILE = Path(__file__).parent / "brands.txt" + +_SUSPICIOUS_TLDS = frozenset( + { + "zip", + "mov", + "xyz", + "top", + "pw", + "tk", + "ml", + "ga", + "cf", + "cfd", + "gq", + "help", + "fit", + "work", + "surf", + "click", + "link", + "loan", + "download", + "stream", + "racing", + "review", + "trade", + "win", + "bid", + "party", + "date", + "faith", + "science", + "cricket", + "accountant", + "men", + } +) + +HOSTING_PLATFORMS = frozenset( + { + "netlify.app", + "vercel.app", + "pages.dev", + "github.io", + "gitlab.io", + "web.app", + "firebaseapp.com", + "amazonaws.com", + "azurewebsites.net", + "herokuapp.com", + "glitch.me", + "replit.app", + "surge.sh", + "render.com", + "fly.dev", + "railway.app", + "cyclic.app", + "workers.dev", + "wasmer.app", + "godaddysites.com", + "duckdns.org", + "free.nf", + "my.id", + "backblazeb2.com", + } +) + +_URL_SHORTENERS = frozenset( + { + "bit.ly", + "cutt.ly", + "goo.gl", + "is.gd", + "ow.ly", + "qrco.de", + "rebrand.ly", + "shorturl.at", + "t.co", + "tinyurl.com", + } +) + +_TRUSTED_BENIGN_DOMAINS = frozenset( + { + "example.com", + "httpbin.org", + } +) + +_SENSITIVE_PATH_TOKENS = frozenset( + { + "account", + "admin", + "auth", + "billing", + "claim", + "confirm", + "credential", + "invoice", + "login", + "password", + "pay", + "portal", + "recover", + "secure", + "session", + "signin", + "support", + "suporte", + "validate", + "verification", + "verify", + "wallet", + } +) + +_SEEDED_BRAND_LABELS = frozenset( + { + "adobe", + "amazon", + "apple", + "binance", + "coinbase", + "discord", + "dpd", + "facebook", + "google", + "instagram", + "iphone", + "metamask", + "microsoft", + "netflix", + "office", + "outlook", + "paypal", + "roblox", + "societegenerale", + "telegram", + "tmobile", + "t-mobile", + "trezor", + "whatsapp", + "wps", + "yahoo", + } +) + +_GENERIC_BRAND_LABELS = frozenset( + { + "bank", + "blog", + "cafe", + "card", + "cloud", + "finance", + "gov", + "home", + "map", + "mvno", + "news", + "pay", + "place", + "shop", + "shopping", + "support", + } +) + +_OPEN_REDIRECT_PARAMS = frozenset( + { + "url", + "redirect", + "redirect_url", + "redirect_uri", + "next", + "return", + "return_url", + "returnurl", + "goto", + "destination", + "target", + "link", + "forward", + "continue", + "redir", + "out", + } +) + + +def _is_ip_address(host: str) -> bool: + # IPv6는 urlparse가 [] 제거하지 않으므로 strip 필요 + host = host.strip("[]") + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +@lru_cache(maxsize=1) +def _brand_labels() -> frozenset[str]: + labels = set(_SEEDED_BRAND_LABELS) + if not _BRANDS_FILE.exists(): + return frozenset(labels) + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + label = line.split(".", 1)[0].lower() + if len(label) >= 3 and label not in _GENERIC_BRAND_LABELS: + labels.add(label) + return frozenset(labels) + + +@lru_cache(maxsize=1) +def _trusted_brand_domains() -> frozenset[str]: + domains: set[str] = set() + if not _BRANDS_FILE.exists(): + return frozenset(domains) + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + domains.add(line.lower()) + return frozenset(domains) + + +def is_trusted_registered_domain(url: str) -> bool: + ext = extract_url_parts(url) + registered_domain = (ext.top_domain_under_public_suffix or "").lower() + return bool( + registered_domain + and ( + registered_domain in _trusted_brand_domains() + or registered_domain in _TRUSTED_BENIGN_DOMAINS + ) + ) + + +def _brand_labels_in_url_text(text: str) -> set[str]: + lowered = text.lower() + tokens = re.findall(r"[a-z0-9]+", lowered) + found: set[str] = set() + for label in _brand_labels(): + if len(label) < 3: + continue + if label in lowered or label in tokens: + found.add(label) + return found + + +def check_patterns(url: str) -> list[DomainHeuristicSignal]: + signals: list[DomainHeuristicSignal] = [] + parsed = urlparse(url) + host = parsed.hostname or "" + + if _is_ip_address(host): + signals.append(DomainHeuristicSignal.IP_DIRECT) + return signals # IP면 도메인 기반 신호 분석 불필요 + + if parsed.scheme == "http": + signals.append(DomainHeuristicSignal.NO_HTTPS) + + ext = extract_url_parts(url) + registered_domain = (ext.top_domain_under_public_suffix or host).lower() + registered_label = ext.domain.lower() + is_trusted_brand_domain = registered_domain in _trusted_brand_domains() + + if parsed.username or parsed.password or "@" in parsed.netloc: + signals.append(DomainHeuristicSignal.URL_USERINFO) + + # 서브도메인 레이블 과다 + subdomain_labels = [s for s in ext.subdomain.split(".") if s] + if len(subdomain_labels) >= settings.subdomain_label_threshold: + signals.append(DomainHeuristicSignal.SUBDOMAIN_OVERUSE) + + # Punycode/IDN 호모글리프 + if "xn--" in host: + signals.append(DomainHeuristicSignal.PUNYCODE_IDN) + + # 하이픈 과다 (등록 도메인 레이블 기준) + domain_label = ext.domain + hyphen_count = domain_label.count("-") + if ( + hyphen_count >= settings.hyphen_count_threshold + or len(domain_label) >= settings.domain_label_length_threshold + ): + signals.append(DomainHeuristicSignal.HYPHEN_OVERUSE) + + # 의심 TLD + if ext.suffix in _SUSPICIOUS_TLDS: + signals.append(DomainHeuristicSignal.SUSPICIOUS_TLD) + + # 오픈 리다이렉트 파라미터 + qs = parse_qs(parsed.query, keep_blank_values=True) + if any(k.lower() in _OPEN_REDIRECT_PARAMS for k in qs): + signals.append(DomainHeuristicSignal.OPEN_REDIRECT_PARAM) + + # private suffix(github.io, vercel.app 등)는 tld parser에서 suffix로 잡히므로 + # user.github.io처럼 subdomain이 없어도 플랫폼 사용자 공간으로 본다. + is_hosting_platform = False + if ext.suffix: + domain_suffix = f"{ext.domain}.{ext.suffix}" + if ext.suffix in HOSTING_PLATFORMS or ( + ext.subdomain and domain_suffix in HOSTING_PLATFORMS + ): + signals.append(DomainHeuristicSignal.HOSTING_PLATFORM) + is_hosting_platform = True + + if registered_domain in _URL_SHORTENERS: + signals.append(DomainHeuristicSignal.URL_SHORTENER) + + url_text = " ".join(part for part in (host, parsed.path, parsed.query) if part) + url_tokens = set(re.findall(r"[a-z]+", url_text.lower())) + sensitive_path = bool(url_tokens & _SENSITIVE_PATH_TOKENS) + url_brands = _brand_labels_in_url_text(url_text) + + if sensitive_path and not is_trusted_brand_domain: + signals.append(DomainHeuristicSignal.SENSITIVE_PATH) + + if url_brands and not is_trusted_brand_domain and registered_label not in url_brands: + signals.append(DomainHeuristicSignal.BRAND_IN_URL) + + if is_hosting_platform and (url_brands or sensitive_path): + signals.append(DomainHeuristicSignal.FREE_HOSTING_LURE) + + return signals diff --git a/app/services/domain_heuristic/rdap.py b/app/services/domain_heuristic/rdap.py new file mode 100644 index 0000000..53ed54c --- /dev/null +++ b/app/services/domain_heuristic/rdap.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import asyncio +import time +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime + +import httpx +from cachetools import TTLCache + +from app.core.config import settings +from app.core.logging import get_logger +from app.core.tld import extract_url_parts +from app.schemas.domain_heuristic import RdapInfo + +logger = get_logger(__name__) + +# domain → RdapInfo | None. TTL 만료 + LRU eviction 모두 cachetools 가 관리. +# 24h TTL 동안 무작위 도메인 트래픽으로 dict 가 무한 성장하던 종전 동작을 maxsize 로 천장 박음. +# TTLCache 는 thread-safe 가 아니지만 단일 이벤트 루프에서 동기 접근만 하므로 안전. +_cache: TTLCache[str, RdapInfo | None] = TTLCache( + maxsize=settings.rdap_cache_max_entries, + ttl=settings.rdap_cache_ttl_seconds, +) + +# 동시 요청 합치기용 — 같은 도메인 요청이 겹치면 하나의 Future만 실제로 fetch +_inflight: dict[str, asyncio.Future[tuple[RdapInfo | None, str | None]]] = {} + +# 커넥션/TLS 재사용을 위해 모듈 레벨에서 싱글턴 사용 +_client: httpx.AsyncClient | None = None +_rate_limit_until: float | None = None +_DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS = 30.0 +_MAX_RATE_LIMIT_COOLDOWN_SECONDS = 120.0 + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None: + # RFC 7480 SHOULD: authoritative RDAP 서버로 redirect될 때 일부 구현은 + # Accept 없으면 406 또는 HTML을 반환 → 기본 헤더로 명시 + _client = httpx.AsyncClient( + timeout=settings.rdap_timeout_seconds, + headers={"Accept": "application/rdap+json"}, + ) + return _client + + +async def aclose_client() -> None: + """앱 셧다운 훅에서 호출. 테스트에서 client 상태 초기화에도 사용.""" + global _client + if _client is not None: + await _client.aclose() + _client = None + + +def _extract_registrar(data: dict) -> str | None: # type: ignore[type-arg] + for entity in data.get("entities", []): + roles = entity.get("roles", []) + if "registrar" in roles: + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) > 1: + for entry in vcard[1]: + if isinstance(entry, list) and entry and entry[0] == "fn": + return str(entry[3]) + return None + + +def _parse_datetime(s: str | None) -> datetime | None: + if not s: + return None + try: + # Python 3.11+의 fromisoformat은 Z suffix를 직접 파싱 + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + except ValueError: + return None + + +def _parse_rdap_response(domain: str, data: dict) -> RdapInfo: # type: ignore[type-arg] + events = {e["eventAction"]: e.get("eventDate") for e in data.get("events", [])} + created_date = _parse_datetime(events.get("registration")) + expiry_date = _parse_datetime(events.get("expiration")) + + now = datetime.now(tz=UTC) + domain_age_days: int | None = None + is_new_domain = False + if created_date: + domain_age_days = (now - created_date).days + is_new_domain = domain_age_days < settings.rdap_new_domain_threshold_days + + return RdapInfo( + domain=domain, + registrar=_extract_registrar(data), + created_date=created_date, + expiry_date=expiry_date, + domain_age_days=domain_age_days, + is_new_domain=is_new_domain, + ) + + +def _encode_for_url(domain: str) -> str | None: + """IDN 도메인을 punycode로 변환. ASCII 도메인은 그대로 통과.""" + try: + return domain.encode("idna").decode("ascii") + except UnicodeError: + return None + + +def _retry_after_seconds(value: str | None) -> float: + if not value: + return _DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS + + try: + seconds = float(value) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + except (TypeError, ValueError): + return _DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=UTC) + seconds = (retry_at.astimezone(UTC) - datetime.now(tz=UTC)).total_seconds() + + return min(max(seconds, 0.0), _MAX_RATE_LIMIT_COOLDOWN_SECONDS) + + +def _remember_rate_limit(domain: str, retry_after: str | None) -> None: + global _rate_limit_until + cooldown_seconds = _retry_after_seconds(retry_after) + _rate_limit_until = time.monotonic() + cooldown_seconds + logger.warning( + "rdap.rate_limited", + domain=domain, + retry_after=retry_after, + cooldown_seconds=cooldown_seconds, + ) + + +def _rate_limited_now() -> bool: + return _rate_limit_until is not None and time.monotonic() < _rate_limit_until + + +def _rate_limit_remaining_seconds() -> float: + if _rate_limit_until is None: + return 0.0 + return round(max(_rate_limit_until - time.monotonic(), 0.0), 3) + + +async def _fetch_rdap(domain: str) -> tuple[RdapInfo | None, str | None]: + encoded = _encode_for_url(domain) + if encoded is None: + return None, "invalid_domain" + + rdap_url = f"{settings.rdap_bootstrap_url}{encoded}" + try: + client = _get_client() + resp = await client.get(rdap_url, follow_redirects=True) + resp.raise_for_status() + data = resp.json() + except httpx.TimeoutException: + return None, "timeout" + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None, "not_found" + if exc.response.status_code == 429: + _remember_rate_limit(domain, exc.response.headers.get("retry-after")) + return None, "rate_limited" + return None, "http_error" + except Exception as exc: + logger.warning( + "rdap.unexpected_error", + domain=domain, + error=str(exc), + error_type=type(exc).__name__, + ) + return None, "unexpected" + + try: + result = _parse_rdap_response(domain, data) + except Exception as exc: + logger.warning( + "rdap.parse_error", + domain=domain, + error=str(exc), + error_type=type(exc).__name__, + ) + return None, "parse_error" + + _cache[domain] = result + return result, None + + +async def lookup_rdap(url: str) -> tuple[RdapInfo | None, str | None]: + """(RdapInfo | None, error_code | None) 반환. 에러 시 파이프라인 중단하지 않음.""" + ext = extract_url_parts(url) + domain = ext.top_domain_under_public_suffix + if not domain: + return None, "no_domain" + + if _rate_limited_now(): + logger.info( + "rdap.rate_limit_cooldown_active", + domain=domain, + remaining_seconds=_rate_limit_remaining_seconds(), + ) + return None, "rate_limited" + + # TTLCache 가 만료된 엔트리는 자동으로 KeyError 처리 — 명시적 만료 체크 불필요. + try: + return _cache[domain], None + except KeyError: + pass + + # 같은 도메인 in-flight 요청이 있으면 결과 공유 — RDAP 서버 부하 방지 + existing = _inflight.get(domain) + if existing is not None: + return await existing + + loop = asyncio.get_running_loop() + fut: asyncio.Future[tuple[RdapInfo | None, str | None]] = loop.create_future() + _inflight[domain] = fut + try: + result = await _fetch_rdap(domain) + if not fut.done(): + fut.set_result(result) + return result + except BaseException as exc: + if not fut.done(): + fut.set_exception(exc) + # If the owner task is cancelled before another waiter consumes this future, + # retrieve the exception to avoid noisy "Future exception was never retrieved". + fut.add_done_callback(lambda done: done.exception()) + raise + finally: + _inflight.pop(domain, None) diff --git a/app/services/domain_heuristic/typosquatting.py b/app/services/domain_heuristic/typosquatting.py new file mode 100644 index 0000000..6475d46 --- /dev/null +++ b/app/services/domain_heuristic/typosquatting.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from pathlib import Path + +from app.core.logging import get_logger +from app.core.tld import extract_url_parts +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.patterns import HOSTING_PLATFORMS + +logger = get_logger(__name__) + +_BRANDS_FILE = Path(__file__).parent / "brands.txt" + + +def _load_brands() -> list[tuple[str, str]]: + """brands.txt를 읽어 (domain_label, tld) 튜플 리스트로 반환. 모듈 로드 시 1회 실행.""" + if not _BRANDS_FILE.exists(): + # silent fail은 typosquatting 검사 영구 무력화로 이어져 false negative 폭증 — 명시적 로깅 + logger.error("typosquatting.brands_file_missing", path=str(_BRANDS_FILE)) + return [] + seen: set[tuple[str, str]] = set() + brands: list[tuple[str, str]] = [] + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + ext = extract_url_parts(f"https://{line}") + if ext.domain and ext.suffix: + key = (ext.domain, ext.suffix) + if key not in seen: + seen.add(key) + brands.append(key) + if not brands: + logger.error("typosquatting.brands_empty", path=str(_BRANDS_FILE)) + return brands + + +_BRANDS: list[tuple[str, str]] | None = None + + +def _get_brands() -> list[tuple[str, str]]: + global _BRANDS + if _BRANDS is None: + _BRANDS = _load_brands() + return _BRANDS + + +def _levenshtein(a: str, b: str) -> int: + """레벤슈타인 거리 — 외부 라이브러리 없이 DP로 구현.""" + m, n = len(a), len(b) + dp = list(range(n + 1)) + for i in range(1, m + 1): + prev = dp[0] + dp[0] = i + for j in range(1, n + 1): + temp = dp[j] + if a[i - 1] == b[j - 1]: + dp[j] = prev + else: + dp[j] = 1 + min(prev, dp[j], dp[j - 1]) + prev = temp + return dp[n] + + +_MIN_FUZZY_LEN = 5 # 4자 이하 브랜드는 편집거리 1~2 오탐이 심해 완전일치만 인정 +_SUSPICIOUS_SUFFIXES = frozenset( + { + "bid", + "cf", + "click", + "date", + "faith", + "ga", + "gq", + "link", + "loan", + "men", + "ml", + "mov", + "party", + "pw", + "racing", + "review", + "science", + "stream", + "surf", + "tk", + "top", + "trade", + "win", + "xyz", + "zip", + } +) + + +def check_typosquatting(url: str) -> DomainHeuristicSignal | None: + ext = extract_url_parts(url) + if not ext.top_domain_under_public_suffix: + return None + + target_domain = ext.domain + target_suffix = ext.suffix + + # 호스팅 플랫폼 자체 도메인(netlify.app, vercel.app 등)은 brand 변종(netlify.com)과 + # label 동일 + suffix 상이로 TYPO_DOMAIN 오탐되므로 조기 컷아웃 + if ext.top_domain_under_public_suffix in HOSTING_PLATFORMS: + return None + + is_typo = False + for brand_domain, brand_suffix in _get_brands(): + if target_domain == brand_domain and target_suffix == brand_suffix: + return None # 완전 일치 → 정상 도메인 + + if target_suffix != brand_suffix and target_suffix not in _SUSPICIOUS_SUFFIXES: + continue + + if abs(len(target_domain) - len(brand_domain)) > 2: + continue # 길이 차이가 2 초과면 거리가 반드시 2 초과 → 스킵 + + distance = _levenshtein(target_domain, brand_domain) + if distance == 0 and target_suffix != brand_suffix: + is_typo = True + elif 1 <= distance <= 2 and min(len(target_domain), len(brand_domain)) >= _MIN_FUZZY_LEN: + # "ing" vs "bing", "kb" vs "kbx" 같은 단어가 정상 도메인을 오탐하는 문제 차단 + is_typo = True + + return DomainHeuristicSignal.TYPO_DOMAIN if is_typo else None diff --git a/app/services/evaluation/__init__.py b/app/services/evaluation/__init__.py new file mode 100644 index 0000000..926936f --- /dev/null +++ b/app/services/evaluation/__init__.py @@ -0,0 +1 @@ +"""Evaluation helpers for detector quality measurement.""" diff --git a/app/services/evaluation/metrics.py b/app/services/evaluation/metrics.py new file mode 100644 index 0000000..011be0d --- /dev/null +++ b/app/services/evaluation/metrics.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections.abc import Iterable, Set +from dataclasses import dataclass + +from app.schemas.pipeline import Verdict + + +@dataclass(frozen=True) +class VerdictCase: + expected: Verdict + predicted: Verdict + + +@dataclass(frozen=True) +class VerdictMetrics: + total: int + correct: int + true_positive: int + false_positive: int + true_negative: int + false_negative: int + accuracy: float + positive_precision: float + positive_recall: float + positive_f1: float + false_positive_rate: float + + +def _ratio(numerator: int, denominator: int) -> float: + if denominator == 0: + return 0.0 + return round(numerator / denominator, 6) + + +def compute_verdict_metrics( + cases: Iterable[VerdictCase], + *, + positive_expected: Set[Verdict], + positive_predicted: Set[Verdict], +) -> VerdictMetrics: + rows = list(cases) + total = len(rows) + correct = sum(1 for row in rows if row.expected == row.predicted) + + true_positive = 0 + false_positive = 0 + true_negative = 0 + false_negative = 0 + for row in rows: + expected_positive = row.expected in positive_expected + predicted_positive = row.predicted in positive_predicted + if expected_positive and predicted_positive: + true_positive += 1 + elif not expected_positive and predicted_positive: + false_positive += 1 + elif expected_positive and not predicted_positive: + false_negative += 1 + else: + true_negative += 1 + + precision = _ratio(true_positive, true_positive + false_positive) + recall = _ratio(true_positive, true_positive + false_negative) + f1 = ( + 0.0 + if precision + recall == 0 + else round((2 * precision * recall) / (precision + recall), 6) + ) + + return VerdictMetrics( + total=total, + correct=correct, + true_positive=true_positive, + false_positive=false_positive, + true_negative=true_negative, + false_negative=false_negative, + accuracy=_ratio(correct, total), + positive_precision=precision, + positive_recall=recall, + positive_f1=f1, + false_positive_rate=_ratio(false_positive, false_positive + true_negative), + ) diff --git a/app/services/normalizer/__init__.py b/app/services/normalizer/__init__.py new file mode 100644 index 0000000..d975755 --- /dev/null +++ b/app/services/normalizer/__init__.py @@ -0,0 +1,3 @@ +from app.services.normalizer.normalize import normalize_url + +__all__ = ["normalize_url"] diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py new file mode 100644 index 0000000..a65bbd4 --- /dev/null +++ b/app/services/normalizer/normalize.py @@ -0,0 +1,136 @@ +"""URL 정규화 — 파이프라인 1단계. + +위협 DB 조회, 휴리스틱, 콘텐츠 분석 등 후속 단계에서 +URL을 일관되게 비교할 수 있도록 정규 형태로 변환함. +""" + +from __future__ import annotations + +import re +import unicodedata +from urllib.parse import urlparse, urlunparse + +from app.core.config import settings +from app.core.exceptions import NormalizationError +from app.schemas.normalize import NormalizeResult + +_DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443, "ftp": 21} +_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https", "ftp"}) + +_UNRESERVED: frozenset[str] = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) + +_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") +_PCT_ENCODED_RE = re.compile(r"%([0-9A-Fa-f]{2})") + + +def normalize_url(raw_url: str) -> NormalizeResult: + """정규화된 NormalizeResult 반환. 실패 시 NormalizationError.""" + original = raw_url.strip() + if not original: + raise NormalizationError(message="빈 URL입니다.") + + cleaned = _CONTROL_CHAR_RE.sub("", original) + cleaned = unicodedata.normalize("NFC", cleaned) + + # 길이 체크 전에 스킴 확정해야 함 — 스킴 붙이면 바이트 늘어남. + # 단순 "://" 체크는 "example.com/path://weird" 같은 케이스 놓침. + scheme_was_added = not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", cleaned) + if scheme_was_added: + cleaned = "http://" + cleaned + + max_len = settings.normalizer_max_url_length + if len(cleaned) > max_len: + raise NormalizationError( + message=f"URL이 최대 길이({max_len}자)를 초과합니다.", + ) + + try: + parsed = urlparse(cleaned) + except ValueError as e: + raise NormalizationError(message=f"URL 파싱 실패: {e}") from e + + scheme = parsed.scheme.lower() + if scheme not in _ALLOWED_SCHEMES: + raise NormalizationError(message=f"지원하지 않는 스킴: {scheme}") + + hostname = parsed.hostname or "" + if ":" in hostname: + raise NormalizationError(message="IPv6 주소는 지원하지 않습니다.") + hostname = _normalize_idn(hostname.lower()) + if not hostname: + raise NormalizationError(message="호스트가 비어있습니다.") + + # parsed.hostname으로 netloc 재조립하면 userinfo 자동 제거됨. + # 피싱에 악용되는 패턴임 (google.com@evil.com). + try: + port = parsed.port + except ValueError as e: + raise NormalizationError(message=f"유효하지 않은 포트: {e}") from e + if port and port == _DEFAULT_PORTS.get(scheme): + port = None + netloc = hostname if port is None else f"{hostname}:{port}" + + path = _normalize_path(parsed.path) + params = _normalize_pct_encoding(parsed.params) + query = _normalize_pct_encoding(parsed.query) + + normalized = urlunparse((scheme, netloc, path, params, query, "")) + return NormalizeResult( + original_url=original, + normalized_url=normalized, + scheme_was_added=scheme_was_added, + ) + + +def _normalize_idn(hostname: str) -> str: + """유니코드 호스트명을 punycode로 변환. 위협 DB 매칭 호환용.""" + try: + return hostname.encode("idna").decode("ascii") + except UnicodeError: + return hostname + + +def _normalize_path(raw_path: str) -> str: + path = raw_path or "/" + path = _normalize_pct_encoding(path) + + # RFC상 //a와 /a는 다른 리소스지만, 공격자가 슬래시 추가해서 + # 위협 DB 패턴 매칭 우회하는 케이스 방지. + path = re.sub(r"/{2,}", "/", path) + + path = _resolve_dot_segments(path) + return path or "/" + + +def _resolve_dot_segments(path: str) -> str: + """dot segment(. / ..) 해소.""" + segments = path.split("/") + resolved: list[str] = [] + for seg in segments: + if seg == ".": + continue + if seg == "..": + if resolved and resolved[-1] != "": + resolved.pop() + else: + resolved.append(seg) + if path.startswith("/") and (not resolved or resolved[0] != ""): + resolved.insert(0, "") + return "/".join(resolved) + + +def _normalize_pct_encoding(value: str) -> str: + """비예약 문자 디코딩(%41->A), 예약 문자 hex 대문자화(%2f->%2F).""" + if not value: + return value + + def _replace(match: re.Match[str]) -> str: + hex_str = match.group(1) + char = chr(int(hex_str, 16)) + if char in _UNRESERVED: + return char + return f"%{hex_str.upper()}" + + return _PCT_ENCODED_RE.sub(_replace, value) diff --git a/app/services/page_unavailability.py b/app/services/page_unavailability.py new file mode 100644 index 0000000..94049d4 --- /dev/null +++ b/app/services/page_unavailability.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from app.schemas.content_analysis import ContentAnalysisResult +from app.schemas.unchain import UnchainResult + +PAGE_UNAVAILABLE_CODE = "PAGE_UNAVAILABLE" + +_UNAVAILABLE_ERRORS: tuple[str, ...] = ( + "dns_failure", + "timeout", + "connect_error", + "blocked_host", + "invalid_host", +) + + +def _message_for_status(status_code: int) -> str: + if status_code == 404: + return "페이지를 찾을 수 없습니다." + if 400 <= status_code < 500: + return "페이지 요청이 거부되었거나 찾을 수 없습니다." + return "대상 서버 오류로 페이지를 확인할 수 없습니다." + + +def _message_for_error(error: str) -> str: + if error == "dns_failure": + return "도메인 주소를 확인할 수 없습니다." + if error == "timeout": + return "페이지 응답 시간이 초과되었습니다." + if error in {"connect_error"} or error.startswith("connection_refused"): + return "페이지에 연결할 수 없습니다." + if error in {"blocked_host", "invalid_host"}: + return "내부망 또는 차단된 호스트라 분석하지 않았습니다." + if error.startswith("server_error_"): + return "대상 서버 오류로 페이지를 확인할 수 없습니다." + if error.startswith("http_error_"): + return "페이지를 가져오지 못했습니다." + return "페이지를 확인할 수 없습니다." + + +def _status_from_error(error: str) -> int | None: + for prefix in ("server_error_", "http_error_"): + if error.startswith(prefix): + raw = error.removeprefix(prefix) + if raw.isdigit(): + return int(raw) + return None + + +def unchain_page_unavailable(unchain: UnchainResult) -> tuple[str, int | None] | None: + """Return user message/status when unchain already proved the page is unavailable.""" + for hop in reversed(unchain.hops): + if hop.status_code >= 400: + return _message_for_status(hop.status_code), hop.status_code + + error = unchain.error + if error is None: + return None + if error in _UNAVAILABLE_ERRORS or error.startswith( + ("connection_refused", "server_error_", "http_error_") + ): + status_code = _status_from_error(error) + message = _message_for_status(status_code) if status_code else _message_for_error(error) + return message, status_code + return None + + +def content_page_unavailable(content: ContentAnalysisResult) -> tuple[str, int | None] | None: + """Return user message/status when content fetch could not reach an analyzable page.""" + if content.fetched: + return None + error = content.error + if error is None: + return None + if error in _UNAVAILABLE_ERRORS or error.startswith("http_error_"): + status_code = content.status_code or _status_from_error(error) + message = content.reason or ( + _message_for_status(status_code) if status_code else _message_for_error(error) + ) + return message, status_code + return None diff --git a/app/services/pipeline.py b/app/services/pipeline.py new file mode 100644 index 0000000..ec85997 --- /dev/null +++ b/app/services/pipeline.py @@ -0,0 +1,625 @@ +"""URL 분석 파이프라인 오케스트레이터.""" + +from __future__ import annotations + +import asyncio +import time +from collections.abc import Awaitable +from contextlib import suppress +from typing import TYPE_CHECKING, TypeVar +from urllib.parse import urlparse + +import structlog + +from app.core.config import settings +from app.core.exceptions import NormalizationError +from app.core.tld import extract_url_parts +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.domain_heuristic import ( + DomainHeuristicResult, + DomainHeuristicSignal, + DomainHeuristicSkippedReason, +) +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import ( + PipelineFailure, + PipelineStage, + PipelineStages, + PipelineStageTimings, + PipelineSuccess, + PipelineTimings, + Verdict, +) +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult +from app.services.content_analyzer import analyze_content, skipped_already_danger +from app.services.domain_heuristic import check_domain_heuristic +from app.services.normalizer import normalize_url +from app.services.page_unavailability import ( + PAGE_UNAVAILABLE_CODE, + content_page_unavailable, + unchain_page_unavailable, +) +from app.services.pipeline_deadline import ( + PipelineDeadline, + PipelineStageTimeoutError, + timed_out_content_result, + timed_out_domain_result, + timed_out_threat_db_result, + timed_out_unchain_result, +) +from app.services.threat_db import check_threat_db +from app.services.unchainer import unchain_url + +logger = structlog.get_logger(__name__) + +T = TypeVar("T") + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +def _elapsed_seconds(started: float) -> float: + return round(time.perf_counter() - started, 6) + + +def _set_stage_timing( + timings: PipelineStageTimings, + stage: PipelineStage, + started: float, +) -> None: + setattr(timings, stage.value, _elapsed_seconds(started)) + + +def _build_timings(started: float, stage_timings: PipelineStageTimings) -> PipelineTimings: + return PipelineTimings( + total_seconds=_elapsed_seconds(started), + stages=stage_timings, + ) + + +async def _timed_async_stage( + timings: PipelineStageTimings, + stage: PipelineStage, + coro: Awaitable[T], +) -> T: + started = time.perf_counter() + try: + return await coro + finally: + _set_stage_timing(timings, stage, started) + + +def _stage_normalize(log: structlog.stdlib.BoundLogger, original_url: str) -> NormalizeResult: + result = normalize_url(original_url) + log.info("pipeline.normalize.done", normalized_url=result.normalized_url) + return result + + +async def _stage_unchain(log: structlog.stdlib.BoundLogger, normalized_url: str) -> UnchainResult: + result = await unchain_url(normalized_url) + log.info( + "pipeline.unchain.done", + final_url=result.final_url, + hops=result.hop_count, + signals=result.signals, + ) + return result + + +async def _stage_threat_db( + log: structlog.stdlib.BoundLogger, + final_url: str, + session: AsyncSession, + original_url: str | None = None, +) -> ThreatDbResult: + if original_url and original_url != final_url: + result = await check_threat_db(session, final_url, original_url=original_url) + else: + result = await check_threat_db(session, final_url) + log.info( + "pipeline.threat_db.done", + is_malicious=result.is_malicious, + sources_checked=result.sources_checked, + ) + return result + + +async def _stage_domain_heuristic( + log: structlog.stdlib.BoundLogger, final_url: str +) -> DomainHeuristicResult: + result = await check_domain_heuristic(final_url) + log.info( + "pipeline.domain_heuristic.done", + score=result.score, + signals=[s.value for s in result.signals], + ) + return result + + +def _collect_upstream_signals( + threat: ThreatDbResult, heuristic: DomainHeuristicResult +) -> tuple[str, ...]: + """선행 단계 시그널을 한 줄 코드 배열로 평탄화. AI 프롬프트의 사전 정보로 쓰인다. + + threat_db 측은 enum 시그널이 없어 매치 여부를 명시 코드(GSB_THREAT/URLHAUS_THREAT)로 + 합성해 넣는다. 이 단계에 도달했다는 건 short-circuit 미발동 = is_malicious=False 라는 + 뜻이라 두 코드 모두 평소엔 비어있지만, 미래에 약한 매치가 합산만 막혔을 케이스도 + 누락 없이 전달하기 위해 명시적으로 합산한다. + """ + codes: list[str] = [s.value for s in heuristic.signals] + if threat.gsb.is_threat: + codes.append("GSB_THREAT") + if threat.urlhaus.is_threat: + codes.append("URLHAUS_THREAT") + return tuple(codes) + + +def _augment_heuristic_with_redirect_signals( + heuristic: DomainHeuristicResult, + unchain: UnchainResult, +) -> DomainHeuristicResult: + signals = list(heuristic.signals) + score = heuristic.score + if ( + any(signal.startswith("cross_origin:") for signal in unchain.signals) + and DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN not in signals + ): + signals.append(DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN) + score = min( + score + settings.score_weight_redirect_cross_origin, + settings.domain_heuristic_score_cap, + ) + if signals == heuristic.signals and score == heuristic.score: + return heuristic + return heuristic.model_copy(update={"signals": signals, "score": score}) + + +def _page_unavailable_content( + final_url: str, + *, + message: str, + status_code: int | None, +) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=status_code, + score=0, + signals=[ContentSignal.FETCH_FAILED], + reason=message, + error="page_unavailable", + ) + + +async def _stage_content_analysis( + log: structlog.stdlib.BoundLogger, + final_url: str, + upstream_signals: tuple[str, ...], +) -> ContentAnalysisResult: + result = await analyze_content(final_url, upstream_signals=upstream_signals) + log.info( + "pipeline.content_analysis.done", + fetched=result.fetched, + score=result.score, + signals=[s.value for s in result.signals], + ai_verdict=result.ai_verdict.value if result.ai_verdict else None, + upstream_signals=list(upstream_signals), + ) + return result + + +def _preceding_score(threat: ThreatDbResult, heuristic: DomainHeuristicResult) -> int: + """2~3단계 누적 점수. 4단계 건너뛸지 판단하는 기준.""" + score = heuristic.score + if threat.gsb.is_threat: + score += settings.score_weight_gsb + if threat.urlhaus.is_threat: + score += settings.score_weight_urlhaus + return score + + +def _total_score( + threat: ThreatDbResult, + heuristic: DomainHeuristicResult, + content: ContentAnalysisResult, +) -> int: + """전 단계 합산 후 100 으로 캡. content.score 는 4단계가 실제로 돌았을 때만 비-0.""" + if threat.is_malicious: + return settings.score_total_cap + return min( + _preceding_score(threat, heuristic) + content.score, + settings.score_total_cap, + ) + + +def _decide_verdict(score: int, threat: ThreatDbResult) -> Verdict: + """Known malicious 매치는 danger. 나머지는 점수 구간으로 매핑.""" + if threat.is_malicious: + return Verdict.DANGER + if score >= settings.score_danger_threshold: + return Verdict.DANGER + if score >= settings.score_caution_threshold: + return Verdict.CAUTION + return Verdict.SAFE + + +def _skipped_heuristic(final_url: str) -> DomainHeuristicResult: + """threat match 로 조기 종료된 경우 heuristic 자리에 들어갈 placeholder. + + score=0 이라 합산에 영향 없고, skipped_reason 으로 "우리가 건너뛴 것" 을 식별 가능하게 + 남긴다. 응답 스키마의 heuristic 필드가 필수라서 빈 객체라도 채워야 하는 제약 때문. + + domain 값은 정상 경로(check_domain_heuristic)와 동일하게 등록 가능 도메인을 쓴다. + full host 를 쓰면 같은 필드에 의미가 두 가지로 갈리고, 다운스트림에서 + `signin.evil.example.com` 와 `example.com` 이 다른 버킷으로 잡힌다. + """ + ext = extract_url_parts(final_url) + domain = ext.top_domain_under_public_suffix or (urlparse(final_url).hostname or "") + return DomainHeuristicResult( + domain=domain, + score=0, + signals=[], + rdap=None, + rdap_error=None, + skipped_reason=DomainHeuristicSkippedReason.THREAT_MATCHED, + ) + + +def _page_unavailable_failure( + *, + analysis_id: str, + original_url: str, + final_url: str, + failed_at_stage: PipelineStage, + error: str, + status_code: int | None, + started: float, + stage_timings: PipelineStageTimings, +) -> PipelineFailure: + return PipelineFailure( + analysis_id=analysis_id, + original_url=original_url, + final_url=final_url, + failed_at_stage=failed_at_stage, + error=error, + error_code=PAGE_UNAVAILABLE_CODE, + status_code=status_code, + timings=_build_timings(started, stage_timings), + ) + + +async def _run_stage_2_and_3( + log: structlog.stdlib.BoundLogger, + final_url: str, + original_url: str, + session: AsyncSession, + timings: PipelineStageTimings, +) -> tuple[ThreatDbResult, DomainHeuristicResult, bool]: + """2·3단계를 병렬 실행한다. threat_db 가 먼저 malicious 로 끝나면 heuristic 을 + 즉시 cancel 하고 조기 종료 플래그를 세워 반환한다. + + 반환: (threat, heuristic, short_circuited). + short_circuited=True 면 heuristic 은 placeholder 이고 4단계는 skip 대상. + `CancelledError` 와 stage 내부 예외는 남은 task 를 정리한 뒤 그대로 re-raise. + """ + threat_task = asyncio.create_task( + _timed_async_stage( + timings, + PipelineStage.THREAT_DB, + _stage_threat_db(log, final_url, session, original_url), + ) + ) + heur_task = asyncio.create_task( + _timed_async_stage( + timings, + PipelineStage.DOMAIN_HEURISTIC, + _stage_domain_heuristic(log, final_url), + ) + ) + + try: + done, _ = await asyncio.wait( + {threat_task, heur_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + + # short-circuit 시 두 분기 모두 _skipped_heuristic 으로 통일한다 — task 종료 순서에 + # 따라 응답 score 가 placeholder(0) ↔ 실제 점수(15~80) 로 갈리는 비결정성을 제거. + # verdict 는 어차피 DANGER 강제라 사용자 영향은 없지만, 옵저버빌리티/회귀 재현성을 + # 위해 동일 입력에 동일 score 가 나오도록 박는다. + if threat_task in done: + threat: ThreatDbResult = threat_task.result() + if threat.is_malicious: + # RDAP 대기로 5s 가까이 떠 있을 수 있는 heuristic 을 cancel 로 회수. + heur_task.cancel() + with suppress(asyncio.CancelledError): + await heur_task + return threat, _skipped_heuristic(final_url), True + heuristic: DomainHeuristicResult = await heur_task + return threat, heuristic, False + + # heuristic 이 먼저 끝난 경로. threat 는 이미 거의 끝났을 가능성이 높으므로 대기한다. + heuristic = heur_task.result() + threat = await threat_task + if threat.is_malicious: + # 양쪽 다 완료된 뒤 malicious 로 드러난 경우 — heuristic 은 실제 점수가 있지만 + # 위 분기와 응답 일관성을 맞추기 위해 placeholder 로 갈아치운다. + return threat, _skipped_heuristic(final_url), True + return threat, heuristic, False + except BaseException: + # 상위 cancel 또는 stage 내부 예외 — 남은 task 가 고아로 떠돌지 않게 정리하고 전파. + for task in (threat_task, heur_task): + if not task.done(): + task.cancel() + raise + + +async def run_pipeline( + analysis_id: str, + original_url: str, + session: AsyncSession, +) -> PipelineSuccess | PipelineFailure: + log = logger.bind(analysis_id=analysis_id) + log.info("pipeline.start", url=original_url) + total_started = time.perf_counter() + deadline = PipelineDeadline() + stage_timings = PipelineStageTimings() + + stage_started = time.perf_counter() + try: + norm = _stage_normalize(log, original_url) + except NormalizationError as exc: + _set_stage_timing(stage_timings, PipelineStage.NORMALIZE, stage_started) + log.warning("pipeline.failed", stage=PipelineStage.NORMALIZE, error=str(exc)) + return PipelineFailure( + analysis_id=analysis_id, + original_url=original_url, + failed_at_stage=PipelineStage.NORMALIZE, + error=str(exc), + timings=_build_timings(total_started, stage_timings), + ) + _set_stage_timing(stage_timings, PipelineStage.NORMALIZE, stage_started) + + stage_started = time.perf_counter() + try: + unchain: UnchainResult = await deadline.run( + PipelineStage.UNCHAIN.value, + _timed_async_stage( + stage_timings, + PipelineStage.UNCHAIN, + unchain_url( + norm.normalized_url, + prefer_https_when_schemeless=norm.scheme_was_added, + ), + ), + settings.pipeline_unchain_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning("pipeline.stage_timeout", stage=PipelineStage.UNCHAIN) + unchain = timed_out_unchain_result(norm.normalized_url) + if stage_timings.unchain is None: + _set_stage_timing(stage_timings, PipelineStage.UNCHAIN, stage_started) + except Exception as exc: + log.warning( + "pipeline.stage_error", + stage=PipelineStage.UNCHAIN, + error=str(exc), + error_type=type(exc).__name__, + ) + unchain = timed_out_unchain_result(norm.normalized_url, error="stage_error") + if stage_timings.unchain is None: + _set_stage_timing(stage_timings, PipelineStage.UNCHAIN, stage_started) + + if unavailable := unchain_page_unavailable(unchain): + message, status_code = unavailable + log.info( + "pipeline.page_unavailable", + stage=PipelineStage.UNCHAIN, + final_url=unchain.final_url, + status_code=status_code, + error=unchain.error, + ) + try: + threat, heuristic, _ = await deadline.run( + "reputation", + _run_stage_2_and_3( + log, + unchain.final_url, + norm.normalized_url, + session, + stage_timings, + ), + settings.pipeline_reputation_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning("pipeline.stage_timeout", stage="reputation") + threat = timed_out_threat_db_result(unchain.final_url) + heuristic = timed_out_domain_result(unchain.final_url) + except Exception as exc: + log.warning( + "pipeline.stage_error", + stage="reputation", + error=str(exc), + error_type=type(exc).__name__, + ) + threat = timed_out_threat_db_result(unchain.final_url) + heuristic = timed_out_domain_result(unchain.final_url) + + heuristic = _augment_heuristic_with_redirect_signals(heuristic, unchain) + content = _page_unavailable_content( + unchain.final_url, + message=message, + status_code=status_code, + ) + score = _total_score(threat, heuristic, content) + if threat.is_malicious or score >= settings.score_caution_threshold: + verdict = _decide_verdict(score, threat) + return PipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=PipelineStages( + normalize=norm, + unchain=unchain, + threat_db=threat, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) + return _page_unavailable_failure( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + failed_at_stage=PipelineStage.UNCHAIN, + error=message, + status_code=status_code, + started=total_started, + stage_timings=stage_timings, + ) + + # 2·3단계는 둘 다 unchain.final_url 만 필요하고 서로 독립이라 병렬로 돈다. + # threat_db 가 먼저 malicious 로 끝나면 verdict 가 이미 danger 로 확정이므로 + # heuristic 을 cancel 하고 4단계까지 skip — 여기서 조기 종료가 일어난다. + try: + threat, heuristic, short_circuited = await deadline.run( + "reputation", + _run_stage_2_and_3( + log, + unchain.final_url, + norm.normalized_url, + session, + stage_timings, + ), + settings.pipeline_reputation_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning("pipeline.stage_timeout", stage="reputation") + threat = timed_out_threat_db_result(unchain.final_url) + heuristic = timed_out_domain_result(unchain.final_url) + short_circuited = False + + except Exception as exc: + log.warning( + "pipeline.stage_error", + stage="reputation", + error=str(exc), + error_type=type(exc).__name__, + ) + threat = timed_out_threat_db_result(unchain.final_url) + heuristic = timed_out_domain_result(unchain.final_url) + short_circuited = False + + heuristic = _augment_heuristic_with_redirect_signals(heuristic, unchain) + + if short_circuited: + log.info( + "pipeline.short_circuit", + reason="threat_db_match", + gsb_threat=threat.gsb.is_threat, + urlhaus_threat=threat.urlhaus.is_threat, + ) + stage_started = time.perf_counter() + content = skipped_already_danger(unchain.final_url) + _set_stage_timing(stage_timings, PipelineStage.CONTENT_ANALYSIS, stage_started) + else: + # known malicious 는 verdict 가 이미 외부 DB 로 확정됐으므로 페이지를 받아보지 않는다. + # 휴리스틱 danger 는 페이지가 존재하지 않을 수 있으므로 content fetch 로 가용성을 확인한다. + preceding = _preceding_score(threat, heuristic) + if threat.is_malicious: + log.info( + "pipeline.content_analysis.skipped", + reason="threat_db_match", + preceding_score=preceding, + ) + stage_started = time.perf_counter() + content = skipped_already_danger(unchain.final_url) + _set_stage_timing(stage_timings, PipelineStage.CONTENT_ANALYSIS, stage_started) + else: + upstream = _collect_upstream_signals(threat, heuristic) + try: + content = await deadline.run( + PipelineStage.CONTENT_ANALYSIS.value, + _timed_async_stage( + stage_timings, + PipelineStage.CONTENT_ANALYSIS, + _stage_content_analysis(log, unchain.final_url, upstream), + ), + settings.pipeline_content_timeout_seconds, + ) + except PipelineStageTimeoutError: + log.warning("pipeline.stage_timeout", stage=PipelineStage.CONTENT_ANALYSIS) + content = timed_out_content_result(unchain.final_url) + except Exception as exc: + log.warning( + "pipeline.stage_error", + stage=PipelineStage.CONTENT_ANALYSIS, + error=str(exc), + error_type=type(exc).__name__, + ) + content = timed_out_content_result(unchain.final_url) + + if unavailable := content_page_unavailable(content): + message, status_code = unavailable + log.info( + "pipeline.page_unavailable", + stage=PipelineStage.CONTENT_ANALYSIS, + final_url=unchain.final_url, + status_code=status_code, + error=content.error, + ) + score = _total_score(threat, heuristic, content) + if threat.is_malicious or score >= settings.score_caution_threshold: + verdict = _decide_verdict(score, threat) + return PipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=PipelineStages( + normalize=norm, + unchain=unchain, + threat_db=threat, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) + return _page_unavailable_failure( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + failed_at_stage=PipelineStage.CONTENT_ANALYSIS, + error=message, + status_code=status_code, + started=total_started, + stage_timings=stage_timings, + ) + + score = _total_score(threat, heuristic, content) + verdict = _decide_verdict(score, threat) + log.info( + "pipeline.done", + final_url=unchain.final_url, + verdict=verdict.value, + score=score, + ) + return PipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + verdict=verdict, + score=score, + timings=_build_timings(total_started, stage_timings), + stages=PipelineStages( + normalize=norm, + unchain=unchain, + threat_db=threat, + domain_heuristic=heuristic, + content_analysis=content, + ), + ) diff --git a/app/services/pipeline_deadline.py b/app/services/pipeline_deadline.py new file mode 100644 index 0000000..73846cc --- /dev/null +++ b/app/services/pipeline_deadline.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import asyncio +import time +from collections.abc import Awaitable +from typing import TypeVar +from urllib.parse import urlparse + +from app.core.config import settings +from app.core.tld import extract_url_parts +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.threat_db import GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import UnchainResult + +T = TypeVar("T") + + +class PipelineStageTimeoutError(TimeoutError): + def __init__(self, stage: str) -> None: + super().__init__(f"{stage} timed out") + self.stage = stage + + +class PipelineDeadline: + def __init__(self, total_seconds: float | None = None) -> None: + self._started = time.perf_counter() + self._total_seconds = total_seconds or settings.pipeline_total_timeout_seconds + + def remaining(self) -> float: + return max(0.0, self._total_seconds - (time.perf_counter() - self._started)) + + def budget(self, stage_seconds: float) -> float: + # 작은 양수로 wait_for까지 진입시켜 stage timeout 경로를 일관되게 탄다. + return max(0.001, min(stage_seconds, self.remaining())) + + async def run(self, stage: str, coro: Awaitable[T], stage_seconds: float) -> T: + try: + return await asyncio.wait_for(coro, timeout=self.budget(stage_seconds)) + except TimeoutError as exc: + raise PipelineStageTimeoutError(stage) from exc + + +def timed_out_unchain_result(url: str, *, error: str = "pipeline_timeout") -> UnchainResult: + return UnchainResult( + input_url=url, + final_url=url, + hops=[], + hop_count=0, + timed_out=True, + error=error, + signals=[error], + ) + + +def timed_out_threat_db_result(final_url: str) -> ThreatDbResult: + return ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=0, + gsb=GSBResult(checked=False, is_threat=False, error="pipeline_timeout"), + urlhaus=URLhausResult(checked=False, is_threat=False, error="pipeline_timeout"), + threat_types=[], + ) + + +def timed_out_domain_result(final_url: str) -> DomainHeuristicResult: + ext = extract_url_parts(final_url) + domain = ext.top_domain_under_public_suffix or (urlparse(final_url).hostname or "") + return DomainHeuristicResult( + domain=domain, + score=0, + signals=[], + rdap=None, + rdap_error="pipeline_timeout", + ) + + +def timed_out_content_result(final_url: str) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=None, + score=settings.score_weight_content_fetch_failed, + signals=[ContentSignal.FETCH_FAILED], + reason="콘텐츠 분석 시간이 초과되었습니다.", + error="pipeline_timeout", + ) diff --git a/app/services/threat_db/__init__.py b/app/services/threat_db/__init__.py new file mode 100644 index 0000000..45d287e --- /dev/null +++ b/app/services/threat_db/__init__.py @@ -0,0 +1,8 @@ +"""외부 위협 DB 대조 단계 (Stage 2). + +GSB(Google Safe Browsing) 실시간 조회와 URLhaus 로컬 스냅샷 조회를 병렬로 수행. +""" + +from app.services.threat_db.check import check_threat_db + +__all__ = ["check_threat_db"] diff --git a/app/services/threat_db/check.py b/app/services/threat_db/check.py new file mode 100644 index 0000000..0b8c636 --- /dev/null +++ b/app/services/threat_db/check.py @@ -0,0 +1,113 @@ +"""GSB + URLhaus 병렬 조회 후 결과 병합.""" + +from __future__ import annotations + +import asyncio + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.schemas.threat_db import GSBResult, ThreatDbResult, URLhausResult +from app.services.threat_db.gsb import check_gsb +from app.services.threat_db.urlhaus import check_urlhaus + +logger = get_logger(__name__) + + +def _merge_threat_types(gsb: GSBResult, urlhaus: URLhausResult) -> list[str]: + seen: list[str] = [] + for m in gsb.matches: + if m.threat_type and m.threat_type not in seen: + seen.append(m.threat_type) + if urlhaus.threat and urlhaus.threat not in seen: + seen.append(urlhaus.threat) + return seen + + +def _candidate_urls(final_url: str, original_url: str | None) -> list[str]: + candidates = [final_url] + if original_url and original_url != final_url: + candidates.append(original_url) + return candidates + + +def _merge_gsb(results: list[GSBResult]) -> GSBResult: + checked = any(result.checked for result in results) + matches: list = [] + error = None + for result in results: + if result.matches: + matches.extend(result.matches) + if error is None and result.error: + error = result.error + return GSBResult( + checked=checked, + is_threat=bool(matches), + matches=matches, + error=None if checked else error, + ) + + +def _merge_urlhaus(results: list[URLhausResult]) -> URLhausResult: + for result in results: + if result.is_threat: + return result + checked = any(result.checked for result in results) + error = next((result.error for result in results if result.error), None) + return URLhausResult(checked=checked, is_threat=False, error=None if checked else error) + + +async def check_threat_db( + session: AsyncSession, + final_url: str, + *, + original_url: str | None = None, +) -> ThreatDbResult: + """final_url 을 GSB + URLhaus 와 병렬 대조해 판정 결과 반환. + + 어느 한쪽이 실패해도 다른 쪽 결과로 판정한다. 두 쪽 다 실패 시 + is_malicious=False, sources_checked=0 로 반환하여 상위 레이어가 보수적으로 처리. + """ + candidates = _candidate_urls(final_url, original_url) + gsb_tasks = [check_gsb(url) for url in candidates] + urlhaus_tasks = [check_urlhaus(session, url) for url in candidates] + + raw_results = await asyncio.gather(*gsb_tasks, *urlhaus_tasks, return_exceptions=True) + + # CancelledError 는 상위 task 의 취소 신호이므로 절대 삼키지 않는다. + # (shutdown / 요청 timeout 시 degraded 결과를 영속화하는 사고 방지) + for raw in raw_results: + if isinstance(raw, asyncio.CancelledError): + raise raw + + gsb_results: list[GSBResult] = [] + urlhaus_results: list[URLhausResult] = [] + for raw in raw_results[: len(candidates)]: + if isinstance(raw, BaseException): + logger.warning("threat_db.gsb_unexpected", error=str(raw)) + gsb_results.append(GSBResult(checked=False, is_threat=False, error="unexpected")) + else: + gsb_results.append(raw) + for raw in raw_results[len(candidates) :]: + if isinstance(raw, BaseException): + logger.warning("threat_db.urlhaus_unexpected", error=str(raw)) + urlhaus_results.append( + URLhausResult(checked=False, is_threat=False, error="unexpected") + ) + else: + urlhaus_results.append(raw) + + gsb = _merge_gsb(gsb_results) + urlhaus = _merge_urlhaus(urlhaus_results) + + is_malicious = gsb.is_threat or urlhaus.is_threat + sources_checked = sum((gsb.checked, urlhaus.checked)) + + return ThreatDbResult( + final_url=final_url, + is_malicious=is_malicious, + sources_checked=sources_checked, + gsb=gsb, + urlhaus=urlhaus, + threat_types=_merge_threat_types(gsb, urlhaus), + ) diff --git a/app/services/threat_db/gsb.py b/app/services/threat_db/gsb.py new file mode 100644 index 0000000..48cf5b1 --- /dev/null +++ b/app/services/threat_db/gsb.py @@ -0,0 +1,103 @@ +"""Google Safe Browsing Lookup API 클라이언트. + +실패(타임아웃/인증/레이트리밋/서버 에러)는 모두 raise 하지 않고 +`GSBResult(checked=False, error=...)` 로 반환해 상위 파이프라인을 보호한다. +""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.analysis import GSBMatch, GSBResult + +logger = get_logger(__name__) + +# 키 값별로 1회만 경고. 운영 중 키가 토글되거나(hot reload) 테스트가 키를 +# 바꿔 끼우면 새 상태에 대해 다시 한 번 경고가 나간다. +_WARNED_KEY_STATES: set[str] = set() + + +def _log_missing_key_once() -> None: + state = settings.gsb_api_key or "" + if state in _WARNED_KEY_STATES: + return + _WARNED_KEY_STATES.add(state) + logger.warning("gsb.api_key_not_configured") + + +def reset_missing_key_warning() -> None: + """테스트/hot-reload 시 throttle 상태 초기화.""" + _WARNED_KEY_STATES.clear() + + +def _build_request_body(url: str) -> dict[str, Any]: + return { + "client": { + "clientId": settings.gsb_client_id, + "clientVersion": settings.gsb_client_version, + }, + "threatInfo": { + "threatTypes": [ + "MALWARE", + "SOCIAL_ENGINEERING", + "UNWANTED_SOFTWARE", + "POTENTIALLY_HARMFUL_APPLICATION", + ], + "platformTypes": ["ANY_PLATFORM"], + "threatEntryTypes": ["URL"], + "threatEntries": [{"url": url}], + }, + } + + +async def check_gsb(url: str) -> GSBResult: + """GSB Lookup API 호출 결과를 GSBResult 로 반환.""" + if not settings.gsb_api_key: + _log_missing_key_once() + return GSBResult(checked=False, is_threat=False, error="not_configured") + + params = {"key": settings.gsb_api_key} + body = _build_request_body(url) + + try: + async with httpx.AsyncClient(timeout=settings.gsb_timeout_seconds) as client: + resp = await client.post(settings.gsb_api_url, params=params, json=body) + except httpx.TimeoutException: + return GSBResult(checked=False, is_threat=False, error="timeout") + except httpx.HTTPError: + return GSBResult(checked=False, is_threat=False, error="http_error") + + status = resp.status_code + if status in (400, 401, 403): + return GSBResult(checked=False, is_threat=False, error="auth_error") + if status == 429: + return GSBResult(checked=False, is_threat=False, error="rate_limited") + if 500 <= status < 600: + return GSBResult(checked=False, is_threat=False, error=f"server_error_{status}") + if status != 200: + return GSBResult(checked=False, is_threat=False, error="http_error") + + try: + data = resp.json() + except ValueError: + return GSBResult(checked=False, is_threat=False, error="invalid_response") + + raw_matches = data.get("matches") or [] + matches = [ + GSBMatch( + threat_type=m.get("threatType", "UNKNOWN"), + platform_type=m.get("platformType"), + cache_duration=m.get("cacheDuration"), + ) + for m in raw_matches + ] + return GSBResult( + checked=True, + is_threat=bool(matches), + matches=matches, + error=None, + ) diff --git a/app/services/threat_db/match_keys.py b/app/services/threat_db/match_keys.py new file mode 100644 index 0000000..accf888 --- /dev/null +++ b/app/services/threat_db/match_keys.py @@ -0,0 +1,35 @@ +"""URLhaus 조회·동기화에 쓰는 매칭 키 생성. + +host 한 개가 기본이지만 GitHub/GitLab/Dropbox 같은 다중 테넌트 호스트는 +계정/리포/공유 파일 레벨에서 악성 여부가 갈리므로 host+path-prefix 키만 사용한다. +""" + +from __future__ import annotations + +from urllib.parse import urlparse + +from app.core.config import settings + + +def derive_keys(url: str) -> list[str]: + """URL 에서 매칭 키 후보를 더 구체적인 순서로 반환. + + 반환: [host_path] 또는 [host] + host 추출 실패 시 빈 리스트. + """ + parsed = urlparse(url) + host = (parsed.hostname or "").lower() + if not host: + return [] + + required = settings.urlhaus_multitenant_hosts.get(host) + if required is None or required <= 0: + return [host] + + segments = [seg for seg in parsed.path.split("/") if seg] + if len(segments) < required: + return [] + + prefix = "/".join(segments[:required]) + host_path = f"{host}/{prefix}" + return [host_path] diff --git a/app/services/threat_db/urlhaus.py b/app/services/threat_db/urlhaus.py new file mode 100644 index 0000000..6602ebd --- /dev/null +++ b/app/services/threat_db/urlhaus.py @@ -0,0 +1,113 @@ +"""URLhaus 로컬 SQLite 조회. + +URL 완전일치 → match_key(host_path/host) 조회 순. +""" + +from __future__ import annotations + +from typing import Literal +from urllib.parse import urlsplit, urlunsplit + +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.models.urlhaus_entry import URLhausEntry +from app.schemas.analysis import URLhausResult +from app.services.threat_db.match_keys import derive_keys + +logger = get_logger(__name__) + + +def _split_tags(tags: str | None) -> list[str]: + if not tags: + return [] + return [t.strip() for t in tags.split(",") if t.strip()] + + +def _to_result( + entry: URLhausEntry, + match_type: Literal["url", "host", "host_path"], + matched_key: str, +) -> URLhausResult: + return URLhausResult( + checked=True, + is_threat=True, + match_type=match_type, + matched_key=matched_key, + threat=entry.threat, + tags=_split_tags(entry.tags), + urlhaus_link=entry.urlhaus_link, + ) + + +def _url_variants(url: str) -> list[str]: + variants: list[str] = [] + + def add(candidate: str) -> None: + if candidate and candidate not in variants: + variants.append(candidate) + + add(url) + try: + parts = urlsplit(url) + except ValueError: + return variants + if not parts.scheme or not parts.netloc: + return variants + + path_variants = [parts.path] + if parts.path == "": + path_variants.append("/") + elif parts.path == "/": + path_variants.append("") + elif parts.path.endswith("/"): + path_variants.append(parts.path.rstrip("/")) + else: + path_variants.append(parts.path + "/") + + schemes = [parts.scheme] + if parts.scheme == "https": + schemes.append("http") + elif parts.scheme == "http": + schemes.append("https") + + for scheme in schemes: + for path in path_variants: + add(urlunsplit((scheme, parts.netloc, path, parts.query, parts.fragment))) + return variants + + +async def check_urlhaus(session: AsyncSession, url: str) -> URLhausResult: + """URLhaus 로컬 스냅샷에서 URL 매칭 여부 조회.""" + try: + # 1) URL 완전일치 + for candidate in _url_variants(url): + stmt = select(URLhausEntry).where(URLhausEntry.url == candidate) + row = (await session.execute(stmt)).scalar_one_or_none() + if row is not None: + return _to_result(row, "url", row.url) + + # 2) match_key IN (...) + keys = derive_keys(url) + if not keys: + return URLhausResult(checked=True, is_threat=False) + + stmt = select(URLhausEntry).where(URLhausEntry.match_key.in_(keys)) + rows = (await session.execute(stmt)).scalars().all() + if not rows: + return URLhausResult(checked=True, is_threat=False) + + # host_path(더 구체적) 우선 + by_key = {r.match_key: r for r in rows} + for key in keys: + hit = by_key.get(key) + if hit is not None: + match_type: Literal["host", "host_path"] = "host_path" if "/" in key else "host" + return _to_result(hit, match_type, key) + + return URLhausResult(checked=True, is_threat=False) + except SQLAlchemyError as e: + logger.warning("urlhaus.db_error", error=str(e)) + return URLhausResult(checked=False, is_threat=False, error="db_error") diff --git a/app/services/threat_db/urlhaus_sync.py b/app/services/threat_db/urlhaus_sync.py new file mode 100644 index 0000000..16b81aa --- /dev/null +++ b/app/services/threat_db/urlhaus_sync.py @@ -0,0 +1,228 @@ +"""URLhaus CSV 다운로드 → 로컬 SQLite upsert. + +CSV 포맷(abuse.ch): + # comment lines starting with '#' + id,dateadded,url,url_status,last_online,threat,tags,urlhaus_link,reporter + +설계 메모 +--------- +- **부분 진행 보존**: 전체 행을 단일 트랜잭션으로 감싸지 않고 `CHUNK_SIZE` 단위로 + 커밋한다. 중간에 DB 오류가 나도 직전 청크까지의 결과는 영속화되며, `stats` 는 + 실제로 커밋된 카운트만 보고한다. +- **insert/update 정확 분리**: SQLite 의 `INSERT ... ON CONFLICT DO UPDATE` 는 + cursor 반환 값으로 두 경로를 구분할 수 없다. 청크 처리 직전에 해당 청크의 id + 들이 이미 존재하는지 한 번 SELECT 해 결과를 분류한다. +- **시각 정규화**: SQLite 에는 모두 naive UTC 로 저장한다. `_parse_dt` 와 + `synced_at` 양쪽이 동일한 규약을 따라야 비교/조회가 깨지지 않는다. +""" + +from __future__ import annotations + +import csv +import io +from collections.abc import Iterable, Iterator +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urlparse + +import httpx +from sqlalchemy import select +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config import settings +from app.core.logging import get_logger +from app.db.session import SessionLocal +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db.match_keys import derive_keys + +logger = get_logger(__name__) + +_CSV_FIELDS = [ + "id", + "dateadded", + "url", + "url_status", + "last_online", + "threat", + "tags", + "urlhaus_link", + "reporter", +] + +# 청크 크기 — 너무 작으면 트랜잭션 오버헤드, 너무 크면 부분 진행 보존 효과 약화. +CHUNK_SIZE = 500 + +RawCsvRow = dict[str, str] +ValueRow = dict[str, Any] +SyncStats = dict[str, int] + + +def _parse_dt(raw: str | None) -> datetime | None: + """CSV 의 시각 문자열을 naive UTC datetime 으로 정규화.""" + if not raw or raw in ("None", "N/A"): + return None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S UTC"): + try: + # SQLite 컬럼이 naive 이므로 tzinfo 를 떼서 일관성 유지. + return datetime.strptime(raw.strip(), fmt).replace(tzinfo=None) + except ValueError: + continue + return None + + +def _derive_match_key(url: str) -> tuple[str, str] | None: + """(host, match_key) 반환. host 없으면 None.""" + keys = derive_keys(url) + if not keys: + return None + parsed = urlparse(url) + host = (parsed.hostname or "").lower() + # derive_keys 는 [host_path] 또는 [host] — 첫 원소가 저장할 match_key. + return host, keys[0] + + +async def _fetch_csv() -> str | None: + try: + async with httpx.AsyncClient(timeout=settings.urlhaus_download_timeout_seconds) as client: + resp = await client.get(settings.urlhaus_recent_csv_url) + except httpx.HTTPError as e: + logger.warning("urlhaus_sync.fetch_failed", error=str(e)) + return None + if resp.status_code != 200: + logger.warning("urlhaus_sync.fetch_bad_status", status=resp.status_code) + return None + return resp.text + + +def _iter_raw_rows(csv_text: str) -> Iterator[list[str]]: + """CSV 의 데이터 행을 list 로 yield. 주석/빈 줄 스킵.""" + clean = io.StringIO( + "\n".join(line for line in csv_text.splitlines() if line and not line.startswith("#")) + ) + reader = csv.reader(clean, quotechar='"', skipinitialspace=True) + yield from reader + + +def _build_values(raw: RawCsvRow, now: datetime) -> ValueRow | None: + """CSV 한 행을 ORM values dict 로 변환. 검증 실패 시 None.""" + try: + entry_id = int(raw["id"]) + except (TypeError, ValueError): + return None + + url = raw.get("url") or "" + if not url: + return None + + derived = _derive_match_key(url) + if derived is None: + return None + host, match_key = derived + + return { + "id": entry_id, + "url": url, + "host": host, + "match_key": match_key, + "threat": raw.get("threat") or None, + "tags": raw.get("tags") or None, + "url_status": raw.get("url_status") or None, + "date_added": _parse_dt(raw.get("dateadded")), + "last_online": _parse_dt(raw.get("last_online")), + "urlhaus_link": raw.get("urlhaus_link") or None, + "reporter": raw.get("reporter") or None, + "synced_at": now, + } + + +def _chunked(rows: Iterable[ValueRow], size: int) -> Iterator[list[ValueRow]]: + chunk: list[ValueRow] = [] + for row in rows: + chunk.append(row) + if len(chunk) >= size: + yield chunk + chunk = [] + if chunk: + yield chunk + + +async def _flush_chunk(session: AsyncSession, values_list: list[ValueRow]) -> tuple[int, int]: + """한 청크를 upsert 하고 (inserted, updated) 카운트 반환. + + SQLite 의 `INSERT ... ON CONFLICT DO UPDATE` 는 cursor 결과로 insert/update + 를 구분할 수 없으므로, 청크 단위로 한 번 SELECT 해서 사전 분류한다. + + multi-row VALUES 한 번의 INSERT 로 청크 전체를 처리 — 종전 행당 INSERT 호출은 + 청크 500건 기준 500회 round-trip 이라 SQLite 라도 비용이 더 든다. + """ + if not values_list: + return 0, 0 + + ids = [v["id"] for v in values_list] + existing_ids = set( + (await session.execute(select(URLhausEntry.id).where(URLhausEntry.id.in_(ids)))) + .scalars() + .all() + ) + + stmt = sqlite_insert(URLhausEntry).values(values_list) + # 모든 values 가 동일한 키 셋을 갖는다는 전제 — _build_values 가 보장. + update_cols = {c: stmt.excluded[c] for c in values_list[0] if c != "id"} + stmt = stmt.on_conflict_do_update(index_elements=["id"], set_=update_cols) + await session.execute(stmt) + + updated = sum(1 for v in values_list if v["id"] in existing_ids) + inserted = len(values_list) - updated + return inserted, updated + + +async def sync_urlhaus() -> SyncStats: + """URLhaus CSV 를 동기화하고 {inserted, updated, total, failed} 통계 반환. + + `stats` 는 **실제 DB 에 커밋된 행 수** 만 누적한다. 중간 청크에서 오류가 나도 + 직전까지의 청크는 영속화되어 있고, 그만큼만 카운트된다. + """ + stats: SyncStats = {"inserted": 0, "updated": 0, "total": 0, "failed": 0} + + csv_text = await _fetch_csv() + if csv_text is None: + return stats + + now = datetime.now(UTC).replace(tzinfo=None) + + def _values_stream() -> Iterator[ValueRow]: + for raw in _iter_raw_rows(csv_text): + stats["total"] += 1 + if len(raw) < len(_CSV_FIELDS): + stats["failed"] += 1 + continue + row_dict = dict(zip(_CSV_FIELDS, raw[: len(_CSV_FIELDS)], strict=True)) + values = _build_values(row_dict, now) + if values is None: + stats["failed"] += 1 + continue + yield values + + async with SessionLocal() as session: + for chunk in _chunked(_values_stream(), CHUNK_SIZE): + try: + async with session.begin(): + inserted, updated = await _flush_chunk(session, chunk) + stats["inserted"] += inserted + stats["updated"] += updated + except SQLAlchemyError as e: + logger.warning( + "urlhaus_sync.chunk_failed", + error=str(e), + inserted_so_far=stats["inserted"], + updated_so_far=stats["updated"], + ) + # 실패한 청크는 롤백 — 그만큼은 retry 대상으로 failed 에 누적. + stats["failed"] += len(chunk) + # 후속 청크는 새 트랜잭션으로 계속 시도. + continue + + logger.info("urlhaus_sync.completed", **stats) + return stats diff --git a/app/services/unchainer/__init__.py b/app/services/unchainer/__init__.py new file mode 100644 index 0000000..7d06557 --- /dev/null +++ b/app/services/unchainer/__init__.py @@ -0,0 +1,3 @@ +from app.services.unchainer.unchain import unchain_url + +__all__ = ["unchain_url"] diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py new file mode 100644 index 0000000..92e81df --- /dev/null +++ b/app/services/unchainer/unchain.py @@ -0,0 +1,341 @@ +"""URL 언체이닝 — 파이프라인 2단계. + +정규화된 URL의 리다이렉트 체인을 끝까지 추적해서 +최종 목적지 URL과 경로상 의심 신호를 수집함. + +httpx.AsyncClient 는 모듈 레벨 싱글턴으로 재사용 — 매 요청마다 새 client 를 만들면 +TCP/TLS 핸드셰이크 비용을 매번 새로 치르고 connection pool 도 공유되지 않는다. +RDAP / fetch 와 동일 패턴이며 lifespan 에서 aclose_client() 로 명시 종료한다. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +from urllib.parse import urljoin, urlparse, urlunparse + +import httpx + +from app.core.config import settings +from app.core.dns_cache import resolve_host_addrs +from app.core.tld import extract_url_parts +from app.schemas.analysis import HopRecord, UnchainResult + +_REDIRECT_CODES: frozenset[int] = frozenset({301, 302, 303, 307, 308}) + +# 리다이렉트 대상으로 허용하는 스킴 (javascript:, data: 등 차단) +_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"}) + + +# 모듈 레벨 싱글턴 — 첫 호출 때 lazy init, lifespan 에서 aclose. +_client: httpx.AsyncClient | None = None + + +def _build_client() -> httpx.AsyncClient: + return httpx.AsyncClient( + timeout=httpx.Timeout( + settings.unchain_timeout_seconds, + connect=settings.unchain_connect_timeout_seconds, + ), + follow_redirects=False, + verify=True, + cookies=None, + ) + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None: + _client = _build_client() + return _client + + +async def aclose_client() -> None: + """앱 셧다운 훅에서 호출. 테스트에서도 client 상태 초기화에 쓴다.""" + global _client + if _client is not None: + await _client.aclose() + _client = None + + +def _is_private_ip(ip_str: str) -> bool: + """외부 접근 불가(is_global=False)인 IP인지 확인. + + is_reserved는 NAT64(64:ff9b::/96) 같은 공개 도달 가능 대역까지 잡아서 + 오탐이 발생하므로, is_global 기반으로 판단. + """ + try: + addr = ipaddress.ip_address(ip_str) + except ValueError: + return True # 파싱 실패 시 안전하게 차단 + return not addr.is_global + + +async def _check_host_safety(url: str) -> str | None: + """호스트를 DNS 조회 후 내부 IP이면 에러 사유 반환, 안전하면 None.""" + parsed = urlparse(url) + hostname = parsed.hostname + if not hostname: + return "invalid_host" + + try: + addrinfo = await resolve_host_addrs(hostname) + except OSError: + # gaierror 도 OSError 서브클래스. dns_cache 가 캐시하지 않은 실패만 여기로 올라옴. + return "dns_failure" + + for _, _, _, _, sockaddr in addrinfo: + if _is_private_ip(sockaddr[0]): + return "ssrf_blocked" + + return None + + +def _https_variant(url: str) -> str | None: + parsed = urlparse(url) + if parsed.scheme != "http" or not parsed.hostname: + return None + return urlunparse(parsed._replace(scheme="https")) + + +def _registered_domain_from_host(host: str | None) -> str: + if not host: + return "" + ext = extract_url_parts(f"https://{host.strip('[]')}/") + return (ext.top_domain_under_public_suffix or host).lower() + + +def _is_registered_domain_change(left_host: str | None, right_host: str | None) -> bool: + return _registered_domain_from_host(left_host) != _registered_domain_from_host(right_host) + + +def _is_analyzable_html_response(resp: httpx.Response) -> bool: + content_type = resp.headers.get("content-type", "").lower() + return 200 <= resp.status_code < 400 and "text/html" in content_type + + +async def _https_responds(client: httpx.AsyncClient, url: str, headers: dict[str, str]) -> bool: + https_url = _https_variant(url) + if https_url is None: + return False + safety_error = await _check_host_safety(https_url) + if safety_error is not None: + return False + resp: httpx.Response | None = None + try: + req = client.build_request("HEAD", https_url, headers=headers) + resp = await asyncio.wait_for( + client.send(req, stream=True), + timeout=settings.schemeless_https_probe_timeout_seconds, + ) + return _is_analyzable_html_response(resp) + except (TimeoutError, httpx.HTTPError): + return False + finally: + if resp is not None: + await resp.aclose() + + +async def unchain_url(url: str, *, prefer_https_when_schemeless: bool = False) -> UnchainResult: + """리다이렉트 체인을 추적하고 최종 URL·hop 기록·의심 신호를 반환.""" + try: + return await asyncio.wait_for( + _unchain_url_inner(url, prefer_https_when_schemeless=prefer_https_when_schemeless), + timeout=settings.unchain_chain_timeout_seconds, + ) + except TimeoutError: + return UnchainResult( + input_url=url, + final_url=url, + hops=[], + hop_count=0, + timed_out=True, + error="chain_timeout", + signals=[], + ) + + +async def _unchain_url_inner( + url: str, + *, + prefer_https_when_schemeless: bool = False, +) -> UnchainResult: + """실제 체인 추적 로직. unchain_url에서 총 timeout으로 감싸서 호출.""" + hops: list[HopRecord] = [] + signals: list[str] = [] + visited: set[str] = set() + current_url = url + error: str | None = None + timed_out = False + + headers = { + "User-Agent": settings.unchain_user_agent, + "Accept": "*/*", + "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7", + } + + client: httpx.AsyncClient | None = None + if prefer_https_when_schemeless: + client = _get_client() + if client is not None and await _https_responds(client, current_url, headers): + https_url = _https_variant(current_url) + if https_url is not None: + current_url = https_url + signals.append("schemeless_https_upgrade") + + for _ in range(settings.unchain_max_hops): + if current_url in visited: + signals.append("redirect_loop") + break + visited.add(current_url) + + # SSRF·DNS Rebinding 방어: 매 hop마다 목적지 IP 검증 + safety_error = await _check_host_safety(current_url) + if safety_error is not None: + error = safety_error + if safety_error == "ssrf_blocked": + signals.append("ssrf_blocked") + break + + if client is None: + client = _get_client() + + hop, next_url, hop_error = await _follow_one_hop( + client, + current_url, + headers, + signals, + ) + + if hop is not None: + hops.append(hop) + + if hop_error is not None: + error = hop_error + if hop_error == "timeout": + timed_out = True + break + + # 리다이렉트가 아니면 체인 종료 + if next_url is None: + break + + current_url = next_url + else: + # max_hops 도달 + signals.append("max_hops_reached") + + return UnchainResult( + input_url=url, + final_url=current_url, + hops=hops, + hop_count=len(hops), + timed_out=timed_out, + error=error, + signals=signals, + ) + + +async def _follow_one_hop( + client: httpx.AsyncClient, + url: str, + headers: dict[str, str], + signals: list[str], +) -> tuple[HopRecord | None, str | None, str | None]: + """단일 hop 요청. (hop_record, next_url_or_None, error_or_None) 반환. + + HEAD 우선 → 실패 시(상태 코드·네트워크 에러 모두) GET 폴백. + """ + for method in ("HEAD", "GET"): + try: + # 헤더만 필요하므로 스트리밍으로 열고 본문은 즉시 닫음 + req = client.build_request(method, url, headers=headers) + resp = await client.send(req, stream=True) + await resp.aclose() + except httpx.TimeoutException: + if method == "HEAD": + continue + return None, None, "timeout" + except httpx.ConnectError as e: + if method == "HEAD": + continue + cause = str(e).lower() + if "name or service not known" in cause or "getaddrinfo" in cause: + return None, None, "dns_failure" + return None, None, f"connection_refused: {e}" + except httpx.HTTPError as e: + if method == "HEAD": + continue + return None, None, f"http_error: {e}" + + # HEAD에서 405 등 클라이언트 에러면 GET 폴백 + if method == "HEAD" and resp.status_code in {405, 403, 400}: + continue + + # 5xx 서버 에러 → 체인 중단 + if resp.status_code >= 500: + hop = HopRecord( + url=url, + status_code=resp.status_code, + method=method, + ) + return hop, None, f"server_error_{resp.status_code}" + + if resp.status_code in _REDIRECT_CODES: + raw_location = resp.headers.get("location") + if not raw_location: + # 일부 서버는 HEAD 응답에서 Location을 생략하므로 GET으로 재시도 + if method == "HEAD": + continue + hop = HopRecord( + url=url, + status_code=resp.status_code, + method=method, + ) + return hop, None, "missing_location_header" + + next_url = urljoin(url, raw_location) + parsed_next = urlparse(next_url) + + # 허용되지 않는 스킴 방어 (javascript:, data: 등) + if parsed_next.scheme not in _ALLOWED_SCHEMES: + hop = HopRecord( + url=url, + status_code=resp.status_code, + raw_location=raw_location, + location=next_url, + method=method, + ) + signals.append(f"unsafe_scheme:{parsed_next.scheme}") + return hop, None, f"unsafe_redirect_scheme:{parsed_next.scheme}" + + parsed_url = urlparse(url) + + hop = HopRecord( + url=url, + status_code=resp.status_code, + raw_location=raw_location, + location=next_url, + method=method, + ) + + # 스킴 다운그레이드 감지 (https → http) + if parsed_url.scheme == "https" and parsed_next.scheme == "http": + signals.append("scheme_downgrade") + + # 등록 도메인 변화. www → bare 같은 정상 canonical redirect 는 제외한다. + if _is_registered_domain_change(parsed_url.hostname, parsed_next.hostname): + signals.append(f"cross_origin:{parsed_url.hostname}->{parsed_next.hostname}") + + return hop, next_url, None + + # 리다이렉트 아닌 정상 응답 → 체인 종료 + hop = HopRecord( + url=url, + status_code=resp.status_code, + method=method, + ) + return hop, None, None + + # 여기 도달 불가하지만 타입 안전 + return None, None, "unexpected_fallthrough" # pragma: no cover diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..6be3486 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +alembic upgrade head +exec uvicorn app.main:app --host 0.0.0.0 --port 8000 diff --git a/pyproject.toml b/pyproject.toml index c8dfd42..6c4de16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,18 @@ dependencies = [ "structlog>=24.4.0", "httpx>=0.28.0", "tenacity>=9.0.0", + "apscheduler>=3.10", + "tldextract>=5.1.0", + "beautifulsoup4>=4.12.0", + "lxml>=5.3.0", + "cachetools>=5.5.0", + "openai>=1.60.0", ] [project.optional-dependencies] +render = [ + "playwright>=1.45.0", +] dev = [ "pytest>=8.3.0", "pytest-asyncio>=0.24.0", @@ -70,6 +79,7 @@ exclude = ["alembic/"] [tool.pytest.ini_options] asyncio_mode = "auto" +asyncio_default_test_loop_scope = "function" testpaths = ["tests"] addopts = "-ra --strict-markers --strict-config" filterwarnings = ["error"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/api/test_analyze_callback.py b/tests/api/test_analyze_callback.py new file mode 100644 index 0000000..67d3e05 --- /dev/null +++ b/tests/api/test_analyze_callback.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from app.api.v1.endpoints import analyze as analyze_endpoint +from app.schemas.pipeline import PipelineFailure, PipelineStage + + +class _FakeSessionContext: + async def __aenter__(self) -> object: + return object() + + async def __aexit__(self, *args: object) -> None: + return None + + +@pytest.mark.asyncio +async def test_background_task_posts_pipeline_result_callback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result = PipelineFailure( + analysis_id="aid-1", + original_url="not a url", + failed_at_stage=PipelineStage.NORMALIZE, + error="invalid", + ) + mock_run_pipeline = AsyncMock(return_value=result) + mock_callback = AsyncMock(return_value=True) + monkeypatch.setattr(analyze_endpoint, "SessionLocal", lambda: _FakeSessionContext()) + monkeypatch.setattr(analyze_endpoint, "run_pipeline", mock_run_pipeline) + monkeypatch.setattr(analyze_endpoint, "post_analysis_callback", mock_callback) + + await analyze_endpoint._run_pipeline_task( + analysis_id="aid-1", + original_url="not a url", + request_id="rid-1", + ) + + mock_run_pipeline.assert_awaited_once() + mock_callback.assert_awaited_once() + kwargs = mock_callback.await_args.kwargs + assert mock_callback.await_args.args == (result,) + assert kwargs["request_id"] == "rid-1" + assert kwargs["elapsed_ms"] >= 0 + assert kwargs["analyzed_at"].tzinfo is not None diff --git a/tests/api/test_analyze_db_independent.py b/tests/api/test_analyze_db_independent.py new file mode 100644 index 0000000..15ebe16 --- /dev/null +++ b/tests/api/test_analyze_db_independent.py @@ -0,0 +1,105 @@ +"""DB 비의존 분석 엔드포인트 테스트.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from unittest.mock import AsyncMock + +import httpx +import pytest_asyncio +from app.api.v1.endpoints import analyze as analyze_endpoint +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineStages, + DbIndependentPipelineSuccess, +) +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import PipelineStageTimings, PipelineTimings, Verdict +from app.schemas.unchain import UnchainResult +from fastapi import FastAPI +from httpx import ASGITransport + + +@pytest_asyncio.fixture +async def client() -> AsyncIterator[httpx.AsyncClient]: + app = FastAPI() + app.include_router(analyze_endpoint.router, prefix=settings.api_v1_prefix) + transport = ASGITransport(app=app) + headers = {"X-Internal-Api-Key": settings.internal_api_key} + async with httpx.AsyncClient( + transport=transport, + base_url="http://test", + headers=headers, + ) as c: + yield c + + +def _success_result() -> DbIndependentPipelineSuccess: + final_url = "https://example.com/login" + return DbIndependentPipelineSuccess( + analysis_id="aid-api", + original_url=final_url, + final_url=final_url, + verdict=Verdict.SAFE, + score=0, + timings=PipelineTimings( + total_seconds=0.001, + stages=PipelineStageTimings( + normalize=0.001, + unchain=0.001, + domain_heuristic=0.001, + content_analysis=0.001, + ), + ), + stages=DbIndependentPipelineStages( + normalize=NormalizeResult(original_url=final_url, normalized_url=final_url), + unchain=UnchainResult(input_url=final_url, final_url=final_url), + domain_heuristic=DomainHeuristicResult( + domain="example.com", + score=0, + signals=[], + ), + content_analysis=ContentAnalysisResult( + final_url=final_url, + fetched=True, + score=0, + signals=[], + ), + ), + ) + + +async def test_analyze_db_independent_sync_returns_result( + client: httpx.AsyncClient, + monkeypatch, +) -> None: + mock_run = AsyncMock(return_value=_success_result()) + monkeypatch.setattr(analyze_endpoint, "run_db_independent_pipeline", mock_run) + + resp = await client.post( + f"{settings.api_v1_prefix}/analyze/db-independent/sync", + json={"url": "https://example.com/login"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "success" + assert body["verdict"] == "safe" + assert "threat_db" not in body["stages"] + mock_run.assert_awaited_once() + assert mock_run.await_args.kwargs["original_url"] == "https://example.com/login" + + +async def test_analyze_db_independent_sync_requires_internal_key() -> None: + app = FastAPI() + app.include_router(analyze_endpoint.router, prefix=settings.api_v1_prefix) + transport = ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as c: + resp = await c.post( + f"{settings.api_v1_prefix}/analyze/db-independent/sync", + json={"url": "https://example.com/login"}, + ) + + assert resp.status_code == 401 diff --git a/tests/api/test_stages.py b/tests/api/test_stages.py new file mode 100644 index 0000000..756510c --- /dev/null +++ b/tests/api/test_stages.py @@ -0,0 +1,178 @@ +"""/api/v1/* 단계별 운영 엔드포인트 테스트. + +X-Internal-Api-Key 인증 + 단계 단독 호출 동작을 검증한다. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from unittest.mock import AsyncMock, patch + +import httpx +import pytest_asyncio +from app.api.v1.endpoints import stages as stages_endpoint +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict, ContentAnalysisResult, ContentSignal +from app.services.content_analyzer.fetch import FetchResult +from fastapi import FastAPI +from httpx import ASGITransport + + +@pytest_asyncio.fixture +async def client() -> AsyncIterator[httpx.AsyncClient]: + """stages 라우터만 마운트한 얇은 FastAPI 앱 — lifespan / DB / 스케줄러를 건드리지 않는다.""" + app = FastAPI() + app.include_router(stages_endpoint.router, prefix=settings.api_v1_prefix) + transport = ASGITransport(app=app) + headers = {"X-Internal-Api-Key": settings.internal_api_key} + async with httpx.AsyncClient( + transport=transport, base_url="http://test", headers=headers + ) as c: + yield c + + +_ENDPOINT_FETCH = f"{settings.api_v1_prefix}/content/fetch-extract" +_ENDPOINT_ANALYSIS = f"{settings.api_v1_prefix}/content-analysis" + + +class TestFetchExtractEndpoint: + async def test_returns_fetch_and_features_on_success(self, client: httpx.AsyncClient) -> None: + html = ( + "NAVER" + '
' + 'NAVER logo' + ) + fetch = FetchResult( + ok=True, + url="https://evil-naver.test/", + status_code=200, + html=html, + ) + with patch( + "app.api.v1.endpoints.stages.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://evil-naver.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["url"] == "https://evil-naver.test/" + assert body["fetch"]["ok"] is True + assert body["fetch"]["status_code"] == 200 + assert body["fetch"]["html_length"] == len(html) + assert body["features"]["title"] == "NAVER" + assert body["features"]["has_password_field"] is True + assert body["features"]["image_alts"] == ["NAVER logo"] + assert body["features"]["form_field_summaries"] + assert body["html_preview"] is not None + assert "NAVER" in body["html_preview"] + + async def test_fetch_failure_omits_features(self, client: httpx.AsyncClient) -> None: + fetch = FetchResult(ok=False, url="https://down.test/", error="connect_error") + with patch( + "app.api.v1.endpoints.stages.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://down.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["fetch"]["ok"] is False + assert body["fetch"]["error"] == "connect_error" + assert body["features"] is None + assert body["html_preview"] is None + + async def test_html_preview_caps_bytes(self, client: httpx.AsyncClient) -> None: + big = "" + ("x" * 10000) + "" + fetch = FetchResult(ok=True, url="https://big.test/", status_code=200, html=big) + with patch( + "app.api.v1.endpoints.stages.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://big.test/"}) + + preview = resp.json()["html_preview"] + assert preview is not None + # 2048 bytes 상한 — 그 이하로 잘려야 한다 + assert len(preview) <= 2048 + + async def test_missing_url_returns_422(self, client: httpx.AsyncClient) -> None: + resp = await client.post(_ENDPOINT_FETCH, json={}) + assert resp.status_code == 422 + + async def test_invalid_scheme_rejected(self, client: httpx.AsyncClient) -> None: + """file:// 같은 비허용 스킴은 normalize_url 에서 400 으로 떨어진다.""" + resp = await client.post(_ENDPOINT_FETCH, json={"url": "file:///etc/passwd"}) + assert resp.status_code == 400 + assert "invalid url" in resp.json()["detail"] + + +class TestContentAnalysisEndpoint: + async def test_returns_full_content_analysis(self, client: httpx.AsyncClient) -> None: + expected = ContentAnalysisResult( + final_url="https://x.test/", + fetched=True, + score=80, + signals=[ContentSignal.BRAND_IMPERSONATION_FORM], + title="NAVER", + has_password_field=True, + brand_impersonation=True, + ai_verdict=AIVerdict.PHISHING, + ai_reason="브랜드 위장", + ) + with patch( + "app.api.v1.endpoints.stages.analyze_content", + AsyncMock(return_value=expected), + ): + resp = await client.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["final_url"] == "https://x.test/" + assert body["score"] == 80 + assert body["signals"] == ["BRAND_IMPERSONATION_FORM"] + assert body["ai_verdict"] == "phishing" + assert body["ai_reason"] == "브랜드 위장" + + async def test_does_not_skip_based_on_preceding_score( + self, client: httpx.AsyncClient + ) -> None: + """파이프라인과 달리 단계 단독 엔드포인트는 항상 실제 분석을 수행한다.""" + expected = ContentAnalysisResult( + final_url="https://x.test/", + fetched=False, + score=0, + signals=[], + error="connect_error", + ) + with patch( + "app.api.v1.endpoints.stages.analyze_content", + AsyncMock(return_value=expected), + ) as mock_analyze: + await client.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + + mock_analyze.assert_awaited_once_with("https://x.test/") + + +class TestAuthRequired: + """모든 stages 엔드포인트는 X-Internal-Api-Key 헤더 인증이 필수다.""" + + async def test_missing_header_rejected_401(self) -> None: + app = FastAPI() + app.include_router(stages_endpoint.router, prefix=settings.api_v1_prefix) + transport = ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as c: + resp = await c.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + assert resp.status_code == 401 + + async def test_wrong_key_rejected_401(self) -> None: + app = FastAPI() + app.include_router(stages_endpoint.router, prefix=settings.api_v1_prefix) + transport = ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, + base_url="http://test", + headers={"X-Internal-Api-Key": "wrong-key"}, + ) as c: + resp = await c.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + assert resp.status_code == 401 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..4b61e53 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +"""공용 테스트 픽스처.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING + +os.environ.setdefault("TLDEXTRACT_CACHE", "/private/tmp/linclean-tldextract-cache") + +import pytest +import pytest_asyncio + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +@pytest.fixture(autouse=True) +def _disable_scheduler(monkeypatch: pytest.MonkeyPatch) -> None: + """테스트에서는 APScheduler 기동 금지.""" + from app.core.config import settings + + monkeypatch.setattr(settings, "scheduler_enabled", False) + + +@pytest_asyncio.fixture +async def async_session() -> AsyncIterator[AsyncSession]: + """인메모리 SQLite 세션 — 테스트마다 새 DB.""" + # 모델 로드는 DB fixture 가 필요한 테스트에서만 수행한다. + import app.models # noqa: F401 + from app.db.base import Base + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + session_maker = async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False) + async with session_maker() as session: + yield session + + await engine.dispose() diff --git a/tests/core/test_config.py b/tests/core/test_config.py new file mode 100644 index 0000000..7fee82d --- /dev/null +++ b/tests/core/test_config.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from app.core.config import Settings + + +def test_rdap_cache_ttl_defaults_to_one_week() -> None: + settings = Settings(_env_file=None, internal_api_key="test-key") + + assert settings.rdap_cache_ttl_seconds == 60 * 60 * 24 * 7 diff --git a/tests/core/test_tld.py b/tests/core/test_tld.py new file mode 100644 index 0000000..965f6d8 --- /dev/null +++ b/tests/core/test_tld.py @@ -0,0 +1,21 @@ +from __future__ import annotations + + +def test_extract_url_parts_handles_multi_label_suffix() -> None: + from app.core.tld import extract_url_parts + + result = extract_url_parts("https://login.example.co.kr/path") + + assert result.domain == "example" + assert result.suffix == "co.kr" + assert result.subdomain == "login" + + +def test_extract_url_parts_handles_private_hosting_suffix() -> None: + from app.core.tld import extract_url_parts + + result = extract_url_parts("https://shop-login.vercel.app/path") + + assert result.domain == "shop-login" + assert result.suffix == "vercel.app" + assert result.top_domain_under_public_suffix == "shop-login.vercel.app" diff --git a/tests/corpus/urls.yaml b/tests/corpus/urls.yaml new file mode 100644 index 0000000..fe8b48e --- /dev/null +++ b/tests/corpus/urls.yaml @@ -0,0 +1,146 @@ +- id: mju-sso-login + name: "명지대학교 통합로그인 SSO 클론" + source_url: "https://sso.mju.ac.kr/sso/auth?response_type=code&client_id=lms&state=Random%20String&redirect_uri=https://lms.mju.ac.kr/ilos/sso/sso_response.jsp" + url_env: "TEST_SITE_MJU_SSO_LOGIN_URL" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/mju-sso-login" + category: "legitimate-login-page" + purpose: "false-positive-regression" + expected_verdict: "safe" + notes: + - "Static test fixture with authentication disabled." + - "Use deployed URL via TEST_SITE_MJU_SSO_LOGIN_URL for end-to-end engine tests." + +- id: obituary-kakao-apk-download + name: "故 홍길동 카카오부고 APK 다운로드 유도" + source_url: "https://kakaobugo.com/m/691?mode=sample" + url_env: "TEST_SITE_OBITUARY_KAKAO_APK_DOWNLOAD_URL" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/obituary-kakao-apk-download" + category: "social-engineering-apk-download" + purpose: "malicious-download-regression" + expected_verdict: "malicious" + notes: + - "Static KakaoBugo-style obituary fixture for 故 홍길동." + - "Condolence flower button opens a KakaoTalk-themed APK download prompt." + - "Downloaded .apk is a harmless text fixture, not an Android package." + - "Use deployed URL via TEST_SITE_OBITUARY_KAKAO_APK_DOWNLOAD_URL for end-to-end engine tests." + +- id: nhis-fuel-subsidy-check + name: "고유가 피해지원금 대상 여부 조회 NHIS 사칭 폼" + source_url: "https://www.nhis.or.kr/" + url_env: "TEST_SITE_NHIS_FUEL_SUBSIDY_CHECK_URL" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/nhis-fuel-subsidy-check" + category: "public-agency-impersonation-pii-form" + purpose: "true-positive-regression" + expected_verdict: "phishing" + notes: + - "Static NHIS-style eligibility lookup fixture with PII submission disabled." + - "Contains name and resident registration number fields for detector regression coverage." + - "Use deployed URL via TEST_SITE_NHIS_FUEL_SUBSIDY_CHECK_URL for end-to-end engine tests." + +- id: naver-impersonation-login + name: "NAVER 사칭 로그인 폼 클론" + source_url: "https://nid.naver.com/nidlogin.login?mode=form&url=https://www.naver.com/" + url_env: "TEST_SITE_NAVER_IMPERSONATION_LOGIN_URL" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/naver-impersonation-login" + category: "brand-impersonation-login-page" + purpose: "true-positive-regression" + expected_verdict: "danger" + expected_signals: + - "BRAND_IMPERSONATION_FORM" + - "LOGO_ALT_IMPERSONATION" + - "CREDENTIAL_FORM_EXTERNAL" + notes: + - "Static NAVER-like login fixture with hidden non-visual test marker." + - "Credential form uses a dummy external action and client-side submit blocking." + - "Use deployed URL via TEST_SITE_NAVER_IMPERSONATION_LOGIN_URL for end-to-end engine tests." + +- id: redirect-chain-single-safe + name: "단일 리다이렉트 후 정상 최종 페이지" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/1" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "unchainer-regression" + expected_verdict: "safe" + expected_unchain: + final_path: "/final/safe" + signals_absent: + - "max_hops_reached" + - "unsafe_scheme" + - "redirect_loop" + notes: + - "TEST_SITE_REDIRECT_CHAIN_URL과 path를 결합해 테스트 URL을 만든다." + - "각 redirect 응답은 0.5초 지연 후 반환된다." + +- id: redirect-chain-multi-hop-safe + name: "다중 hop 리다이렉트 후 정상 최종 페이지" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/3" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "unchainer-regression" + expected_verdict: "safe" + expected_unchain: + final_path: "/final/safe" + required_intermediate_paths: + - "/hop/1" + - "/hop/2" + signals_absent: + - "max_hops_reached" + - "unsafe_scheme" + - "redirect_loop" + +- id: redirect-chain-external + name: "외부 테스트 도메인으로 이동하는 리다이렉트" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/external" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "cross-origin-regression" + expected_verdict: "safe" + expected_unchain: + final_url: "https://example.com/" + signals_present: + - "cross_origin" + notes: + - "실제 악성 도메인이 아닌 example.com만 사용한다." + +- id: redirect-chain-unsafe-scheme + name: "비허용 scheme 리다이렉트 차단" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/unsafe-scheme" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "unsafe-scheme-regression" + expected_verdict: "warning" + expected_unchain: + error: "unsafe_redirect_scheme:javascript" + signals_present: + - "unsafe_scheme:javascript" + +- id: redirect-chain-max-hop + name: "최대 hop 수 초과 리다이렉트" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/max-hop" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "max-hop-regression" + expected_verdict: "warning" + expected_unchain: + signals_present: + - "max_hops_reached" + default_max_hops: 5 + +- id: redirect-chain-to-suspicious + name: "리다이렉트 후 의심 최종 페이지" + url_env: "TEST_SITE_REDIRECT_CHAIN_URL" + path: "/redirect/to-suspicious" + fixture_repo: "https://github.com/403-Forb2dden/LinClean-Test-Sites/tree/main/redirect-chain" + category: "redirect-chain" + purpose: "final-page-classification-regression" + expected_verdict: "warning" + expected_unchain: + final_path: "/final/suspicious" + notes: + - "최종 페이지는 의심 문구와 비활성 카드 인증 폼을 포함한다." + - "카드번호, CVC, 만료일, 카드 비밀번호 앞 2자리 입력 UI는 readOnly이며 실제 수집이나 다운로드가 없다." diff --git a/tests/demo/__init__.py b/tests/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/demo/demo_content_analysis.py b/tests/demo/demo_content_analysis.py new file mode 100644 index 0000000..2bf0669 --- /dev/null +++ b/tests/demo/demo_content_analysis.py @@ -0,0 +1,107 @@ +"""페이지 콘텐츠 정적 분석 데모. + +사용법: + python -m tests.demo.demo_content_analysis + +실제 네트워크로 HTML 을 취득해 BS4 로 파싱하고 규칙 기반 점수를 계산한다. +AI 어댑터는 기본값 `NullAIProvider` — `OPENAI_API_KEY` 환경변수가 있으면 +이 스크립트가 `OpenAIProvider` 로 교체해서 gpt-4o-mini 추론도 같이 보여준다. + +외부 사이트가 응답하지 않으면 해당 케이스는 `fetched=False`, `error=...` 로 +degraded 반환되고 파이프라인은 계속 돈다 (의도된 동작). +""" + +from __future__ import annotations + +import asyncio + +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult +from app.services.content_analyzer import analyze_content +from app.services.content_analyzer.ai import set_ai_provider +from app.services.content_analyzer.ai_openai import OpenAIProvider + +DIVIDER = "-" * 72 + + +CASES: list[tuple[str, str]] = [ + # --- 정상 페이지 --- + ("정상 페이지 (example.com)", "https://example.com/"), + ("자기 브랜드 도메인 (naver.com)", "https://www.naver.com/"), + # --- 페치/파싱 경로 점검 --- + ("HTML 샘플 (httpbin)", "https://httpbin.org/html"), + ("비-HTML content-type (favicon)", "https://www.google.com/favicon.ico"), + # --- degraded 경로 --- + ( + "존재하지 않는 호스트 (connect_error 기대)", + "https://this-host-definitely-does-not-exist-xyzabc-123.test/", + ), + # --- 리다이렉트 방지 동작 확인 --- + ( + "3xx 응답 (unexpected_redirect — unchainer 가 먼저 해소했어야 하는 케이스)", + "http://google.com/", + ), +] + + +def _fmt_ratio(r: float | None) -> str: + return "-" if r is None else f"{r:.2f}" + + +def _print_result(label: str, url: str, result: ContentAnalysisResult) -> None: + print(f" input : {url}") + print(f" fetched : {result.fetched}") + if not result.fetched: + print(f" error : {result.error}") + else: + print(f" title : {result.title!r}") + print(f" password_form : {result.has_password_field}") + print(f" meta_refresh : {result.has_meta_refresh}") + print(f" ext_link_ratio: {_fmt_ratio(result.external_link_ratio)}") + print(f" brand_imperson: {result.brand_impersonation}") + print(f" alt_imperson : {result.logo_alt_impersonation}") + print(f" score : {result.score}") + print(f" signals : {[s.value for s in result.signals]}") + verdict = result.ai_verdict.value if result.ai_verdict else None + print(f" ai_verdict : {verdict} (reason={result.ai_reason!r})") + if result.ai_error: + print(f" ai_error : {result.ai_error}") + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Content Analysis Demo") + print("=" * 72) + print() + + provider: OpenAIProvider | None = None + if settings.openai_api_key: + provider = OpenAIProvider() + set_ai_provider(provider) + print(f" AI: OpenAIProvider ({settings.openai_model})") + else: + print(" AI: NullAIProvider (OPENAI_API_KEY 미설정 — ai_verdict 는 None)") + print(f" danger threshold (skip 기준): {settings.score_danger_threshold}") + print() + + try: + for i, (desc, url) in enumerate(CASES, 1): + print(f" [{i:02d}] {desc}") + try: + result = await analyze_content(url) + except Exception as exc: # 데모에서만 — 운영 코드는 파이프라인이 degraded 처리 + print(f" !! 예외: {type(exc).__name__}: {exc}") + print(DIVIDER) + continue + _print_result(desc, url, result) + print(DIVIDER) + finally: + if provider is not None: + await provider.aclose() + + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/demo/demo_domain_heuristic.py b/tests/demo/demo_domain_heuristic.py new file mode 100644 index 0000000..57cd8b0 --- /dev/null +++ b/tests/demo/demo_domain_heuristic.py @@ -0,0 +1,84 @@ +""" +RDAP 조회는 실제 네트워크(`rdap.org` bootstrap)로 나감. +환경에 따라 일부 도메인은 RDAP 서버가 응답하지 않을 수 있으며, +그 경우 `rdap_error` 만 찍히고 파이프라인은 그대로 진행됨. +""" + +from __future__ import annotations + +import asyncio + +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.services.domain_heuristic import check_domain_heuristic +from app.services.domain_heuristic.rdap import aclose_client + +DIVIDER = "-" * 72 + + +CASES: list[tuple[str, str]] = [ + # --- 정상 도메인 (신호 없음 예상) --- + ("정상 도메인 (google)", "https://www.google.com/"), + ("정상 도메인 (naver)", "https://www.naver.com/"), + # --- 패턴 기반 신호 --- + ("IP 직접 접근", "http://192.168.1.1/login"), + ("HTTPS 미사용", "http://example.com/"), + ("의심 TLD (.xyz)", "https://free-money.xyz/"), + ("의심 TLD (.zip)", "https://invoice.zip/"), + ("하이픈 과다 + 긴 레이블", "https://login-secure-naver-auth-verify.com/"), + ("서브도메인 과다 중첩", "https://signin.auth.account.verify.example.com/"), + ("Punycode / IDN", "https://xn--nver-9na.com/"), + ("오픈 리다이렉트 파라미터", "https://example.com/go?url=http://evil.xyz"), + # --- 호스팅 플랫폼 (서브도메인 경유 vs 플랫폼 루트) --- + ("호스팅 플랫폼 경유 (github.io)", "https://someuser.github.io/repo/"), + ("호스팅 플랫폼 경유 (netlify.app)", "https://myapp.netlify.app/"), + ("호스팅 플랫폼 루트 (typo 오탐 방어)", "https://netlify.app/"), + # --- 타이포스쿼팅 --- + ("타이포스쿼팅 (distance 1)", "https://naverr.com/"), + ("타이포스쿼팅 (suffix 상이)", "https://naver.net/"), + # --- DGA 유사 --- + ("DGA 유사 (무작위 문자열)", "https://xjqkzvnpwmrbtld.com/"), + # --- RDAP 신규 도메인 (실제 신규 등록 도메인이 있을 때만 NEW_DOMAIN 발동) --- + ("RDAP 조회 (기성 도메인)", "https://www.cloudflare.com/"), +] + + +def _fmt_rdap(result: DomainHeuristicResult) -> str: + if result.rdap_error: + return f"error={result.rdap_error}" + if result.rdap is None: + return "없음" + rdap = result.rdap + return ( + f"registrar={rdap.registrar!r} " + f"age_days={rdap.domain_age_days} " + f"is_new={rdap.is_new_domain}" + ) + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Domain Heuristic Demo") + print("=" * 72) + + try: + for i, (desc, url) in enumerate(CASES, 1): + print(f"\n [{i:02d}] {desc}") + print(f" input : {url}") + + result = await check_domain_heuristic(url) + + print(f" domain : {result.domain}") + print(f" score : {result.score}") + print(f" signals: {[s.value for s in result.signals]}") + print(f" rdap : {_fmt_rdap(result)}") + print(DIVIDER) + finally: + # 데모 종료 시 RDAP httpx client 정리 (앱 lifespan 외부에서 실행되므로) + await aclose_client() + + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/demo/demo_normalize.py b/tests/demo/demo_normalize.py new file mode 100644 index 0000000..148544b --- /dev/null +++ b/tests/demo/demo_normalize.py @@ -0,0 +1,80 @@ +"""URL 정규화 데모. + +사용법: + python -m tests.demo.demo_normalize +""" + +from __future__ import annotations + +from app.core.exceptions import NormalizationError +from app.services.normalizer import normalize_url + +DIVIDER = "-" * 72 + +CASES: list[tuple[str, str]] = [ + ("Scheme/host lowercasing, path case preserved", "HTTP://EXAMPLE.COM/Path/To/Page"), + ("Default port removal (80, 443)", "https://example.com:443/secure"), + ("Non-default port kept", "https://example.com:8080/api"), + ("Fragment removal", "https://example.com/page#section-2"), + ("Missing scheme defaults to https", "example.com/hello"), + ("Unreserved pct-decode (%7E->~)", "https://example.com/%7Euser/%41%42"), + ("Reserved hex uppercased (%2f->%2F)", "https://example.com/a%2fb%3fc"), + ("Dot-segment resolution (/../)", "https://example.com/a/b/../c/./d"), + ("Consecutive slash collapse", "https://example.com//a///b"), + ("Userinfo stripped (phishing vector)", "https://google.com@evil.com/login"), + ("Params pct-encoding normalized", "https://example.com/path;%7eparam?q=1"), + ("IDN unicode -> punycode", "https://xn--n3h.com/path"), + ("IDN punycode passthrough", "https://xn--n3h.com/path"), + ("Control character removal", "https://exam\x00ple.com/pa\x01th"), + ("Whitespace trimming", " https://example.com/page "), + ("Combined", " HTTP://EXAMPLE.COM:80/a/b/../c/./d?q=%7e&x=%2f#top "), +] + +ERROR_CASES: list[tuple[str, str]] = [ + ("Empty string", ""), + ("Whitespace only", " "), + ("Unsupported scheme", "ftp2://example.com"), + ("No host", "https://"), + ("Exceeds max length", "https://example.com/" + "a" * 1024), +] + + +def main() -> None: + print() + print("=" * 72) + print(" LinClean -- URL Normalization Demo") + print("=" * 72) + + print("\n[Success cases]") + print(DIVIDER) + + for i, (desc, raw) in enumerate(CASES, 1): + result = normalize_url(raw) + display_input = repr(raw) if any(ord(c) < 32 for c in raw) else raw + print(f" [{i:02d}] {desc}") + print(f" input : {display_input}") + print(f" original : {result.original_url}") + print(f" normalized : {result.normalized_url}") + print(DIVIDER) + + print("\n[Error cases (NormalizationError)]") + print(DIVIDER) + + for i, (desc, raw) in enumerate(ERROR_CASES, 1): + display_input = repr(raw) if len(raw) > 60 else (repr(raw) if not raw.strip() else raw) + try: + normalize_url(raw) + print(f" [{i:02d}] {desc}") + print(f" input : {display_input}") + print(" result : WARN -- expected error not raised!") + except NormalizationError as e: + print(f" [{i:02d}] {desc}") + print(f" input : {display_input}") + print(f" error : {e.message}") + print(DIVIDER) + + print() + + +if __name__ == "__main__": + main() diff --git a/tests/demo/demo_threat_db.py b/tests/demo/demo_threat_db.py new file mode 100644 index 0000000..3cffa38 --- /dev/null +++ b/tests/demo/demo_threat_db.py @@ -0,0 +1,102 @@ +"""외부 위협 DB 대조 데모. + +사용법: + alembic upgrade head # 최초 1회 + python -m tests.demo.demo_threat_db + +환경변수 `GSB_API_KEY` 가 설정돼 있으면 실제 GSB Lookup API 호출. +없으면 GSB 는 `checked=False, error=not_configured` 로 반환된다. + +URLhaus 는 로컬 SQLite 스냅샷 기반이므로 실행 첫 부분에서 1회 +`sync_urlhaus()` 를 호출해 CSV 를 수신한다. +""" + +from __future__ import annotations + +import asyncio + +from app.db.session import SessionLocal +from app.models.urlhaus_entry import URLhausEntry +from app.schemas.analysis import ThreatDbResult +from app.services.threat_db import check_threat_db +from app.services.threat_db.urlhaus_sync import sync_urlhaus +from sqlalchemy import func, select + +DIVIDER = "-" * 72 + + +# GSB 공식 테스트 URL — 실제 악성이 아니지만 Lookup API 가 항상 MALWARE 로 +# 매치시켜 주는 시그니처. GSB_API_KEY 가 있어야 threat=True 로 반환된다. +GSB_TEST_URL = "http://malware.testing.google.test/testing/malware/" + + +async def _pick_urlhaus_sample() -> str | None: + """동기화된 DB 에서 URLhaus 등재 URL 하나를 랜덤 추출. + + 등재 URL 은 CSV 싱크 시점마다 달라지므로 하드코딩 불가 — 방금 받은 + 스냅샷에서 아무거나 하나 뽑아서 실제 매치가 일어나는지 시연한다. + """ + async with SessionLocal() as session: + stmt = select(URLhausEntry.url).order_by(func.random()).limit(1) + row = (await session.execute(stmt)).scalar_one_or_none() + return row + + +def _print_result(label: str, url: str, result: ThreatDbResult) -> None: + print(DIVIDER) + print(f"[{label}]") + print(f" input : {url}") + print(f" malicious : {result.is_malicious}") + print(f" sources_check : {result.sources_checked}") + print(f" threat_types : {result.threat_types}") + print( + f" gsb : checked={result.gsb.checked} " + f"threat={result.gsb.is_threat} error={result.gsb.error}" + ) + print( + f" urlhaus : checked={result.urlhaus.checked} " + f"threat={result.urlhaus.is_threat} " + f"match_type={result.urlhaus.match_type} " + f"matched_key={result.urlhaus.matched_key} " + f"error={result.urlhaus.error}" + ) + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Threat DB (GSB + URLhaus) Demo") + print("=" * 72) + print() + + print(">>> 1) URLhaus 스냅샷 동기화 (수 MB CSV 다운로드·파싱 — 실제 수십 초 소요)") + # stats = {inserted, updated, total, failed} — 청크(500행) 단위 커밋이라 + # 중간에 깨져도 직전 청크까지는 영속화되고 그만큼만 집계된다. + stats = await sync_urlhaus() + print(f" stats: {stats}") + print() + + urlhaus_sample = await _pick_urlhaus_sample() + + cases: list[tuple[str, str]] = [ + # 1) 둘 다 미매치 — 정상 URL + ("둘 다 미매치 (정상 URL)", "https://www.google.com/"), + # 2) 위협 DB 에 없는 평범한 샘플 도메인 + ("위험 DB 미등재 샘플 도메인", "https://example.com/"), + # 3) GSB 공식 테스트 URL — API 키 있으면 항상 MALWARE 매치 + ("GSB 공식 테스트 URL (MALWARE 매치)", GSB_TEST_URL), + ] + if urlhaus_sample: + cases.append(("URLhaus 로컬 스냅샷 등재 URL", urlhaus_sample)) + else: + print(" (URLhaus 샘플이 비어있어 URLhaus 매치 케이스는 건너뜀)") + + async with SessionLocal() as session: + for label, url in cases: + result = await check_threat_db(session, url) + _print_result(label, url, result) + print(DIVIDER) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/demo/demo_unchain.py b/tests/demo/demo_unchain.py new file mode 100644 index 0000000..c7b6ea9 --- /dev/null +++ b/tests/demo/demo_unchain.py @@ -0,0 +1,79 @@ +"""URL 언체이닝 데모. + +사용법: + python -m tests.demo.demo_unchain +""" + +from __future__ import annotations + +import asyncio + +from app.services.unchainer import unchain_url + +DIVIDER = "-" * 72 + +# 실제 리다이렉트가 일어나는 공개 URL들 (데모 시점에 동작 보장 불가) +CASES: list[tuple[str, str]] = [ + # --- 기본 동작 --- + ("리다이렉트 없음 (200 OK)", "https://www.google.com/"), + ("HTTP→HTTPS 영구 이동 (301)", "http://github.com/"), + ("단축 URL (bit.ly → 실제 목적지)", "https://bit.ly/4tGFJkf"), + # --- 301 vs 302 차이 --- + ( + "301 Permanent: 주소가 영구적으로 바뀜", + "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=301", + ), + ( + "302 Found: 임시로 다른 곳으로 보냄", + "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=302", + ), + # --- 다중 hop·경로 해석 --- + ("다중 hop 체인 (3회 리다이렉트)", "https://httpbin.org/redirect/3"), + ("상대 경로 Location 해석", "https://httpbin.org/relative-redirect/2"), + # --- 의심 신호 탐지 --- + ( + "스킴 다운그레이드 (HTTPS→HTTP)", + "https://httpbin.org/redirect-to?url=http://httpbin.org/get&status_code=302", + ), + ( + "크로스 오리진 (호스트 변경)", + "https://httpbin.org/redirect-to?url=https://www.google.com&status_code=302", + ), + # --- 에러 --- + ("존재하지 않는 도메인 (DNS 실패)", "https://this-domain-does-not-exist-12345.com/"), +] + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- URL Unchaining Demo") + print("=" * 72) + + for i, (desc, url) in enumerate(CASES, 1): + print(f"\n [{i:02d}] {desc}") + print(f" input : {url}") + + result = await unchain_url(url) + + print(f" final : {result.final_url}") + print(f" hops : {result.hop_count}") + + for j, hop in enumerate(result.hops): + loc = f" -> {hop.location}" if hop.location else "" + print(f" [{j}] {hop.method} {hop.status_code}{loc}") + + if result.signals: + print(f" signals: {result.signals}") + if result.error: + print(f" error : {result.error}") + if result.timed_out: + print(" ⚠ TIMED OUT") + + print(DIVIDER) + + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/content_analyzer/__init__.py b/tests/services/content_analyzer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/content_analyzer/test_ai.py b/tests/services/content_analyzer/test_ai.py new file mode 100644 index 0000000..5918a77 --- /dev/null +++ b/tests/services/content_analyzer/test_ai.py @@ -0,0 +1,58 @@ +"""ai.py — 프로바이더 Protocol + Null 기본 어댑터.""" + +from __future__ import annotations + +import pytest +from app.schemas.content_analysis import AIVerdict +from app.services.content_analyzer.ai import ( + AIInference, + AIPromptContext, + NullAIProvider, + get_ai_provider, + set_ai_provider, +) + + +@pytest.fixture(autouse=True) +def _reset_provider() -> None: + """각 테스트 종료 시 기본 NullAIProvider 로 복구.""" + yield + set_ai_provider(NullAIProvider()) + + +async def test_null_provider_returns_none() -> None: + ctx = AIPromptContext( + final_url="https://x.test/", + title="hi", + has_password_field=False, + has_meta_refresh=False, + image_alts=(), + external_link_ratio=None, + ) + result = await NullAIProvider().infer(ctx) + assert result is None + + +async def test_default_provider_is_null() -> None: + assert isinstance(get_ai_provider(), NullAIProvider) + + +async def test_set_custom_provider() -> None: + class FakeProvider: + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + return AIInference(verdict=AIVerdict.PHISHING, reason="mock") + + set_ai_provider(FakeProvider()) + provider = get_ai_provider() + ctx = AIPromptContext( + final_url="https://x.test/", + title=None, + has_password_field=False, + has_meta_refresh=False, + image_alts=(), + external_link_ratio=None, + ) + inference = await provider.infer(ctx) + assert inference is not None + assert inference.verdict == AIVerdict.PHISHING + assert inference.reason == "mock" diff --git a/tests/services/content_analyzer/test_ai_openai.py b/tests/services/content_analyzer/test_ai_openai.py new file mode 100644 index 0000000..c46ce72 --- /dev/null +++ b/tests/services/content_analyzer/test_ai_openai.py @@ -0,0 +1,293 @@ +"""OpenAIProvider — Chat Completions 기반 피싱 추론 어댑터.""" + +from __future__ import annotations + +import asyncio +import json +import sys +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict +from app.services.content_analyzer.ai import AIPromptContext +from app.services.content_analyzer.ai_openai import OpenAIProvider + + +@pytest.fixture(autouse=True) +def _set_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_api_key", "test-key") + + +def _ctx() -> AIPromptContext: + return AIPromptContext( + final_url="https://evil-naver.test/signin", + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=False, + image_alts=("NAVER",), + external_link_ratio=0.9, + ) + + +def _mock_http_response( + content: str, + *, + usage: dict[str, int] | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + raise_for_status=lambda: None, + json=lambda: { + "choices": [{"message": {"content": content}}], + "usage": usage, + }, + ) + + +def _mock_http_client( + content: str, + *, + usage: dict[str, int] | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + post=AsyncMock(return_value=_mock_http_response(content, usage=usage)), + aclose=AsyncMock(), + ) + + +def _usage(prompt: int, completion: int) -> dict[str, int]: + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": prompt + completion, + } + + +async def test_openai_returns_verdict_on_success() -> None: + payload = json.dumps({"verdict": "phishing", "reason": "브랜드 불일치"}) + client = _mock_http_client(payload, usage=_usage(120, 18)) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider().infer(_ctx()) + + assert result is not None + assert result.verdict == AIVerdict.PHISHING + assert result.reason == "브랜드 불일치" + assert result.model == settings.openai_model + assert result.token_usage is not None + assert result.token_usage.prompt_tokens == 120 + assert result.token_usage.completion_tokens == 18 + assert result.token_usage.total_tokens == 138 + + +async def test_openai_missing_usage_returns_none_token_usage() -> None: + """usage 가 비어 있으면 token_usage=None 으로 떨어뜨린다 (추론 자체는 성공).""" + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider().infer(_ctx()) + + assert result is not None + assert result.token_usage is None + + +async def test_openai_custom_model_overrides_settings() -> None: + """생성자에 model 을 넘기면 settings.openai_model 보다 우선한다 — 모델 비교용.""" + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider(model="gpt-4o").infer(_ctx()) + + request_json = client.post.await_args.kwargs["json"] + assert request_json["model"] == "gpt-4o" + assert result is not None + assert result.model == "gpt-4o" + + +async def test_openai_returns_none_without_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_api_key", None) + provider = OpenAIProvider() + assert await provider.infer(_ctx()) is None + + +async def test_openai_uses_configured_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_model", "gpt-4o-mini") + + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(_ctx()) + + request_json = client.post.await_args.kwargs["json"] + assert request_json["model"] == "gpt-4o-mini" + assert request_json["response_format"]["type"] == "json_schema" + assert request_json["response_format"]["json_schema"]["strict"] is True + assert request_json["max_tokens"] == settings.openai_max_output_tokens + + +async def test_openai_prompt_requests_short_expert_plain_korean_reason() -> None: + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(_ctx()) + + system_msg = client.post.await_args.kwargs["json"]["messages"][0]["content"] + assert "100자" in system_msg + assert "보안 전문가" in system_msg + assert "쉬운 한국어" in system_msg + assert "사용자 행동 가이드" in system_msg + assert "비밀번호" in system_msg + assert "결제" in system_msg + + +async def test_openai_truncates_reason_to_100_chars() -> None: + long_reason = "가" * 150 + client = _mock_http_client(json.dumps({"verdict": "suspicious", "reason": long_reason})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider().infer(_ctx()) + + assert result is not None + assert len(result.reason) == 100 + + +async def test_openai_uses_http_client_without_sdk_resource_import() -> None: + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + httpx_client_cls = MagicMock(return_value=client) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", httpx_client_cls): + await OpenAIProvider(timeout_seconds=3.0).infer(_ctx()) + + assert httpx_client_cls.call_args.kwargs["base_url"] == "https://api.openai.com/v1" + assert httpx_client_cls.call_args.kwargs["timeout"] == 3.0 + assert httpx_client_cls.call_args.kwargs["trust_env"] is False + assert client.post.await_args.args == ("/chat/completions",) + + +async def test_openai_request_has_hard_timeout() -> None: + async def never_returns(*_: object, **__: object) -> object: + await asyncio.sleep(10) + return _mock_http_response(json.dumps({"verdict": "benign", "reason": "late"})) + + client = SimpleNamespace(post=never_returns, aclose=AsyncMock()) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + started = time.perf_counter() + result = await OpenAIProvider(timeout_seconds=0.01).infer(_ctx()) + + assert result is None + assert time.perf_counter() - started < 0.5 + + +async def test_openai_provider_does_not_import_openai_sdk() -> None: + sys.modules.pop("openai", None) + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(_ctx()) + + assert "openai" not in sys.modules + + +async def test_openai_user_prompt_includes_upstream_signals() -> None: + """ctx.upstream_signals 가 user payload 의 upstream_signals 배열로 직렬화돼야 한다.""" + client = _mock_http_client( + json.dumps({"verdict": "phishing", "reason": "타이포 + 비밀번호 폼"}) + ) + ctx = AIPromptContext( + final_url="https://evil-naverr.test/signin", + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=False, + image_alts=("NAVER",), + external_link_ratio=0.9, + upstream_signals=("TYPO_DOMAIN", "NEW_DOMAIN"), + ) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(ctx) + + user_msg = client.post.await_args.kwargs["json"]["messages"][1]["content"] + body = json.loads(user_msg) + assert body["upstream_signals"] == ["TYPO_DOMAIN", "NEW_DOMAIN"] + + +async def test_openai_user_prompt_includes_high_signal_structured_context() -> None: + client = _mock_http_client( + json.dumps({"verdict": "suspicious", "reason": "민감정보 입력 유도"}) + ) + ctx = AIPromptContext( + final_url="https://nhis-support.test/", + title="고유가 피해지원금 대상 조회", + has_password_field=False, + has_meta_refresh=False, + image_alts=(), + external_link_ratio=None, + body_text_snippets=("국민건강보험 고유가 피해지원금 지급대상 여부 조회",), + form_field_summaries=( + "label=주민등록번호 name=resident_registration_number placeholder=주민등록번호", + ), + cta_texts=("지원금 대상 조회하기",), + download_links=(), + sensitive_field_types=("resident_registration_number",), + korean_lure_keywords=("지원금",), + public_agency_keywords=("국민건강보험",), + ) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(ctx) + + user_msg = client.post.await_args.kwargs["json"]["messages"][1]["content"] + body = json.loads(user_msg) + assert body["body_text"] == ["국민건강보험 고유가 피해지원금 지급대상 여부 조회"] + assert body["form_fields"] == [ + "label=주민등록번호 name=resident_registration_number placeholder=주민등록번호" + ] + assert body["cta_texts"] == ["지원금 대상 조회하기"] + assert body["sensitive_field_types"] == ["resident_registration_number"] + assert body["korean_lure_keywords"] == ["지원금"] + assert body["public_agency_keywords"] == ["국민건강보험"] + + +async def test_openai_user_prompt_omits_empty_upstream_signals() -> None: + """기본값(빈 튜플)이면 빈 배열로 직렬화돼 단독 페이지 분석 모드와 동일하게 동작.""" + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + await OpenAIProvider().infer(_ctx()) + + body = json.loads(client.post.await_args.kwargs["json"]["messages"][1]["content"]) + assert body["upstream_signals"] == [] + + +async def test_openai_parse_error_returns_none() -> None: + client = _mock_http_client("not-json") + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider().infer(_ctx()) + assert result is None + + +async def test_openai_unknown_verdict_returns_none() -> None: + """응답 JSON 의 verdict 값이 enum 범위를 벗어나면 None 으로 떨어뜨린다.""" + client = _mock_http_client(json.dumps({"verdict": "unknown_value", "reason": "x"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + assert await OpenAIProvider().infer(_ctx()) is None + + +async def test_openai_api_exception_returns_none() -> None: + client = SimpleNamespace(post=AsyncMock(side_effect=RuntimeError("5xx")), aclose=AsyncMock()) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + result = await OpenAIProvider().infer(_ctx()) + assert result is None + + +async def test_openai_cancelled_propagates() -> None: + client = SimpleNamespace( + post=AsyncMock(side_effect=asyncio.CancelledError()), + aclose=AsyncMock(), + ) + with ( + patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client), + pytest.raises(asyncio.CancelledError), + ): + await OpenAIProvider().infer(_ctx()) + + +async def test_openai_provider_closes_http_client_with_aclose() -> None: + client = _mock_http_client(json.dumps({"verdict": "benign", "reason": "ok"})) + with patch("app.services.content_analyzer.ai_openai.httpx.AsyncClient", return_value=client): + provider = OpenAIProvider() + await provider.infer(_ctx()) + await provider.aclose() + + client.aclose.assert_awaited_once() diff --git a/tests/services/content_analyzer/test_analyze.py b/tests/services/content_analyzer/test_analyze.py new file mode 100644 index 0000000..3bdae78 --- /dev/null +++ b/tests/services/content_analyzer/test_analyze.py @@ -0,0 +1,520 @@ +"""analyze_content — fetch · extract · signals · AI 통합.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict, ContentSignal, TokenUsage +from app.services.content_analyzer.ai import AIInference, AIPromptContext, NullAIProvider +from app.services.content_analyzer.analyze import analyze_content, skipped_already_danger +from app.services.content_analyzer.fetch import FetchResult +from app.services.content_analyzer.render import RenderResult + + +def _mock_fetch(ok: bool, html: str = "", error: str | None = None, status: int = 200): + return patch( + "app.services.content_analyzer.analyze.fetch_page", + AsyncMock( + return_value=FetchResult( + ok=ok, + url="https://x.test/", + status_code=status, + html=html, + error=error, + ) + ), + ) + + +class TestHappyPath: + async def test_clean_page_returns_zero_score(self) -> None: + html = "Hello

ok

" + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://benign.test/") + + assert result.fetched is True + assert result.final_url == "https://benign.test/" + assert result.score == 0 + assert result.signals == [] + assert result.title == "Hello" + assert result.has_password_field is False + assert result.error is None + + +class TestFetchFailure: + async def test_fetch_failure_degraded_result(self) -> None: + with _mock_fetch(ok=False, error="timeout"): + result = await analyze_content("https://down.test/") + + assert result.fetched is False + assert result.error == "timeout" + assert ContentSignal.FETCH_FAILED in result.signals + assert result.score == settings.score_weight_content_fetch_failed + assert result.ai_verdict is None + + async def test_http_404_fetch_failure_has_human_readable_reason(self) -> None: + with _mock_fetch(ok=False, error="http_error_404", status=404): + result = await analyze_content("https://missing.test/") + + assert result.fetched is False + assert result.error == "http_error_404" + assert result.reason == "페이지를 찾을 수 없습니다." + assert result.status_code == 404 + assert result.ai_verdict is None + + async def test_success_exposes_final_url_status_code(self) -> None: + with _mock_fetch(ok=True, html="", status=204): + result = await analyze_content("https://ok.test/") + + assert result.fetched is True + assert result.status_code == 204 + + @pytest.mark.parametrize("error", ["not_html", "too_large", "unexpected_redirect"]) + async def test_benign_fetch_errors_score_zero(self, error: str) -> None: + """정상 컨텐츠(PDF/이미지/대형 정적)·파이프라인 이슈는 시그널만 남기고 가산은 0.""" + with _mock_fetch(ok=False, error=error): + result = await analyze_content("https://x.test/") + + assert result.score == 0 + assert ContentSignal.FETCH_FAILED in result.signals + assert result.error == error + + async def test_fetch_failure_skips_extract_and_ai(self) -> None: + with ( + _mock_fetch(ok=False, error="connect_error"), + patch("app.services.content_analyzer.analyze.extract_features_async") as ext_mock, + patch("app.services.content_analyzer.analyze.score_content") as sig_mock, + patch("app.services.content_analyzer.analyze.get_ai_provider") as ai_mock, + ): + await analyze_content("https://x.test/") + + ext_mock.assert_not_called() + sig_mock.assert_not_called() + ai_mock.assert_not_called() + + def test_skipped_already_danger_has_fixed_user_reason(self) -> None: + result = skipped_already_danger("https://danger.test/") + + assert result.fetched is False + assert result.ai_reason is None + assert result.reason == "위험성이 확인된 URL입니다. 페이지를 열지 않는 것이 좋습니다." + assert result.error == "skipped_already_danger" + + +class TestBrandImpersonationEndToEnd: + async def test_brand_impersonation_form_scored(self) -> None: + html = """ + + NAVER 로그인 + +
+ NAVER + + + """ + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://evil-naver.test/signin") + + assert result.fetched is True + assert result.brand_impersonation is True + assert result.logo_alt_impersonation is True + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.title == "NAVER 로그인" + assert result.has_password_field is True + + +class TestHighSignalContextEndToEnd: + async def test_high_signal_features_surface_and_ai_sees_structured_summary( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.BENIGN, reason="local signals enough") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + html = """ + + 고유가 피해지원금 대상 조회 + +

국민건강보험 고유가 피해지원금 지급대상 여부 조회

+
+ + + +
+ + + """ + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://nhis-support.test/") + + assert result.body_text_snippets + assert "지원금 대상 조회하기" in result.cta_texts + assert "resident_registration_number" in result.sensitive_field_types + assert ContentSignal.PII_COLLECTION_FORM in result.signals + assert len(seen) == 1 + assert "지원금 대상 조회하기" in seen[0].cta_texts + assert "resident_registration_number" in seen[0].sensitive_field_types + assert "국민건강보험" in seen[0].public_agency_keywords + + async def test_ai_provider_still_called_once_with_extended_context( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + calls = 0 + + class CountingAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + nonlocal calls + calls += 1 + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="structured context") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CountingAI(), + ) + html = ( + '' + "카카오톡 최신버전 다운로드" + ) + with _mock_fetch(ok=True, html=html): + await analyze_content("https://obituary.test/") + + assert calls == 1 + + +class TestAI: + async def test_ai_phishing_verdict_adds_score(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.PHISHING, reason="brand mismatch") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + + html = "hi" + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.PHISHING + assert result.ai_reason == "brand mismatch" + assert result.score >= settings.score_weight_ai_phishing + + async def test_ai_suspicious_verdict(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="mild") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.SUSPICIOUS + assert result.score == settings.score_weight_ai_suspicious + assert result.score >= settings.score_caution_threshold + + async def test_ai_suspicious_on_trusted_clean_domain_does_not_score( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="mild") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html="Reddit"): + result = await analyze_content("https://www.reddit.com/") + + assert result.ai_verdict == AIVerdict.SUSPICIOUS + assert result.score == 0 + + @pytest.mark.parametrize("url", ["https://www.lg.co.kr/", "https://www.python.org/"]) + async def test_ai_suspicious_on_trusted_redirect_target_does_not_score( + self, + monkeypatch: pytest.MonkeyPatch, + url: str, + ) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="mild") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html="Trusted"): + result = await analyze_content(url) + + assert result.ai_verdict == AIVerdict.SUSPICIOUS + assert result.score == 0 + + async def test_ai_benign_verdict_no_score(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.BENIGN, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.BENIGN + assert result.score == 0 + + async def test_ai_none_keeps_nulls(self) -> None: + # 기본 provider 는 NullAIProvider → None 반환 + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict is None + assert result.ai_reason is None + assert result.ai_error is None + assert result.ai_model is None + assert result.ai_token_usage is None + + async def test_ai_token_usage_and_model_surface( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """프로바이더가 채워 준 model/token_usage 가 결과에 그대로 실려 나와야 한다.""" + + class TokenAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference( + verdict=AIVerdict.BENIGN, + reason="ok", + model="gpt-4o-mini", + token_usage=TokenUsage( + prompt_tokens=100, + completion_tokens=20, + total_tokens=120, + ), + ) + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: TokenAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_model == "gpt-4o-mini" + assert result.ai_token_usage is not None + assert result.ai_token_usage.total_tokens == 120 + + async def test_ai_exception_is_absorbed(self, monkeypatch: pytest.MonkeyPatch) -> None: + class BoomAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + raise RuntimeError("model exploded") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: BoomAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict is None + assert result.ai_error is not None + # 규칙 기반 결과는 여전히 유효 + assert result.fetched is True + + +class TestSpaShellEndToEnd: + async def test_spa_shell_flag_surfaces_and_ai_sees_it( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """SPA 셸 HTML 에 대해 ContentAnalysisResult.is_spa_shell=True 로 나오고, + AI 프로바이더의 프롬프트 컨텍스트에도 is_spa_shell=True 가 실려 전달된다.""" + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.BENIGN, reason="spa shell, no evidence") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + html = ( + 'x' + '' + '
' + ) + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://spa.test/") + + assert result.is_spa_shell is True + assert result.has_password_field is False + assert ContentSignal.SPA_SHELL in result.signals + # SPA 셸은 규칙 점수 가산 없음 — AI benign 이므로 최종 점수 0 + assert result.score == 0 + assert len(seen) == 1 + assert seen[0].is_spa_shell is True + + +class TestPrecisionSlowPath: + async def test_spa_shell_uses_rendered_dom_for_scoring( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """정적 HTML 이 SPA 셸뿐이어도 렌더링 DOM 에 민감 폼이 있으면 공통 룰로 재점수화한다.""" + + class BenignAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.BENIGN, reason="local evidence") + + static_shell = ( + '조회' + '' + '
' + ) + rendered_html = """ + + 고유가 피해지원금 대상 조회 + +

국민건강보험 고유가 피해지원금 지급대상 여부 조회

+
+ + + +
+ + + """ + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: BenignAI(), + ) + with ( + _mock_fetch(ok=True, html=static_shell), + patch( + "app.services.content_analyzer.analyze.render_page", + AsyncMock( + return_value=RenderResult( + ok=True, + url="https://spa.test/", + html=rendered_html, + ) + ), + ) as mock_render, + ): + result = await analyze_content("https://spa.test/") + + mock_render.assert_awaited_once_with("https://spa.test/") + assert result.precision_analysis_used is True + assert result.precision_analysis_error is None + assert result.is_spa_shell is False + assert ContentSignal.RENDERED_DOM_ANALYZED in result.signals + assert ContentSignal.PII_COLLECTION_FORM in result.signals + assert "resident_registration_number" in result.sensitive_field_types + assert result.score >= settings.score_danger_threshold + + async def test_clean_static_page_skips_precision_slow_path(self) -> None: + html = "Hello

ok

" + with ( + _mock_fetch(ok=True, html=html), + patch( + "app.services.content_analyzer.analyze.render_page", + new_callable=AsyncMock, + ) as mock_render, + ): + result = await analyze_content("https://benign.test/") + + mock_render.assert_not_awaited() + assert result.precision_analysis_used is False + assert result.precision_analysis_error is None + + +class TestCancelledError: + async def test_fetch_cancelled_propagates(self) -> None: + """shutdown/timeout 신호는 degraded 결과로 흡수하지 않고 re-raise.""" + with patch( + "app.services.content_analyzer.analyze.fetch_page", + AsyncMock(side_effect=asyncio.CancelledError()), + ), pytest.raises(asyncio.CancelledError): + await analyze_content("https://x.test/") + + async def test_ai_cancelled_propagates(self, monkeypatch: pytest.MonkeyPatch) -> None: + class CancelAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + raise asyncio.CancelledError() + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CancelAI(), + ) + with _mock_fetch(ok=True, html=""), pytest.raises(asyncio.CancelledError): + await analyze_content("https://x.test/") + + +async def test_null_provider_used_by_default() -> None: + """설정 없이 호출 시 NullAIProvider 가 쓰이고 AI 필드는 None.""" + from app.services.content_analyzer import ai as ai_module + + assert isinstance(ai_module.get_ai_provider(), NullAIProvider) + + +class TestUpstreamSignals: + """analyze_content 가 받은 upstream_signals 를 AI 프롬프트 컨텍스트로 그대로 전달해야 한다.""" + + async def test_upstream_signals_forwarded_to_provider( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + with _mock_fetch(ok=True, html="x"): + await analyze_content( + "https://x.test/", + upstream_signals=["TYPO_DOMAIN", "NEW_DOMAIN"], + ) + + assert len(seen) == 1 + assert seen[0].upstream_signals == ("TYPO_DOMAIN", "NEW_DOMAIN") + + async def test_upstream_signals_default_empty( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """파라미터를 생략하면 빈 튜플 — 디버그 엔드포인트처럼 선행 단계 없이 호출되는 경로 보호.""" + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.BENIGN, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + with _mock_fetch(ok=True, html=""): + await analyze_content("https://x.test/") + + assert seen[0].upstream_signals == () diff --git a/tests/services/content_analyzer/test_extract.py b/tests/services/content_analyzer/test_extract.py new file mode 100644 index 0000000..09bdcce --- /dev/null +++ b/tests/services/content_analyzer/test_extract.py @@ -0,0 +1,349 @@ +"""extract_features — BS4로 title, password, meta refresh, img alt, 외부 링크 비율 추출.""" + +from __future__ import annotations + +import pytest +from app.services.content_analyzer.extract import extract_features + + +class TestTitle: + def test_title_plain(self) -> None: + html = "NAVER 로그인" + features = extract_features(html, base_url="https://evil.test/") + assert features.title == "NAVER 로그인" + + def test_title_trimmed(self) -> None: + html = " Kakao " + features = extract_features(html, base_url="https://x.test/") + assert features.title == "Kakao" + + def test_no_title(self) -> None: + html = "hi" + features = extract_features(html, base_url="https://x.test/") + assert features.title is None + + +class TestPasswordField: + def test_has_password(self) -> None: + html = '
' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is True + + def test_no_password(self) -> None: + html = '
' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is False + + def test_password_type_case_insensitive(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is True + + def test_password_field_after_node_cap_still_detected(self) -> None: + """고위험 password 탐지는 일반 노드 순회 상한으로 우회되면 안 된다.""" + padding = "".join('' for _ in range(1500)) + html = f"{padding}" + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is True + + +class TestCredentialFormAction: + def test_password_form_external_action_detected(self) -> None: + html = ( + '
' + '' + "
" + ) + features = extract_features(html, base_url="https://example.test/login") + assert features.has_password_form_external_action is True + + def test_password_form_same_site_action_not_external(self) -> None: + html = '
' + features = extract_features(html, base_url="https://example.test/") + assert features.has_password_form_external_action is False + + +class TestMetaRefresh: + def test_has_meta_refresh(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is True + + def test_no_meta_refresh(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is False + + def test_meta_refresh_case_insensitive(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is True + + def test_external_meta_refresh_target_detected(self) -> None: + html = '' + features = extract_features(html, base_url="https://example.test/") + assert features.has_meta_refresh is True + assert features.has_external_meta_refresh is True + + def test_same_site_meta_refresh_target_not_external(self) -> None: + html = '' + features = extract_features(html, base_url="https://example.test/") + assert features.has_meta_refresh is True + assert features.has_external_meta_refresh is False + + +class TestImageAlts: + def test_collect_alts(self) -> None: + html = 'NAVERGoogle' + features = extract_features(html, base_url="https://x.test/") + assert features.image_alts == ["NAVER", "Google"] + + def test_empty_alts_ignored(self) -> None: + html = ' ' + features = extract_features(html, base_url="https://x.test/") + assert features.image_alts == [] + + def test_alts_capped_at_extraction(self) -> None: + """악성 페이지가 alt 수만 개를 박아 시그널 비용을 키우지 못하게 200 상한.""" + html = "".join(f'alt-{i}' for i in range(1000)) + features = extract_features(html, base_url="https://x.test/") + assert len(features.image_alts) == 200 + + +class TestExternalLinkRatio: + def test_all_internal(self) -> None: + html = """ + A + B + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 0.0 + + def test_all_external(self) -> None: + html = """ + F + B + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 1.0 + + def test_mixed(self) -> None: + html = """ + A + F + B + X + """ + features = extract_features(html, base_url="https://example.test/") + # 4개 중 2개 외부 + assert features.external_link_ratio == 0.5 + + def test_no_anchors_returns_none(self) -> None: + html = "

no links

" + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio is None + + def test_ignore_non_http_schemes(self) -> None: + """javascript:/mailto:/tel: 등은 네비게이션 링크가 아니므로 분모에서 제외.""" + html = """ + J + M + T + F + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 1.0 + + def test_relative_resolved_against_base(self) -> None: + html = 'S' + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 0.0 + + +class TestHighSignalStaticFeatures: + def test_extracts_body_text_form_fields_and_ctas(self) -> None: + html = """ + + 고유가 피해지원금 대상 조회 + +
+

국민건강보험 고유가 피해지원금 지급대상 여부 조회

+
+ + + + + +
+
+ + + """ + + features = extract_features(html, base_url="https://nhis-support.test/") + + assert any("고유가 피해지원금" in text for text in features.body_text_snippets) + assert any( + "resident_registration_number" in field + for field in features.form_field_summaries + ) + assert any("주민등록번호" in field for field in features.form_field_summaries) + assert "지원금 대상 조회하기" in features.cta_texts + assert "resident_registration_number" in features.sensitive_field_types + assert "phone" in features.sensitive_field_types + assert "지원금" in features.korean_lure_keywords + assert "국민건강보험" in features.public_agency_keywords + + def test_extracts_risky_download_links_and_lure_text(self) -> None: + html = """ + + +

故 홍길동님의 모바일 부고장을 카카오톡으로 확인하세요.

+ 카카오톡 최신버전 다운로드 + 일반 안내문 + + + """ + + features = extract_features(html, base_url="https://obituary.test/") + + assert features.download_links == ["https://obituary.test/downloads/kakaotalk.apk"] + assert "부고" in features.korean_lure_keywords + assert "카카오톡" in features.korean_lure_keywords + assert "카카오톡 최신버전 다운로드" in features.cta_texts + + def test_regular_js_anchor_is_not_risky_download(self) -> None: + html = """ + + + bundle + + + """ + + features = extract_features(html, base_url="https://normal.test/") + + assert features.download_links == [] + + def test_download_js_anchor_is_risky_download(self) -> None: + html = """ + + + download script + + + """ + + features = extract_features(html, base_url="https://suspicious.test/") + + assert features.download_links == ["https://suspicious.test/payload.js"] + + def test_extraction_caps_high_signal_lists(self) -> None: + inputs = "".join( + f'' + f'' + for i in range(200) + ) + links = "".join(f'download {i}' for i in range(200)) + html = f"
{inputs}
{links}" + + features = extract_features(html, base_url="https://x.test/") + + assert len(features.form_field_summaries) <= 80 + assert len(features.download_links) <= 40 + assert len(features.body_text_snippets) <= 40 + + +class TestSpaShellDetection: + def test_react_vite_shell_detected(self) -> None: + html = ( + 'x' + '' + '
' + ) + features = extract_features(html, base_url="https://x.test/") + assert features.is_spa_shell is True + assert features.has_password_field is False # 판정 불가 상태 + + def test_vue_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_nextjs_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_nuxt_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_svelte_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_angular_tag_shell_detected(self) -> None: + html = "" + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_ssr_page_with_form_not_flagged(self) -> None: + """SSR 결과로 이미 form 이 존재하면 id=root 가 있어도 판정 가능 상태 — 셸 아님.""" + html = ( + '
' + '
' + "
" + ) + features = extract_features(html, base_url="https://x.test/") + assert features.is_spa_shell is False + assert features.has_password_field is True + + def test_static_page_without_mount_point_not_flagged(self) -> None: + """마운트 컨테이너도 form 도 없는 일반 정적 페이지는 셸 아님 — 그냥 콘텐츠 없는 페이지.""" + html = "

just text

" + assert extract_features(html, base_url="https://x.test/").is_spa_shell is False + + def test_input_without_form_still_disqualifies(self) -> None: + """form 없이 input 만 있어도 '판정 불가' 는 아니다 — 정적 추출이 이미 결정적이다.""" + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is False + + def test_mount_id_case_insensitive(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + +class TestInvalidHtml: + def test_empty_html_does_not_raise(self) -> None: + features = extract_features("", base_url="https://x.test/") + assert features.title is None + assert features.has_password_field is False + assert features.has_password_form_external_action is False + assert features.has_meta_refresh is False + assert features.has_external_meta_refresh is False + assert features.external_link_ratio is None + assert features.image_alts == [] + + def test_malformed_html_recovers(self) -> None: + """BS4 + lxml 은 깨진 HTML 도 best-effort 파싱 — 예외 없이 반환.""" + html = "X" + features = extract_features(html, base_url="https://x.test/") + assert features.title == "X" + assert features.has_password_field is True + + +@pytest.mark.parametrize( + "href,expected_external", + [ + ("/path", False), + ("https://example.test/x", False), + ("https://www.example.test/x", False), # www. 서브도메인은 같은 사이트로 취급 + ("https://other.test/", True), + ("//other.test/", True), # protocol-relative + ], +) +def test_external_classification(href: str, expected_external: bool) -> None: + html = f'L' + features = extract_features(html, base_url="https://example.test/") + if expected_external: + assert features.external_link_ratio == 1.0 + else: + assert features.external_link_ratio == 0.0 diff --git a/tests/services/content_analyzer/test_fetch.py b/tests/services/content_analyzer/test_fetch.py new file mode 100644 index 0000000..e7f87ca --- /dev/null +++ b/tests/services/content_analyzer/test_fetch.py @@ -0,0 +1,370 @@ +"""fetch_page — httpx 기반 페이지 본문 취득.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.core.config import settings +from app.services.content_analyzer import fetch as fetch_module +from app.services.content_analyzer.fetch import fetch_page + + +@pytest.fixture(autouse=True) +def _reset_singleton_client(monkeypatch: pytest.MonkeyPatch) -> None: + """매 테스트마다 모듈 싱글턴 클라이언트를 리셋 — 테스트 간 mock 누수 방지.""" + fetch_module._client = None + # SSRF DNS 2선이 unit 테스트에서 실제 외부 해석을 시도하면 느려지고 비결정적이 된다. + # 명시적으로 false 반환하도록 패치하고, 해당 동작은 별도 케이스에서 검증한다. + + async def _no_block(_host: str) -> bool: + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", _no_block + ) + yield + fetch_module._client = None + + +def _make_stream_response( + status_code: int = 200, + headers: dict[str, str] | None = None, + body: bytes = b"", +) -> AsyncMock: + """httpx.Response 스트림 mock — aiter_bytes + headers + status_code.""" + resp = AsyncMock() + resp.status_code = status_code + resp.headers = headers or {"content-type": "text/html; charset=utf-8"} + resp.charset_encoding = "utf-8" + + async def _aiter_bytes() -> object: + yield body + + resp.aiter_bytes = _aiter_bytes + resp.aclose = AsyncMock() + resp.aread = AsyncMock(return_value=body) + return resp + + +def _patch_client(send_return: object | None = None, *, send_side_effect: object = None): + """모듈 싱글턴 _get_client 를 mock 으로 갈아끼운다 — send/build_request 만 노출.""" + mock_client = AsyncMock() + mock_client.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + if send_side_effect is not None: + mock_client.send = AsyncMock(side_effect=send_side_effect) + else: + mock_client.send = AsyncMock(return_value=send_return) + return patch( + "app.services.content_analyzer.fetch._get_client", return_value=mock_client + ) + + +def test_nat64_global_reserved_ip_is_allowed() -> None: + """DNS64/NAT64(64:ff9b::/96)는 reserved=True 지만 외부 도달 가능한 global 주소다.""" + assert fetch_module._is_blocked_ip_literal("64:ff9b::7935:df27") is False + + +def test_multicast_ip_is_blocked() -> None: + """is_global=True 인 multicast 는 SSRF 대상이 아니므로 명시적으로 차단한다.""" + assert fetch_module._is_blocked_ip_literal("224.0.0.1") is True + + +async def test_fetch_success_returns_html() -> None: + body = b"Xhi" + resp = _make_stream_response(200, body=body) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + + assert result.ok is True + assert result.status_code == 200 + assert "X" in result.html + assert result.error is None + + +async def test_fetch_timeout() -> None: + with _patch_client(send_side_effect=httpx.TimeoutException("too slow")): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "timeout" + assert result.html == "" + + +async def test_fetch_connect_error() -> None: + with _patch_client(send_side_effect=httpx.ConnectError("dns")): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "connect_error" + + +async def test_fetch_http_4xx() -> None: + resp = _make_stream_response(404, body=b"not found") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error is not None + assert result.error.startswith("http_error") + assert result.status_code == 404 + + +async def test_fetch_http_5xx() -> None: + resp = _make_stream_response(502, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error is not None + assert result.error.startswith("http_error") + + +async def test_fetch_rejects_redirect_status() -> None: + """follow_redirects=False 이므로 3xx 을 받으면 에러로 처리.""" + resp = _make_stream_response(302, headers={"location": "https://other.test/"}) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "unexpected_redirect" + + +async def test_fetch_rejects_content_length_over_cap(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "content_fetch_max_bytes", 1024) + resp = _make_stream_response( + 200, + headers={"content-type": "text/html", "content-length": "999999"}, + ) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "too_large" + + +async def test_fetch_rejects_when_stream_exceeds_cap(monkeypatch: pytest.MonkeyPatch) -> None: + """Content-Length 헤더가 없어도 스트림 누적량이 cap 을 넘으면 too_large 로 거절한다.""" + monkeypatch.setattr(settings, "content_fetch_max_bytes", 64) + + resp = AsyncMock() + resp.status_code = 200 + resp.headers = {"content-type": "text/html"} + resp.charset_encoding = "utf-8" + + async def _aiter_bytes() -> object: + yield b"a" * 100 + yield b"b" * 100 # cap 초과 — 두 번째 청크는 무시되어야 함 + + resp.aiter_bytes = _aiter_bytes + resp.aclose = AsyncMock() + + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + + assert result.ok is False + assert result.error == "too_large" + + +async def test_fetch_rejects_non_html_content_type() -> None: + resp = _make_stream_response(200, headers={"content-type": "image/png"}, body=b"\x89PNG") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "not_html" + + +async def test_fetch_rejects_xhtml() -> None: + """application/xhtml+xml 은 XXE 회색지대라 본 분석에서는 입력 단에서 잘라낸다.""" + resp = _make_stream_response( + 200, + headers={"content-type": "application/xhtml+xml"}, + body=b"x", + ) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "not_html" + + +async def test_fetch_sets_user_agent_from_settings(monkeypatch: pytest.MonkeyPatch) -> None: + """요청 헤더에 content_user_agents 풀에서 뽑은 UA 가 들어가야 한다.""" + monkeypatch.setattr(settings, "content_user_agents", ["TestUA/1.0"]) + sent_requests: list[httpx.Request] = [] + + async def _capture_send(req: httpx.Request, **_kw: object) -> object: + sent_requests.append(req) + return _make_stream_response(200, body=b"") + + mock_client = AsyncMock() + mock_client.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + mock_client.send = _capture_send + + with patch( + "app.services.content_analyzer.fetch._get_client", return_value=mock_client + ): + await fetch_page("https://example.test/") + + assert len(sent_requests) == 1 + assert sent_requests[0].headers["user-agent"] == "TestUA/1.0" + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost/", + "http://127.0.0.1/", + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://169.254.169.254/latest/meta-data/", # AWS IMDS + "http://[::1]/", + "http://metadata.google.internal/", + ], +) +async def test_fetch_rejects_internal_targets(url: str) -> None: + """SSRF 1선 — 사설/loopback/link-local IP 와 잘 알려진 내부 호스트네임은 connect 전에 차단.""" + with patch( + "app.services.content_analyzer.fetch._get_client" + ) as get_client: + result = await fetch_page(url) + assert result.ok is False + assert result.error == "blocked_host" + # 호스트가 차단되면 클라이언트 자체를 가져오지 않는다. + get_client.assert_not_called() + + +async def test_fetch_allows_public_host() -> None: + """공인 IP/도메인은 통과 — 정상 경로 회귀 방지.""" + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is True + + +async def test_fetch_no_follow_redirects_config() -> None: + """모듈 싱글턴 빌더가 follow_redirects=False 로 AsyncClient 를 생성하는지 확인.""" + with patch( + "app.services.content_analyzer.fetch.httpx.AsyncClient" + ) as client_class: + fetch_module._build_client() + kwargs = client_class.call_args.kwargs + assert kwargs.get("follow_redirects") is False + assert kwargs.get("verify") is True + assert kwargs.get("trust_env") is False + + +def test_fetch_client_uses_configured_proxy(monkeypatch: pytest.MonkeyPatch) -> None: + """운영 egress 프록시를 설정하면 분석 fetch 가 그 경로를 사용해야 한다.""" + monkeypatch.setattr(settings, "content_fetch_proxy_url", "http://proxy.local:8080") + with patch("app.services.content_analyzer.fetch.httpx.AsyncClient") as client_class: + fetch_module._build_client() + kwargs = client_class.call_args.kwargs + assert kwargs.get("proxy") == "http://proxy.local:8080" + + +class TestSingletonClient: + async def test_get_client_reuses_instance(self) -> None: + """동일 프로세스 안에서 _get_client 는 같은 인스턴스를 돌려줘야 한다 — 풀 재사용 핵심.""" + client = AsyncMock() + client.aclose = AsyncMock() + with patch("app.services.content_analyzer.fetch._build_client", return_value=client): + c1 = fetch_module._get_client() + c2 = fetch_module._get_client() + assert c1 is c2 + await fetch_module.aclose_client() + + async def test_aclose_client_resets_singleton(self) -> None: + """aclose 후 _get_client 는 새 인스턴스를 만든다 — lifespan 재시작 시나리오 보호.""" + client1 = AsyncMock() + client1.aclose = AsyncMock() + client2 = AsyncMock() + client2.aclose = AsyncMock() + with patch( + "app.services.content_analyzer.fetch._build_client", + side_effect=[client1, client2], + ): + c1 = fetch_module._get_client() + await fetch_module.aclose_client() + assert fetch_module._client is None + c2 = fetch_module._get_client() + assert c2 is not c1 + await fetch_module.aclose_client() + + async def test_aclose_client_idempotent(self) -> None: + await fetch_module.aclose_client() + await fetch_module.aclose_client() + + +class TestDnsRebindGuard: + """SSRF 2선 — 호스트네임이 사설 IP 로 풀리면 connect 전에 차단.""" + + async def test_blocks_host_resolving_to_private_ip( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + async def _resolves_to_private(_host: str) -> bool: + return True + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _resolves_to_private, + ) + # 클라이언트 측 send 가 호출되면 검증 실패 — 사전 차단되어야 한다 + with patch( + "app.services.content_analyzer.fetch._get_client" + ) as get_client: + result = await fetch_page("https://evil-public.test/") + assert result.ok is False + assert result.error == "blocked_host" + get_client.assert_not_called() + + async def test_passes_when_resolved_ips_public( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + async def _resolves_public(_host: str) -> bool: + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _resolves_public, + ) + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is True + + async def test_dns_resolution_failure_does_not_block( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """getaddrinfo OSError 는 block 으로 격상하지 않는다 — connect 단계에서 자연 처리.""" + + async def _raises(_host: str) -> bool: + raise OSError("dns down") + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _raises, + ) + with _patch_client(send_side_effect=httpx.ConnectError("dns")): + result = await fetch_page("https://example.test/") + assert result.error == "connect_error" + + async def test_skips_resolution_for_ip_literal( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """공인 IP 리터럴은 1선에서 통과 — 같은 IP 를 다시 해석할 필요 없으므로 2선을 건너뛴다.""" + called: list[str] = [] + + async def _track(host: str) -> bool: + called.append(host) + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _track, + ) + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + await fetch_page("https://8.8.8.8/") + assert called == [] + + +def test_pick_user_agent_raises_on_empty_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """UA 풀이 비면 침묵 폴백 대신 명시적으로 raise — 운영자가 misconfiguration 을 발견하게.""" + monkeypatch.setattr(settings, "content_user_agents", []) + with pytest.raises(RuntimeError, match="content_user_agents"): + fetch_module._pick_user_agent() diff --git a/tests/services/content_analyzer/test_render.py b/tests/services/content_analyzer/test_render.py new file mode 100644 index 0000000..a0b438d --- /dev/null +++ b/tests/services/content_analyzer/test_render.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import pytest +from app.services.content_analyzer import render + + +@pytest.mark.asyncio +async def test_render_semaphore_is_recreated_per_event_loop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = render._get_render_semaphore() + monkeypatch.setattr(render, "_render_semaphore_loop_ref", lambda: object()) + second = render._get_render_semaphore() + + assert first is not second diff --git a/tests/services/content_analyzer/test_signals.py b/tests/services/content_analyzer/test_signals.py new file mode 100644 index 0000000..d199547 --- /dev/null +++ b/tests/services/content_analyzer/test_signals.py @@ -0,0 +1,288 @@ +"""signals.py — 규칙 기반 점수표.""" + +from __future__ import annotations + +from app.core.config import settings +from app.schemas.content_analysis import ContentSignal +from app.services.content_analyzer.extract import ExtractedFeatures +from app.services.content_analyzer.signals import score_content + + +def _features(**kwargs: object) -> ExtractedFeatures: + base = { + "title": None, + "has_password_field": False, + "has_password_form_external_action": False, + "has_meta_refresh": False, + "has_external_meta_refresh": False, + "external_link_ratio": None, + "image_alts": [], + } + base.update(kwargs) + return ExtractedFeatures(**base) # type: ignore[arg-type] + + +class TestBrandImpersonationForm: + def test_password_with_brand_in_title_on_unrelated_domain(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + ) + result = score_content(features, final_url="https://evil-naver.test/signin") + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals + assert result.brand_impersonation is True + assert result.score >= settings.score_weight_brand_impersonation + + def test_password_with_brand_matching_own_domain_is_not_impersonation(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + ) + result = score_content(features, final_url="https://naver.com/login") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + assert result.brand_impersonation is False + + def test_password_without_brand_in_title_not_impersonation(self) -> None: + features = _features( + title="일반 로그인 페이지", + has_password_field=True, + ) + result = score_content(features, final_url="https://something.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + def test_no_password_even_with_brand_title_not_brand_form(self) -> None: + features = _features(title="NAVER 소개", has_password_field=False) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + +class TestBrandMatchingFalsePositives: + """영문 라벨은 단어 경계 매칭이라 substring 우연 매치를 일으키지 않아야 한다.""" + + def test_pineapple_does_not_match_apple(self) -> None: + """과거 substring 매칭에서 pineapple → apple 로 잡히던 케이스.""" + features = _features(title="Pineapple Express", has_password_field=True) + result = score_content(features, final_url="https://pineapple-shop.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + assert result.brand_impersonation is False + + def test_naverstore_does_not_match_naver(self) -> None: + features = _features(title="naverstore login", has_password_field=True) + result = score_content(features, final_url="https://other.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + def test_googleblog_alt_does_not_match_google(self) -> None: + features = _features(image_alts=["googleblog tutorials"]) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + def test_word_boundary_still_matches_with_punctuation(self) -> None: + # 경계 매칭이라 'NAVER 로그인', 'naver,', 'naver/login' 같은 자연 등장은 그대로 매치 + for title in ["NAVER 로그인", "naver, hello", "Login to naver."]: + features = _features(title=title, has_password_field=True) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals, title + + +class TestLogoAltImpersonation: + def test_brand_in_alt_on_unrelated_domain(self) -> None: + features = _features(image_alts=["KakaoTalk 로고"]) + result = score_content(features, final_url="https://phish.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.logo_alt_impersonation is True + assert result.score >= settings.score_weight_logo_alt_impersonation + + def test_brand_in_alt_matching_own_domain(self) -> None: + features = _features(image_alts=["kakao logo"]) + result = score_content(features, final_url="https://kakao.com/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + def test_no_alt_text(self) -> None: + features = _features(image_alts=[]) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + +class TestMetaRefresh: + def test_meta_refresh_adds_score(self) -> None: + features = _features(has_meta_refresh=True) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.META_REFRESH in result.signals + assert result.score == settings.score_weight_meta_refresh + + def test_external_meta_refresh_adds_stronger_signal(self) -> None: + features = _features(has_meta_refresh=True, has_external_meta_refresh=True) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.META_REFRESH in result.signals + assert ContentSignal.EXTERNAL_META_REFRESH in result.signals + assert result.score == ( + settings.score_weight_meta_refresh + + settings.score_weight_external_meta_refresh + ) + + +class TestCredentialFormExternalAction: + def test_external_password_form_action_adds_score(self) -> None: + features = _features( + has_password_field=True, + has_password_form_external_action=True, + ) + result = score_content(features, final_url="https://example.test/") + assert ContentSignal.CREDENTIAL_FORM_EXTERNAL in result.signals + assert result.score == settings.score_weight_credential_form_external + + +class TestExternalLinkOveruse: + def test_above_threshold_signals(self) -> None: + features = _features(external_link_ratio=0.9) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE in result.signals + assert result.score == settings.score_weight_external_link_overuse + + def test_at_threshold_signals(self) -> None: + features = _features( + external_link_ratio=settings.content_external_link_ratio_threshold, + ) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE in result.signals + + def test_below_threshold_no_signal(self) -> None: + features = _features(external_link_ratio=0.3) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE not in result.signals + + def test_none_ratio_no_signal(self) -> None: + features = _features(external_link_ratio=None) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE not in result.signals + + +class TestHighSignalKoreanPhishingRules: + def test_public_agency_subsidy_resident_number_form_scores_strongly(self) -> None: + features = _features( + title="고유가 피해지원금 대상 여부 조회", + body_text_snippets=["국민건강보험 고유가 피해지원금 지급대상 여부를 조회하세요."], + form_field_summaries=[ + "label=주민등록번호 name=resident_registration_number placeholder=주민등록번호", + "label=휴대폰 번호 name=mobile_phone placeholder=010-0000-0000", + ], + cta_texts=["지원금 대상 조회하기"], + sensitive_field_types=["resident_registration_number", "phone"], + korean_lure_keywords=["지원금"], + public_agency_keywords=["국민건강보험"], + ) + + result = score_content(features, final_url="https://nhis-support.test/") + + assert ContentSignal.PII_COLLECTION_FORM in result.signals + assert ContentSignal.SENSITIVE_ID_FIELD in result.signals + assert ContentSignal.PUBLIC_AGENCY_LURE in result.signals + assert ContentSignal.KOREAN_LURE_TEXT in result.signals + assert result.score >= settings.score_danger_threshold + + def test_kakao_obituary_apk_download_scores_without_credentials(self) -> None: + features = _features( + body_text_snippets=["故 홍길동님의 모바일 부고장을 카카오톡으로 확인하세요."], + cta_texts=["카카오톡 최신버전 다운로드"], + download_links=["https://obituary.test/downloads/kakaotalk.apk"], + korean_lure_keywords=["부고", "카카오톡"], + ) + + result = score_content(features, final_url="https://obituary.test/") + + assert ContentSignal.RISKY_DOWNLOAD_LINK in result.signals + assert ContentSignal.KOREAN_LURE_TEXT in result.signals + assert ContentSignal.PII_COLLECTION_FORM not in result.signals + assert result.score >= settings.score_caution_threshold + + def test_financial_card_fields_score_but_readonly_free_text_alone_does_not(self) -> None: + readonly_like_text_only = _features( + body_text_snippets=["카드번호, CVC, 만료일 안내"], + korean_lure_keywords=[], + ) + text_result = score_content(readonly_like_text_only, final_url="https://safe.test/") + assert ContentSignal.FINANCIAL_FIELD not in text_result.signals + assert text_result.score == 0 + + actual_form = _features( + form_field_summaries=[ + "label=카드번호 name=card_number placeholder=카드번호 16자리", + "label=CVC name=card_cvc placeholder=CVC", + ], + sensitive_field_types=["card", "cvc"], + ) + form_result = score_content(actual_form, final_url="https://pay-check.test/") + assert ContentSignal.FINANCIAL_FIELD in form_result.signals + assert form_result.score >= settings.score_weight_financial_field + + def test_legitimate_login_with_birthdate_guidance_does_not_gain_new_scores(self) -> None: + features = _features( + title="명지대학교 통합로그인", + has_password_field=True, + body_text_snippets=[ + "최초 사용자는 주민등록 기재된 생년월일 6자리 입력 후 비밀번호를 변경해야 함" + ], + form_field_summaries=[ + "label=아이디 name=id placeholder=아이디", + "label=비밀번호 name=pw type=password placeholder=비밀번호", + ], + cta_texts=["로그인"], + sensitive_field_types=[], + korean_lure_keywords=[], + public_agency_keywords=[], + ) + + result = score_content(features, final_url="https://sso.mju.ac.kr/") + + assert ContentSignal.PII_COLLECTION_FORM not in result.signals + assert ContentSignal.SENSITIVE_ID_FIELD not in result.signals + assert ContentSignal.KOREAN_LURE_TEXT not in result.signals + assert result.score == 0 + + +class TestSpaShellSignal: + def test_spa_shell_emits_signal_without_score(self) -> None: + """판정 불가 플래그라 점수는 0, 시그널 코드만 올라간다.""" + features = _features(is_spa_shell=True) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.SPA_SHELL in result.signals + assert result.score == 0 + + def test_spa_shell_does_not_interfere_with_other_rules(self) -> None: + """SPA 셸이어도 alt 브랜드 위장 같은 규칙은 정상 집계된다 (드문 케이스지만 방어).""" + features = _features(is_spa_shell=True, image_alts=["NAVER"]) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.SPA_SHELL in result.signals + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.score == settings.score_weight_logo_alt_impersonation + + +class TestCombinedScoring: + def test_all_signals_sum(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=True, + external_link_ratio=0.95, + image_alts=["NAVER"], + ) + result = score_content(features, final_url="https://evil.test/") + expected = ( + settings.score_weight_brand_impersonation + + settings.score_weight_logo_alt_impersonation + + settings.score_weight_meta_refresh + + settings.score_weight_external_link_overuse + ) + assert result.score == min(expected, settings.content_analysis_score_cap) + + def test_score_cap(self) -> None: + """합산 점수가 cap을 넘지 않는다.""" + features = _features( + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=True, + external_link_ratio=0.95, + image_alts=["NAVER"], + ) + result = score_content(features, final_url="https://evil.test/") + assert result.score <= settings.content_analysis_score_cap diff --git a/tests/services/domain_heuristic/__init__.py b/tests/services/domain_heuristic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/domain_heuristic/test_check.py b/tests/services/domain_heuristic/test_check.py new file mode 100644 index 0000000..c1c081a --- /dev/null +++ b/tests/services/domain_heuristic/test_check.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from datetime import UTC +from unittest.mock import AsyncMock, patch + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo +from app.services.domain_heuristic.check import check_domain_heuristic + +_RDAP_PATH = "app.services.domain_heuristic.check.lookup_rdap" + + +@pytest.mark.asyncio +async def test_ip_direct_url(): + result = await check_domain_heuristic("http://192.168.1.1/login") + assert isinstance(result, DomainHeuristicResult) + assert DomainHeuristicSignal.IP_DIRECT in result.signals + assert result.score == 40 # IP_DIRECT만 발동 + assert result.rdap is None # IP면 RDAP 조회 안 함 + + +@pytest.mark.asyncio +async def test_score_accumulates(): + # http + .xyz TLD → NO_HTTPS(20) + SUSPICIOUS_TLD(25) = 45 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + result = await check_domain_heuristic("http://example.xyz/") + + assert DomainHeuristicSignal.NO_HTTPS in result.signals + assert DomainHeuristicSignal.SUSPICIOUS_TLD in result.signals + assert result.score == 45 + + +@pytest.mark.asyncio +async def test_rdap_failure_does_not_break_pipeline(): + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "timeout") + result = await check_domain_heuristic("https://example.com/") + + assert isinstance(result, DomainHeuristicResult) + assert result.rdap is None + assert result.rdap_error == "timeout" + assert DomainHeuristicSignal.NEW_DOMAIN not in result.signals + + +@pytest.mark.asyncio +async def test_rdap_timeout_preserves_pattern_signals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def slow_rdap(_: str): + import asyncio + + await asyncio.sleep(1) + return None, None + + monkeypatch.setattr("app.services.domain_heuristic.check.lookup_rdap", slow_rdap) + monkeypatch.setattr("app.services.domain_heuristic.check.settings.rdap_timeout_seconds", 0.001) + + result = await check_domain_heuristic("https://t-mobile.htufgk.top/pay/") + + assert result.rdap_error == "timeout" + assert DomainHeuristicSignal.SUSPICIOUS_TLD in result.signals + assert DomainHeuristicSignal.SENSITIVE_PATH in result.signals + assert DomainHeuristicSignal.BRAND_IN_URL in result.signals + assert result.score >= 61 + + +@pytest.mark.asyncio +async def test_new_domain_signal_from_rdap(): + from datetime import datetime + + fake_rdap = RdapInfo( + domain="example.com", + registrar="TestReg", + created_date=datetime(2026, 4, 15, tzinfo=UTC), + expiry_date=None, + domain_age_days=5, + is_new_domain=True, + ) + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (fake_rdap, None) + result = await check_domain_heuristic("https://example.com/") + + assert DomainHeuristicSignal.NEW_DOMAIN in result.signals + assert result.rdap == fake_rdap + + +@pytest.mark.asyncio +async def test_result_domain_field(): + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "timeout") + result = await check_domain_heuristic("https://login.naver.com/") + + assert result.domain == "naver.com" + + +@pytest.mark.asyncio +async def test_hosting_platform_skips_typosquatting(): + # github.io / netlify.app 같은 호스팅 플랫폼은 타이포스쿼팅 판정에서 제외돼야 함 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + r1 = await check_domain_heuristic("https://user.github.io/repo") + r2 = await check_domain_heuristic("https://myapp.netlify.app/") + + assert DomainHeuristicSignal.HOSTING_PLATFORM in r1.signals + assert DomainHeuristicSignal.TYPO_DOMAIN not in r1.signals + assert DomainHeuristicSignal.HOSTING_PLATFORM in r2.signals + assert DomainHeuristicSignal.TYPO_DOMAIN not in r2.signals + + +@pytest.mark.asyncio +async def test_score_clamped_to_cap(): + # 다중 신호가 캡(80)을 넘어도 응답 score는 캡으로 고정돼야 한다. + # http + .xyz + 4-segment subdomain + open-redirect param + 긴 라벨 + # = NO_HTTPS(30) + SUSPICIOUS_TLD(20) + SUBDOMAIN_OVERUSE(25) + OPEN_REDIRECT_PARAM(20) + # + HYPHEN_OVERUSE(20) = 115 → 80으로 클램프 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + result = await check_domain_heuristic( + "http://a.b.c.d.averylongsingledomainlabelname.xyz/?redirect=evil" + ) + + assert result.score == 80 + + +@pytest.mark.asyncio +async def test_known_safe_domains_not_malicious(): + # 실제 정상 도메인들이 score_malicious_threshold(50) 미만으로 분류돼야 함 + safe_urls = [ + "https://bing.com/", + "https://cnn.com/", + "https://bbc.com/", + "https://mit.edu/", + "https://netflix.com/", + "https://spotify.com/", + ] + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + for url in safe_urls: + result = await check_domain_heuristic(url) + assert result.score < 50, f"{url} score={result.score}, signals={result.signals}" + + +@pytest.mark.asyncio +async def test_known_safe_dga_like_domains_are_not_flagged(): + safe_urls = [ + "https://www.stackoverflow.com/", + "https://www.postgresql.org/", + "https://www.typescriptlang.org/", + ] + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + for url in safe_urls: + result = await check_domain_heuristic(url) + assert DomainHeuristicSignal.DGA_LIKE not in result.signals, ( + f"{url} score={result.score}, signals={result.signals}" + ) + assert result.score < 31 + + +@pytest.mark.asyncio +async def test_known_safe_alias_domain_is_not_typo(): + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + result = await check_domain_heuristic("https://www.notion.com/") + + assert DomainHeuristicSignal.TYPO_DOMAIN not in result.signals + assert result.score < 31 diff --git a/tests/services/domain_heuristic/test_dga.py b/tests/services/domain_heuristic/test_dga.py new file mode 100644 index 0000000..8d78918 --- /dev/null +++ b/tests/services/domain_heuristic/test_dga.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.dga import ( + _consonant_ratio, + _shannon_entropy, + check_dga, +) + + +def test_shannon_entropy_uniform() -> None: + # 모든 문자가 동일하면 엔트로피 0 + assert _shannon_entropy("aaaa") == pytest.approx(0.0) + + +def test_shannon_entropy_high() -> None: + # 무작위 문자열은 엔트로피 높음 + entropy = _shannon_entropy("xkzqjvpm") + assert entropy >= 3.0 + + +def test_shannon_entropy_empty() -> None: + assert _shannon_entropy("") == 0.0 + + +def test_consonant_ratio_all_consonants() -> None: + ratio = _consonant_ratio("bxkrptmn") + assert ratio == pytest.approx(1.0) + + +def test_consonant_ratio_all_vowels() -> None: + ratio = _consonant_ratio("aeiou") + assert ratio == pytest.approx(0.0) + + +def test_consonant_ratio_empty() -> None: + assert _consonant_ratio("") == 0.0 + + +def test_dga_high_entropy_flagged() -> None: + # "xkzqjvpmbn" — 높은 엔트로피 + 자음 비율 + signal = check_dga("https://xkzqjvpmbn.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_normal_domain_not_flagged() -> None: + signal = check_dga("https://naver.com/") + assert signal is None + + +def test_dga_google_not_flagged() -> None: + signal = check_dga("https://google.com/") + assert signal is None + + +def test_dga_high_consonant_ratio() -> None: + # "bxkrptmnsdf" — 자음 비율 > 0.6 + signal = check_dga("https://bxkrptmnsdf.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_ip_url_not_flagged() -> None: + # IP URL — 도메인 레이블 없음 + signal = check_dga("http://192.168.1.1/") + assert signal is None + + +def test_dga_uses_registered_domain_label() -> None: + # 서브도메인 무시, 등록 도메인 레이블만 분석 + # "naver"는 정상 엔트로피 → 미발동 + signal = check_dga("https://login.naver.com/") + assert signal is None + + +def test_dga_entropy_only_triggers() -> None: + # 자음 비율 < 0.8이지만 Shannon 엔트로피 >= 3.5 → DGA_LIKE 발동 + # "aeioubcdfghj": 12개 고유 문자, 엔트로피=3.585, 자음 비율=0.583 + signal = check_dga("https://aeioubcdfghj.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_short_label_skipped() -> None: + # 레이블 < 8자는 통계 신뢰도 낮아 스킵 — netflix/spotify 오탐 방지 + assert check_dga("https://netflix.com/") is None + assert check_dga("https://spotify.com/") is None + assert check_dga("https://samsung.com/") is None + + +def test_dga_consonant_ratio_below_new_threshold() -> None: + # "snapchat" 8자, 자음 비율 6/8=0.75 < 0.8 → 미발동 + assert check_dga("https://snapchat.com/") is None + + +def test_dga_long_dga_like_still_flagged() -> None: + # 8자 이상 + 자음 비율 >= 0.8 → 여전히 발동 + assert check_dga("https://bxkrptmnsdf.com/") == DomainHeuristicSignal.DGA_LIKE diff --git a/tests/services/domain_heuristic/test_patterns.py b/tests/services/domain_heuristic/test_patterns.py new file mode 100644 index 0000000..5e70e15 --- /dev/null +++ b/tests/services/domain_heuristic/test_patterns.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.patterns import check_patterns + + +def test_ip_direct_ipv4() -> None: + signals = check_patterns("http://192.168.1.1/login") + assert DomainHeuristicSignal.IP_DIRECT in signals + + +def test_ip_direct_ipv6() -> None: + signals = check_patterns("http://[::1]/login") + assert DomainHeuristicSignal.IP_DIRECT in signals + + +def test_ip_direct_returns_only_ip_signal() -> None: + # IP 감지 시 다른 도메인 신호는 분석하지 않음 + signals = check_patterns("http://192.168.1.1/login?redirect=evil") + assert signals == [DomainHeuristicSignal.IP_DIRECT] + + +def test_no_https() -> None: + signals = check_patterns("http://example.com/") + assert DomainHeuristicSignal.NO_HTTPS in signals + + +def test_https_no_flag() -> None: + signals = check_patterns("https://example.com/") + assert DomainHeuristicSignal.NO_HTTPS not in signals + + +def test_subdomain_overuse() -> None: + signals = check_patterns("https://a.b.c.d.example.com/") + assert DomainHeuristicSignal.SUBDOMAIN_OVERUSE in signals + + +def test_subdomain_normal() -> None: + signals = check_patterns("https://login.naver.com/") + assert DomainHeuristicSignal.SUBDOMAIN_OVERUSE not in signals + + +def test_punycode_idn() -> None: + signals = check_patterns("https://xn--naver-zk5b.com/") + assert DomainHeuristicSignal.PUNYCODE_IDN in signals + + +def test_hyphen_overuse_count() -> None: + signals = check_patterns("https://login-secure-naver-auth.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE in signals + + +def test_hyphen_overuse_length() -> None: + # 20자 이상 레이블 → HYPHEN_OVERUSE 발동 + signals = check_patterns("https://averylongsingledomainlabelname.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE in signals + + +def test_no_hyphen_overuse() -> None: + signals = check_patterns("https://naver.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE not in signals + + +def test_suspicious_tld_xyz() -> None: + signals = check_patterns("https://example.xyz/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in signals + + +def test_suspicious_tld_zip() -> None: + signals = check_patterns("https://example.zip/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in signals + + +def test_normal_tld() -> None: + signals = check_patterns("https://example.com/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD not in signals + + +def test_open_redirect_param() -> None: + signals = check_patterns("https://example.com/login?redirect=http://evil.com") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM in signals + + +def test_open_redirect_param_url() -> None: + signals = check_patterns("https://example.com/go?url=http://evil.com") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM in signals + + +def test_no_open_redirect_param() -> None: + signals = check_patterns("https://example.com/search?q=hello") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM not in signals + + +def test_hosting_platform_netlify() -> None: + signals = check_patterns("https://myapp.netlify.app/") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + + +def test_hosting_platform_github_io() -> None: + signals = check_patterns("https://user.github.io/repo") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + + +def test_private_suffix_user_space_without_subdomain_is_hosting_platform() -> None: + signals = check_patterns("https://datrucsmain.wasmer.app/") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + + +def test_normal_domain_no_hosting_flag() -> None: + signals = check_patterns("https://naver.com/") + assert DomainHeuristicSignal.HOSTING_PLATFORM not in signals + + +def test_hosting_platform_root_not_flagged() -> None: + # 플랫폼 자체 홈페이지는 HOSTING_PLATFORM 미발동 + signals = check_patterns("https://netlify.com/") + assert DomainHeuristicSignal.HOSTING_PLATFORM not in signals + + +def test_userinfo_is_high_risk_url_signal() -> None: + signals = check_patterns("https://google.com@evil-login.example.com/session") + assert DomainHeuristicSignal.URL_USERINFO in signals + + +def test_brand_in_untrusted_url_is_signal() -> None: + signals = check_patterns("https://amazon-clone-kappa-ashen.vercel.app/login") + assert DomainHeuristicSignal.BRAND_IN_URL in signals + assert DomainHeuristicSignal.FREE_HOSTING_LURE in signals + + +def test_shortener_is_signal() -> None: + signals = check_patterns("https://qrco.de/bgoz9a") + assert DomainHeuristicSignal.URL_SHORTENER in signals + + +def test_trusted_brand_login_path_not_sensitive_path() -> None: + signals = check_patterns("https://naver.com/login") + assert DomainHeuristicSignal.SENSITIVE_PATH not in signals + + +def test_generic_brand_labels_do_not_flag_public_normal_sites() -> None: + assert DomainHeuristicSignal.BRAND_IN_URL not in check_patterns("https://www.nasa.gov/") + assert DomainHeuristicSignal.BRAND_IN_URL not in check_patterns("https://apnews.com/") + + +def test_openphish_like_brand_on_private_hosting_is_lure() -> None: + signals = check_patterns("https://www.findmyiphone.vercel.app/") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + assert DomainHeuristicSignal.BRAND_IN_URL in signals + assert DomainHeuristicSignal.FREE_HOSTING_LURE in signals + + +def test_sensitive_login_path_on_untrusted_domain_is_signal() -> None: + signals = check_patterns("https://www.driegang.nl/new-fr-societegenerale/sg/login/fr/login.php") + assert DomainHeuristicSignal.SENSITIVE_PATH in signals + assert DomainHeuristicSignal.BRAND_IN_URL in signals + + +def test_tmobile_pay_on_suspicious_tld_is_high_signal() -> None: + signals = check_patterns("https://t-mobile.htufgk.top/pay/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in signals + assert DomainHeuristicSignal.SENSITIVE_PATH in signals + assert DomainHeuristicSignal.BRAND_IN_URL in signals + + +def test_cfd_and_help_are_suspicious_tlds() -> None: + assert DomainHeuristicSignal.SUSPICIOUS_TLD in check_patterns("https://dpd.yzqpkmr.cfd/com/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in check_patterns( + "https://t-mobile.converselidom.help/pay/" + ) diff --git a/tests/services/domain_heuristic/test_rdap.py b/tests/services/domain_heuristic/test_rdap.py new file mode 100644 index 0000000..682cb55 --- /dev/null +++ b/tests/services/domain_heuristic/test_rdap.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from app.schemas.domain_heuristic import RdapInfo +from app.services.domain_heuristic import rdap as rdap_module +from app.services.domain_heuristic.rdap import _parse_datetime, _parse_rdap_response, lookup_rdap + + +@pytest.fixture(autouse=True) +def _reset_rdap_state(monkeypatch: pytest.MonkeyPatch): + """각 테스트에서 cache/inflight/client 싱글턴 상태 격리.""" + rdap_module._cache.clear() + rdap_module._inflight.clear() + rdap_module._rate_limit_until = None + monkeypatch.setattr(rdap_module, "_client", None) + + +def _install_client(mock_client: AsyncMock) -> None: + rdap_module._client = mock_client + + +def test_parse_datetime_valid(): + result = _parse_datetime("2026-01-01T00:00:00Z") + assert result is not None + assert result.tzinfo == UTC + assert result.year == 2026 + + +def test_parse_datetime_none(): + assert _parse_datetime(None) is None + + +def test_parse_datetime_invalid(): + assert _parse_datetime("not-a-date") is None + + +def test_parse_rdap_new_domain(): + # 시간 의존을 끊기 위해 "지금-10일" 동적 생성 — settings.rdap_new_domain_threshold_days(30) 미만 + recent = (datetime.now(tz=UTC) - timedelta(days=10)).strftime("%Y-%m-%dT%H:%M:%SZ") + expiry = (datetime.now(tz=UTC) + timedelta(days=355)).strftime("%Y-%m-%dT%H:%M:%SZ") + data = { + "events": [ + {"eventAction": "registration", "eventDate": recent}, + {"eventAction": "expiration", "eventDate": expiry}, + ], + "entities": [ + { + "roles": ["registrar"], + "vcardArray": ["vcard", [["fn", {}, "text", "NameCheap, Inc."]]], + } + ], + } + result = _parse_rdap_response("example.com", data) + assert result.domain == "example.com" + assert result.registrar == "NameCheap, Inc." + assert result.is_new_domain is True + assert result.domain_age_days is not None and result.domain_age_days < 30 + + +def test_parse_rdap_old_domain(): + data = { + "events": [ + {"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}, + ], + "entities": [], + } + result = _parse_rdap_response("old.com", data) + assert result.is_new_domain is False + assert result.domain_age_days is not None and result.domain_age_days > 30 + + +@pytest.mark.asyncio +async def test_lookup_rdap_success(): + mock_response = MagicMock() + mock_response.json.return_value = { + "events": [{"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}], + "entities": [], + } + mock_response.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert error is None + assert result is not None + assert result.domain == "example.com" + + +@pytest.mark.asyncio +async def test_lookup_rdap_cache_hit(): + fake_info = RdapInfo( + domain="cached.com", + registrar="TestReg", + created_date=None, + expiry_date=None, + domain_age_days=None, + is_new_domain=False, + ) + # TTLCache 는 value 만 저장하고 만료는 내부 타이머가 관리. + rdap_module._cache["cached.com"] = fake_info + + # client 미설치 — 캐시 히트면 호출 자체가 없어야 함 + result, error = await lookup_rdap("https://cached.com/path") + + assert result == fake_info + assert error is None + + +@pytest.mark.asyncio +async def test_lookup_rdap_timeout(): + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout")) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "timeout" + + +@pytest.mark.asyncio +async def test_lookup_rdap_not_found(): + mock_response = MagicMock(status_code=404) + http_error = httpx.HTTPStatusError("not found", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://unknowndomain99999.com/") + + assert result is None + assert error == "not_found" + + +@pytest.mark.asyncio +async def test_lookup_rdap_http_error(): + mock_response = MagicMock(status_code=500) + http_error = httpx.HTTPStatusError("server error", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "http_error" + + +@pytest.mark.asyncio +async def test_lookup_rdap_rate_limited(): + mock_response = MagicMock(status_code=429, headers={}) + http_error = httpx.HTTPStatusError("rate limited", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "rate_limited" + + +@pytest.mark.asyncio +async def test_lookup_rdap_logs_rate_limited(monkeypatch: pytest.MonkeyPatch): + mock_response = MagicMock(status_code=429, headers={}) + http_error = httpx.HTTPStatusError("rate limited", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + mock_warning = MagicMock() + monkeypatch.setattr(rdap_module.logger, "warning", mock_warning) + + await lookup_rdap("https://example.com/") + + mock_warning.assert_any_call( + "rdap.rate_limited", + domain="example.com", + retry_after=None, + cooldown_seconds=30.0, + ) + + +@pytest.mark.asyncio +async def test_lookup_rdap_rate_limit_retry_after_starts_cooldown( + monkeypatch: pytest.MonkeyPatch, +): + now = 1000.0 + monkeypatch.setattr(rdap_module.time, "monotonic", lambda: now) + mock_response = MagicMock(status_code=429, headers={"retry-after": "30"}) + http_error = httpx.HTTPStatusError("rate limited", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + assert result is None + assert error == "rate_limited" + + result, error = await lookup_rdap("https://other-example.com/") + assert result is None + assert error == "rate_limited" + mock_client.get.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_lookup_rdap_logs_cooldown_skip(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(rdap_module.time, "monotonic", lambda: 1000.0) + rdap_module._rate_limit_until = 1030.0 + mock_info = MagicMock() + monkeypatch.setattr(rdap_module.logger, "info", mock_info) + mock_client = AsyncMock() + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "rate_limited" + mock_client.get.assert_not_awaited() + mock_info.assert_any_call( + "rdap.rate_limit_cooldown_active", + domain="example.com", + remaining_seconds=30.0, + ) + + +@pytest.mark.asyncio +async def test_lookup_rdap_parse_error(): + mock_response = MagicMock() + mock_response.json.return_value = "not a dict" + mock_response.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "parse_error" + + +@pytest.mark.asyncio +async def test_lookup_rdap_dedup_inflight(): + """동시 요청이 겹치면 실제 fetch는 1회만 수행돼야 한다.""" + import asyncio + + call_count = 0 + request_started = asyncio.Event() + release = asyncio.Event() + + async def slow_get(url: str, **_: object) -> MagicMock: + nonlocal call_count + call_count += 1 + request_started.set() + await release.wait() + resp = MagicMock() + resp.json.return_value = { + "events": [{"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}], + "entities": [], + } + resp.raise_for_status = MagicMock() + return resp + + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=slow_get) + _install_client(mock_client) + + task1 = asyncio.create_task(lookup_rdap("https://example.com/")) + task2 = asyncio.create_task(lookup_rdap("https://example.com/")) + task3 = asyncio.create_task(lookup_rdap("https://example.com/")) + await request_started.wait() + release.set() + + results = await asyncio.gather(task1, task2, task3) + + assert call_count == 1 + for info, error in results: + assert error is None + assert info is not None and info.domain == "example.com" + + +@pytest.mark.asyncio +async def test_lookup_rdap_idn_punycode(): + """IDN 도메인은 URL 빌드 시 punycode로 변환돼야 한다.""" + captured_urls: list[str] = [] + + async def capture_get(url: str, **_: object) -> MagicMock: + captured_urls.append(url) + resp = MagicMock() + resp.json.return_value = {"events": [], "entities": []} + resp.raise_for_status = MagicMock() + return resp + + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=capture_get) + _install_client(mock_client) + + await lookup_rdap("https://xn--3e0b707e.com/") # 한국.com + assert captured_urls + assert "xn--3e0b707e.com" in captured_urls[0] + + +@pytest.mark.asyncio +async def test_lookup_rdap_invalid_domain(): + """빈 레이블/비정상 도메인은 invalid_domain 반환.""" + mock_client = AsyncMock() + _install_client(mock_client) + + # tldextract가 도메인으로 받지 않는 형태 — ASCII지만 내부 regex 확인 + # 실제 인코딩 실패 케이스: null byte 포함 + result, error = await lookup_rdap("https://..com/") + assert result is None + # top_domain_under_public_suffix가 비어서 no_domain이거나, encode 실패로 invalid_domain + assert error in {"no_domain", "invalid_domain"} diff --git a/tests/services/domain_heuristic/test_typosquatting.py b/tests/services/domain_heuristic/test_typosquatting.py new file mode 100644 index 0000000..7499466 --- /dev/null +++ b/tests/services/domain_heuristic/test_typosquatting.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic import typosquatting +from app.services.domain_heuristic.typosquatting import _levenshtein, check_typosquatting + + +def test_levenshtein_same(): + assert _levenshtein("naver", "naver") == 0 + + +def test_levenshtein_one_insert(): + assert _levenshtein("naverr", "naver") == 1 + + +def test_levenshtein_one_substitute(): + assert _levenshtein("navcr", "naver") == 1 + + +def test_levenshtein_two(): + assert _levenshtein("naverr", "naver") == 1 + assert _levenshtein("naaver", "naver") == 1 + + +def test_typo_distance_1(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverr.com/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_typo_distance_2(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverro.com/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_exact_match_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naver.com/") + assert signal is None + + +def test_distance_3_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverrrr.com/") + assert signal is None + + +def test_same_label_different_tld(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naver.net/") + assert signal is None + + +def test_same_label_suspicious_tld_still_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naver.xyz/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_no_registered_domain(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("http://localhost/") + assert signal is None + + +def test_empty_brands(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", []) + signal = check_typosquatting("https://naverr.com/") + assert signal is None + + +def test_multi_tld_brand_com_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # net 먼저 등록된 브랜드라도 com 버전은 정상 도메인으로 처리 + monkeypatch.setattr(typosquatting, "_BRANDS", [("daum", "net"), ("daum", "com")]) + assert check_typosquatting("https://daum.com/") is None + + +def test_multi_tld_brand_tv_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # tv 먼저 등록된 브랜드라도 com 버전은 정상 도메인으로 처리 + monkeypatch.setattr(typosquatting, "_BRANDS", [("twitch", "tv"), ("twitch", "com")]) + assert check_typosquatting("https://twitch.com/") is None + + +def test_short_brand_no_fuzzy_match(monkeypatch: pytest.MonkeyPatch) -> None: + # 3자 브랜드 "ing" 기준 bing.com(4자)은 편집거리 1이지만 min(3,4)<5 → 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("ing", "com")]) + assert check_typosquatting("https://bing.com/") is None + + +def test_short_brand_cnn_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # 3자 브랜드 "ocn"과 cnn은 편집거리 2지만 짧은 브랜드라 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("ocn", "com")]) + assert check_typosquatting("https://cnn.com/") is None + + +def test_two_char_brand_no_fuzzy(monkeypatch: pytest.MonkeyPatch) -> None: + # 2자 브랜드 "kb"는 완전 일치 외 모두 스킵 → kbs.com 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("kb", "co.kr")]) + assert check_typosquatting("https://kbs.com/") is None + + +def test_short_brand_exact_label_diff_suffix_still_flagged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 정확히 같은 label이면 suffix 달라도 여전히 typosquat 취급 (distance 0은 길이 무관) + monkeypatch.setattr(typosquatting, "_BRANDS", [("kb", "co.kr")]) + assert check_typosquatting("https://kb.xyz/") == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_hosting_platform_root_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # netlify.app 루트가 brand netlify.com과 label 일치 + suffix 상이로 오탐되던 케이스 + monkeypatch.setattr(typosquatting, "_BRANDS", [("netlify", "com")]) + assert check_typosquatting("https://netlify.app/") is None + + +def test_hosting_platform_root_vercel_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("vercel", "com")]) + assert check_typosquatting("https://vercel.app/") is None diff --git a/tests/services/normalizer/__init__.py b/tests/services/normalizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py new file mode 100644 index 0000000..fa8c2ca --- /dev/null +++ b/tests/services/normalizer/test_normalize.py @@ -0,0 +1,231 @@ +"""normalize_url 단위 테스트.""" + +import pytest +from app.core.exceptions import NormalizationError +from app.services.normalizer.normalize import normalize_url + + +class TestInputValidation: + def test_empty_string_raises(self) -> None: + with pytest.raises(NormalizationError, match="빈 URL"): + normalize_url("") + + def test_whitespace_only_raises(self) -> None: + with pytest.raises(NormalizationError, match="빈 URL"): + normalize_url(" ") + + def test_whitespace_trimmed(self) -> None: + result = normalize_url(" https://example.com/ ") + assert result.original_url == "https://example.com/" + assert result.normalized_url == "https://example.com/" + + def test_control_characters_removed(self) -> None: + result = normalize_url("https://example\x00.com/pa\x01th") + assert "\x00" not in result.normalized_url + assert "\x01" not in result.normalized_url + assert "example.com" in result.normalized_url + + def test_max_length_exceeded_raises(self) -> None: + long_url = "https://example.com/" + "a" * 1024 + with pytest.raises(NormalizationError, match="최대 길이"): + normalize_url(long_url) + + def test_max_length_boundary_passes(self) -> None: + padding = 1024 - len("https://example.com/") + url = "https://example.com/" + "a" * padding + result = normalize_url(url) + assert result.normalized_url.startswith("https://example.com/") + + def test_max_length_checked_after_scheme_prepend(self) -> None: + padding = 1024 - len("http://") - len("example.com/") + url_without_scheme = "example.com/" + "a" * padding + result = normalize_url(url_without_scheme) + assert result.normalized_url.startswith("http://example.com/") + + +class TestOriginalPreservation: + def test_original_url_preserved_after_trim(self) -> None: + result = normalize_url(" HTTP://EXAMPLE.COM/Path ") + assert result.original_url == "HTTP://EXAMPLE.COM/Path" + + def test_original_differs_from_normalized(self) -> None: + result = normalize_url("HTTP://EXAMPLE.COM:80/Path#frag") + assert result.original_url == "HTTP://EXAMPLE.COM:80/Path#frag" + assert result.normalized_url == "http://example.com/Path" + + +class TestScheme: + def test_lowercase_scheme(self) -> None: + result = normalize_url("HTTP://Example.com") + assert result.normalized_url == "http://example.com/" + + def test_https_preserved(self) -> None: + result = normalize_url("HTTPS://Example.com/path") + assert result.normalized_url == "https://example.com/path" + + def test_no_scheme_defaults_to_http_and_marks_added_scheme(self) -> None: + result = normalize_url("example.com/path") + assert result.normalized_url == "http://example.com/path" + assert result.scheme_was_added is True + + def test_explicit_http_does_not_mark_added_scheme(self) -> None: + result = normalize_url("http://example.com/path") + assert result.normalized_url == "http://example.com/path" + assert result.scheme_was_added is False + + def test_scheme_like_string_in_path_gets_scheme_prepended(self) -> None: + result = normalize_url("example.com/path://weird") + assert result.normalized_url.startswith("http://example.com/") + assert result.scheme_was_added is True + + def test_unsupported_scheme_raises(self) -> None: + with pytest.raises(NormalizationError, match="지원하지 않는 스킴"): + normalize_url("ftp2://example.com") + + def test_ftp_allowed(self) -> None: + result = normalize_url("FTP://files.example.com/pub") + assert result.normalized_url.startswith("ftp://") + + +class TestHost: + def test_lowercase_host(self) -> None: + result = normalize_url("https://EXAMPLE.COM/") + assert result.normalized_url == "https://example.com/" + + def test_scheme_only_raises(self) -> None: + with pytest.raises(NormalizationError, match="호스트가 비어있습니다"): + normalize_url("https://") + + def test_userinfo_stripped_for_phishing(self) -> None: + result = normalize_url("https://google.com@evil.com/path") + assert "evil.com" in result.normalized_url + assert "google.com@" not in result.normalized_url + + def test_basic_auth_userinfo_stripped(self) -> None: + result = normalize_url("https://user:pass@example.com/api") + assert "user:pass@" not in result.normalized_url + assert "example.com/api" in result.normalized_url + + +class TestPort: + def test_default_http_port_removed(self) -> None: + result = normalize_url("http://example.com:80/path") + assert result.normalized_url == "http://example.com/path" + + def test_default_https_port_removed(self) -> None: + result = normalize_url("https://example.com:443/path") + assert result.normalized_url == "https://example.com/path" + + def test_default_ftp_port_removed(self) -> None: + result = normalize_url("ftp://example.com:21/file") + assert result.normalized_url == "ftp://example.com/file" + + def test_non_default_port_kept(self) -> None: + result = normalize_url("https://example.com:8080/") + assert ":8080" in result.normalized_url + + def test_out_of_range_port_raises(self) -> None: + with pytest.raises(NormalizationError, match="유효하지 않은 포트"): + normalize_url("https://example.com:99999/") + + def test_non_numeric_port_raises(self) -> None: + with pytest.raises(NormalizationError, match="유효하지 않은 포트"): + normalize_url("example.com:notaport/foo") + + +class TestPath: + def test_empty_path_becomes_slash(self) -> None: + result = normalize_url("https://example.com") + assert result.normalized_url == "https://example.com/" + + def test_path_case_preserved(self) -> None: + result = normalize_url("https://example.com/Path/To/Page") + assert "/Path/To/Page" in result.normalized_url + + def test_double_slash_collapsed(self) -> None: + result = normalize_url("https://example.com//a//b") + assert "//" not in result.normalized_url.split("://", 1)[1] + + def test_dot_segment_resolved(self) -> None: + result = normalize_url("https://example.com/a/b/../c") + assert result.normalized_url == "https://example.com/a/c" + + def test_single_dot_resolved(self) -> None: + result = normalize_url("https://example.com/a/./b") + assert result.normalized_url == "https://example.com/a/b" + + def test_dot_beyond_root(self) -> None: + result = normalize_url("https://example.com/../a") + assert result.normalized_url == "https://example.com/a" + + def test_complex_dot_segments(self) -> None: + result = normalize_url("https://example.com/a/b/c/../../d") + assert result.normalized_url == "https://example.com/a/d" + + +class TestFragment: + def test_fragment_removed(self) -> None: + result = normalize_url("https://example.com/page#section") + assert "#" not in result.normalized_url + + def test_fragment_with_query(self) -> None: + result = normalize_url("https://example.com/page?q=1#top") + assert "#" not in result.normalized_url + assert "q=1" in result.normalized_url + + +class TestPercentEncoding: + def test_unreserved_decoded(self) -> None: + result = normalize_url("https://example.com/%41%42%43") + assert "/ABC" in result.normalized_url + + def test_tilde_decoded(self) -> None: + result = normalize_url("https://example.com/%7Euser") + assert "/~user" in result.normalized_url + + def test_reserved_hex_uppercased(self) -> None: + result = normalize_url("https://example.com/path%2fmore") + assert "%2F" in result.normalized_url + + def test_already_uppercase_hex_unchanged(self) -> None: + result = normalize_url("https://example.com/path%2Fmore") + assert "%2F" in result.normalized_url + + def test_space_encoding_uppercased(self) -> None: + result = normalize_url("https://example.com/my%20page") + assert "%20" in result.normalized_url + + def test_query_encoding_normalized(self) -> None: + result = normalize_url("https://example.com/?q=%61%62%63") + assert "q=abc" in result.normalized_url + + def test_params_encoding_normalized(self) -> None: + result = normalize_url("https://example.com/path;%7eparam?q=1") + assert ";~param" in result.normalized_url + + +class TestIDN: + def test_unicode_to_punycode(self) -> None: + result = normalize_url("https://☃.com/") + assert "xn--" in result.normalized_url + + def test_punycode_stays_punycode(self) -> None: + result = normalize_url("https://xn--n3h.com/") + assert "xn--n3h.com" in result.normalized_url + + def test_ascii_domain_unchanged(self) -> None: + result = normalize_url("https://example.com/") + assert result.normalized_url == "https://example.com/" + + +class TestIntegration: + def test_full_normalization(self) -> None: + raw = " HTTP://EXAMPLE.COM:80/a/../b/./c?q=%7e#frag " + result = normalize_url(raw) + assert result.original_url == "HTTP://EXAMPLE.COM:80/a/../b/./c?q=%7e#frag" + assert result.normalized_url == "http://example.com/b/c?q=~" + + def test_preserves_meaningful_query(self) -> None: + result = normalize_url("https://example.com/search?q=hello&lang=ko") + assert "q=hello" in result.normalized_url + assert "lang=ko" in result.normalized_url diff --git a/tests/services/test_analysis_callback.py b/tests/services/test_analysis_callback.py new file mode 100644 index 0000000..b8cfcf9 --- /dev/null +++ b/tests/services/test_analysis_callback.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import ( + PipelineFailure, + PipelineStage, + PipelineStages, + PipelineStageTimings, + PipelineSuccess, + PipelineTimings, + Verdict, +) +from app.schemas.threat_db import GSBMatch, GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import HopRecord, UnchainResult +from app.services.analysis_callback import post_analysis_callback + + +def _success_result() -> PipelineSuccess: + final_url = "https://example.test/" + return PipelineSuccess( + analysis_id="aid-1", + original_url="https://bit.ly/a", + final_url=final_url, + verdict=Verdict.SAFE, + score=0, + timings=PipelineTimings( + total_seconds=0.123, + stages=PipelineStageTimings( + normalize=0.001, + unchain=0.002, + threat_db=0.003, + domain_heuristic=0.004, + content_analysis=0.005, + ), + ), + stages=PipelineStages( + normalize=NormalizeResult( + original_url="https://bit.ly/a", + normalized_url="https://bit.ly/a", + ), + unchain=UnchainResult( + input_url="https://bit.ly/a", + final_url=final_url, + hops=[ + HopRecord( + url="https://bit.ly/a", + status_code=302, + location=final_url, + ), + HopRecord(url=final_url, status_code=200), + ], + hop_count=2, + ), + threat_db=ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=2, + gsb=GSBResult(checked=True), + urlhaus=URLhausResult(checked=True), + ), + domain_heuristic=DomainHeuristicResult( + domain="example.test", + score=25, + signals=[DomainHeuristicSignal.NEW_DOMAIN], + rdap=RdapInfo( + domain="example.test", + registrar="Example Registrar", + created_date=datetime(2026, 4, 1, tzinfo=UTC), + expiry_date=None, + domain_age_days=6, + is_new_domain=True, + ), + ), + content_analysis=ContentAnalysisResult( + final_url=final_url, + fetched=True, + score=45, + signals=[ContentSignal.BRAND_IMPERSONATION_FORM], + has_password_field=True, + ai_verdict="phishing", + ai_reason="브랜드 사칭 로그인 폼이 확인되었습니다.", + ), + ), + ) + + +def _malicious_result() -> PipelineSuccess: + result = _success_result() + result.stages.threat_db.is_malicious = True + result.stages.threat_db.gsb = GSBResult( + checked=True, + is_threat=True, + matches=[GSBMatch(threat_type="SOCIAL_ENGINEERING")], + ) + result.stages.threat_db.urlhaus = URLhausResult( + checked=True, + is_threat=True, + matched_key="example.test", + ) + return result + + +@pytest.mark.asyncio +async def test_posts_success_callback_payload(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "spring_internal_url", "http://spring.internal") + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.post.return_value = httpx.Response(200, json={"received": True}) + + with patch("app.services.analysis_callback.httpx.AsyncClient", return_value=mock_client): + delivered = await post_analysis_callback( + _success_result(), + request_id="rid-1", + elapsed_ms=123, + analyzed_at=datetime(2026, 4, 7, 5, 42, 11, tzinfo=UTC), + ) + + assert delivered is True + mock_client.post.assert_awaited_once() + url = mock_client.post.await_args.args[0] + kwargs = mock_client.post.await_args.kwargs + assert url == "http://spring.internal/internal/analysis-result" + assert kwargs["headers"] == { + "Content-Type": "application/json", + "X-Internal-Api-Key": settings.internal_api_key, + "X-Request-ID": "rid-1", + } + payload = kwargs["json"] + assert payload["analysisId"] == "aid-1" + assert payload["requestId"] == "rid-1" + assert payload["status"] == "succeeded" + assert payload["originalUrl"] == "https://bit.ly/a" + assert payload["finalUrl"] == "https://example.test/" + assert payload["verdict"] == "safe" + assert payload["score"] == 0 + assert payload["engineVersion"] == settings.app_version + assert payload["analyzedAt"] == "2026-04-07T05:42:11Z" + assert payload["elapsedMs"] == 123 + assert payload["summary"] == "브랜드 사칭 로그인 폼이 확인되었습니다." + assert payload["reasons"] == [ + { + "code": "NEW_DOMAIN", + "stage": 3, + "weight": settings.score_weight_new_domain, + "message": "최근 등록된 도메인입니다.", + }, + { + "code": "BRAND_IMPERSONATION_FORM", + "stage": 4, + "weight": settings.score_weight_brand_impersonation, + "message": "브랜드를 사칭하는 로그인 폼이 확인되었습니다.", + }, + { + "code": "AI_PHISHING", + "stage": 4, + "weight": settings.score_weight_ai_phishing, + "message": "브랜드 사칭 로그인 폼이 확인되었습니다.", + }, + ] + assert payload["stages"] == { + "externalDb": { + "gsb": {"isThreat": False, "matchedTypes": []}, + "urlhaus": {"isThreat": False, "host": "example.test"}, + }, + "unchain": { + "hops": 2, + "chain": ["https://bit.ly/a", "https://example.test/"], + }, + "domainHeuristic": { + "rdap": { + "domain": "example.test", + "registrar": "Example Registrar", + "createdDate": "2026-04-01T00:00:00Z", + "domainAgeDays": 6, + "isNewDomain": True, + }, + "signals": ["NEW_DOMAIN"], + }, + "contentAnalysis": { + "fetched": True, + "hasPasswordField": True, + "aiVerdict": "phishing", + "aiReason": "브랜드 사칭 로그인 폼이 확인되었습니다.", + }, + } + + +@pytest.mark.asyncio +async def test_posts_known_malicious_callback_with_fixed_summary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "spring_internal_url", "http://spring.internal") + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.post.return_value = httpx.Response(200, json={"received": True}) + + with patch("app.services.analysis_callback.httpx.AsyncClient", return_value=mock_client): + delivered = await post_analysis_callback( + _malicious_result(), + request_id="rid-1", + elapsed_ms=123, + analyzed_at=datetime(2026, 4, 7, 5, 42, 11, tzinfo=UTC), + ) + + assert delivered is True + payload = mock_client.post.await_args.kwargs["json"] + assert payload["summary"] == "악성으로 알려진 페이지 입니다." + assert payload["reasons"][:2] == [ + { + "code": "GSB_MATCH", + "stage": 2, + "weight": settings.score_weight_gsb, + "message": "Google Safe Browsing에 악성 URL로 등록되어 있습니다.", + }, + { + "code": "URLHAUS_MATCH", + "stage": 2, + "weight": settings.score_weight_urlhaus, + "message": "URLhaus에 악성 URL로 등록되어 있습니다.", + }, + ] + assert payload["stages"]["externalDb"] == { + "gsb": {"isThreat": True, "matchedTypes": ["SOCIAL_ENGINEERING"]}, + "urlhaus": {"isThreat": True, "host": "example.test"}, + } + + +@pytest.mark.asyncio +async def test_retries_callback_on_non_2xx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "spring_internal_url", "http://spring.internal/") + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.post.side_effect = [ + httpx.Response(500), + httpx.Response(502), + httpx.Response(200), + ] + + with ( + patch("app.services.analysis_callback.httpx.AsyncClient", return_value=mock_client), + patch("app.services.analysis_callback.asyncio.sleep", new_callable=AsyncMock), + ): + delivered = await post_analysis_callback( + _success_result(), + request_id="rid-1", + elapsed_ms=1, + analyzed_at=datetime(2026, 4, 7, 5, 42, 11, tzinfo=UTC), + ) + + assert delivered is True + assert mock_client.post.await_count == 3 + + +@pytest.mark.asyncio +async def test_posts_failure_callback_payload(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "spring_internal_url", "http://spring.internal") + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.post.return_value = httpx.Response(200) + result = PipelineFailure( + analysis_id="aid-fail", + original_url="not a url", + failed_at_stage=PipelineStage.NORMALIZE, + error="invalid url", + timings=PipelineTimings( + total_seconds=0.001, + stages=PipelineStageTimings(normalize=0.001), + ), + ) + + with patch("app.services.analysis_callback.httpx.AsyncClient", return_value=mock_client): + delivered = await post_analysis_callback( + result, + request_id="rid-fail", + elapsed_ms=7, + analyzed_at=datetime(2026, 4, 7, 5, 42, 41, tzinfo=UTC), + ) + + assert delivered is True + payload = mock_client.post.await_args.kwargs["json"] + assert payload["analysisId"] == "aid-fail" + assert payload["requestId"] == "rid-fail" + assert payload["status"] == "failed" + assert payload["originalUrl"] == "not a url" + assert payload["error"] == { + "code": "NORMALIZE_FAILED", + "stage": 1, + "message": "invalid url", + } + assert "timings" not in payload + assert "stages" not in payload + + +@pytest.mark.asyncio +async def test_posts_page_unavailable_callback_without_verdict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "spring_internal_url", "http://spring.internal") + mock_client = AsyncMock() + mock_client.__aenter__.return_value = mock_client + mock_client.post.return_value = httpx.Response(200) + result = PipelineFailure( + analysis_id="aid-missing", + original_url="https://missing.test/", + final_url="https://missing.test/", + failed_at_stage=PipelineStage.CONTENT_ANALYSIS, + error="페이지를 찾을 수 없습니다.", + error_code="PAGE_UNAVAILABLE", + status_code=404, + ) + + with patch("app.services.analysis_callback.httpx.AsyncClient", return_value=mock_client): + delivered = await post_analysis_callback( + result, + request_id="rid-missing", + elapsed_ms=17, + analyzed_at=datetime(2026, 4, 7, 5, 43, 41, tzinfo=UTC), + ) + + assert delivered is True + payload = mock_client.post.await_args.kwargs["json"] + assert payload["status"] == "failed" + assert payload["finalUrl"] == "https://missing.test/" + assert payload["error"] == { + "code": "PAGE_UNAVAILABLE", + "stage": 4, + "message": "페이지를 찾을 수 없습니다.", + "statusCode": 404, + } + assert "verdict" not in payload + assert "score" not in payload diff --git a/tests/services/test_db_independent_pipeline.py b/tests/services/test_db_independent_pipeline.py new file mode 100644 index 0000000..032b98d --- /dev/null +++ b/tests/services/test_db_independent_pipeline.py @@ -0,0 +1,334 @@ +"""외부 threat DB 비의존 파이프라인 회귀 테스트.""" + +from __future__ import annotations + +import asyncio +import inspect +from unittest.mock import AsyncMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.db_independent_pipeline import ( + DbIndependentPipelineFailure, + DbIndependentPipelineSuccess, +) +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import PipelineStage, Verdict +from app.schemas.unchain import UnchainResult +from app.services.db_independent_pipeline import run_db_independent_pipeline + + +def _make_unchain(final_url: str, *, signals: list[str] | None = None) -> UnchainResult: + return UnchainResult( + input_url=final_url, + final_url=final_url, + hops=[], + hop_count=0, + signals=signals or [], + ) + + +def _make_heuristic(score: int) -> DomainHeuristicResult: + return DomainHeuristicResult( + domain="example.com", + score=score, + signals=[DomainHeuristicSignal.HOSTING_PLATFORM] if score else [], + rdap=None, + rdap_error=None, + ) + + +def _make_content(final_url: str, *, score: int = 0) -> ContentAnalysisResult: + return ContentAnalysisResult(final_url=final_url, fetched=True, score=score, signals=[]) + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_never_calls_threat_db() -> None: + final_url = "https://example.com/login" + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new_callable=AsyncMock, + ) as mock_heuristic, + patch( + "app.services.db_independent_pipeline.analyze_content", new_callable=AsyncMock + ) as mock_content, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat_db, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_heuristic.return_value = _make_heuristic(15) + mock_content.return_value = _make_content(final_url, score=20) + + result = await run_db_independent_pipeline("aid-db-free", final_url) + + assert isinstance(result, DbIndependentPipelineSuccess) + assert result.analysis_id == "aid-db-free" + assert result.final_url == final_url + assert result.score == 35 + assert result.verdict == Verdict.CAUTION + assert result.stages.domain_heuristic.score == 15 + assert result.stages.content_analysis.score == 20 + assert result.timings is not None + assert result.timings.stages.threat_db is None + mock_threat_db.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_passes_url_and_redirect_signals_to_content() -> None: + final_url = "https://redirected.example.com/login" + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new_callable=AsyncMock, + ) as mock_heuristic, + patch( + "app.services.db_independent_pipeline.analyze_content", new_callable=AsyncMock + ) as mock_content, + ): + mock_norm.return_value = NormalizeResult( + original_url="https://short.test/a", + normalized_url="https://short.test/a", + ) + mock_unchain.return_value = _make_unchain( + final_url, + signals=["cross_origin:short.test->redirected.example.com"], + ) + mock_heuristic.return_value = _make_heuristic(15) + mock_content.return_value = _make_content(final_url) + + result = await run_db_independent_pipeline("aid-sig", "https://short.test/a") + + assert isinstance(result, DbIndependentPipelineSuccess) + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + upstream = kwargs["upstream_signals"] + if inspect.isawaitable(upstream): + upstream = await upstream + assert upstream == ( + "HOSTING_PLATFORM", + "REDIRECT_CROSS_ORIGIN", + ) + assert DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN in result.stages.domain_heuristic.signals + assert result.stages.domain_heuristic.score > 15 + assert "provider" not in kwargs + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_checks_content_when_heuristic_is_danger() -> None: + final_url = "https://danger.example.com/login" + content_started = asyncio.Event() + + async def _slow_heuristic(_: str) -> DomainHeuristicResult: + await asyncio.sleep(0.01) + return _make_heuristic(65) + + async def _slow_content(_: str, **__: object) -> ContentAnalysisResult: + content_started.set() + return _make_content(final_url, score=20) + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new=AsyncMock(side_effect=_slow_heuristic), + ), + patch( + "app.services.db_independent_pipeline.analyze_content", + new=AsyncMock(side_effect=_slow_content), + ) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + + result = await run_db_independent_pipeline("aid-parallel", final_url) + + assert isinstance(result, DbIndependentPipelineSuccess) + assert result.score == 85 + assert result.verdict == Verdict.DANGER + assert result.stages.content_analysis.fetched is True + assert content_started.is_set() is True + mock_content.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_returns_failure_when_page_unavailable() -> None: + final_url = "https://missing.example.com/" + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new_callable=AsyncMock, + ) as mock_heuristic, + patch( + "app.services.db_independent_pipeline.analyze_content", new_callable=AsyncMock + ) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_heuristic.return_value = _make_heuristic(0) + mock_content.return_value = ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=404, + score=0, + signals=[ContentSignal.FETCH_FAILED], + reason="페이지를 찾을 수 없습니다.", + error="http_error_404", + ) + + result = await run_db_independent_pipeline("aid-missing", final_url) + + assert isinstance(result, DbIndependentPipelineFailure) + assert result.failed_at_stage == PipelineStage.CONTENT_ANALYSIS + assert result.error_code == "PAGE_UNAVAILABLE" + assert result.final_url == final_url + assert result.status_code == 404 + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_verdict_on_strong_unavailable_signal() -> None: + final_url = "http://xj3kq9vbnm2p7zla.com/login" + unchain = _make_unchain(final_url) + unchain.error = "dns_failure" + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new_callable=AsyncMock, + ) as mock_heuristic, + patch( + "app.services.db_independent_pipeline.analyze_content", new_callable=AsyncMock + ) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = unchain + mock_heuristic.return_value = DomainHeuristicResult( + domain="xj3kq9vbnm2p7zla.com", + score=settings.score_caution_threshold, + signals=[DomainHeuristicSignal.DGA_LIKE], + ) + + result = await run_db_independent_pipeline("aid-unavailable-url-signal", final_url) + + assert isinstance(result, DbIndependentPipelineSuccess) + assert result.verdict == Verdict.CAUTION + assert result.score == settings.score_caution_threshold + assert result.stages.content_analysis.fetched is False + assert result.stages.content_analysis.error == "page_unavailable" + mock_content.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_degrades_when_heuristic_fails() -> None: + final_url = "https://error.example.com/login" + + async def _failing_heuristic(_: str) -> DomainHeuristicResult: + await asyncio.sleep(0.01) + raise RuntimeError("heuristic failed") + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new=AsyncMock(side_effect=_failing_heuristic), + ), + patch( + "app.services.db_independent_pipeline.analyze_content", + new_callable=AsyncMock, + ) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_content.return_value = _make_content(final_url) + + result = await run_db_independent_pipeline("aid-error-cleanup", final_url) + + assert isinstance(result, DbIndependentPipelineSuccess) + assert result.stages.domain_heuristic.rdap_error == "pipeline_timeout" + mock_content.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_caps_content_stage_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + final_url = "https://slow-content.example.com/login" + monkeypatch.setattr(settings, "pipeline_content_timeout_seconds", 0.001) + + async def _slow_content(_: str, **__: object) -> ContentAnalysisResult: + await asyncio.sleep(1) + return _make_content(final_url, score=90) + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + patch( + "app.services.db_independent_pipeline.check_domain_heuristic", + new_callable=AsyncMock, + ) as mock_heuristic, + patch( + "app.services.db_independent_pipeline.analyze_content", + new=AsyncMock(side_effect=_slow_content), + ), + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_heuristic.return_value = _make_heuristic(0) + + result = await run_db_independent_pipeline("aid-content-timeout", final_url) + + assert isinstance(result, DbIndependentPipelineSuccess) + assert result.stages.content_analysis.error == "pipeline_timeout" + assert result.timings is not None + assert result.timings.total_seconds < 0.5 + + +@pytest.mark.asyncio +async def test_db_independent_pipeline_returns_failure_on_normalize_error() -> None: + from app.core.exceptions import NormalizationError + + with ( + patch("app.services.db_independent_pipeline.normalize_url") as mock_norm, + patch( + "app.services.db_independent_pipeline.unchain_url", new_callable=AsyncMock + ) as mock_unchain, + ): + mock_norm.side_effect = NormalizationError("invalid") + + result = await run_db_independent_pipeline("aid-bad", "not a url") + + assert isinstance(result, DbIndependentPipelineFailure) + assert result.failed_at_stage == PipelineStage.NORMALIZE + assert result.timings is not None + assert result.timings.stages.normalize is not None + mock_unchain.assert_not_awaited() diff --git a/tests/services/test_evaluation_metrics.py b/tests/services/test_evaluation_metrics.py new file mode 100644 index 0000000..eca60f3 --- /dev/null +++ b/tests/services/test_evaluation_metrics.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from app.schemas.pipeline import Verdict +from app.services.evaluation.metrics import VerdictCase, compute_verdict_metrics + + +def test_compute_verdict_metrics_for_detection_goal() -> None: + cases = [ + VerdictCase(expected=Verdict.DANGER, predicted=Verdict.DANGER), + VerdictCase(expected=Verdict.DANGER, predicted=Verdict.DANGER), + VerdictCase(expected=Verdict.DANGER, predicted=Verdict.CAUTION), + VerdictCase(expected=Verdict.DANGER, predicted=Verdict.SAFE), + VerdictCase(expected=Verdict.SAFE, predicted=Verdict.SAFE), + VerdictCase(expected=Verdict.SAFE, predicted=Verdict.SAFE), + VerdictCase(expected=Verdict.SAFE, predicted=Verdict.CAUTION), + VerdictCase(expected=Verdict.SAFE, predicted=Verdict.DANGER), + ] + + metrics = compute_verdict_metrics( + cases, + positive_expected={Verdict.DANGER}, + positive_predicted={Verdict.DANGER, Verdict.CAUTION}, + ) + + assert metrics.total == 8 + assert metrics.accuracy == 0.5 + assert metrics.positive_recall == 0.75 + assert metrics.positive_precision == 0.6 + assert metrics.positive_f1 == 0.666667 + assert metrics.false_positive_rate == 0.5 diff --git a/tests/services/test_pipeline.py b/tests/services/test_pipeline.py new file mode 100644 index 0000000..3958ab3 --- /dev/null +++ b/tests/services/test_pipeline.py @@ -0,0 +1,857 @@ +"""run_pipeline 통합 회귀 — stage 추가 시 응답 스키마/순서 검증.""" + +from __future__ import annotations + +import asyncio +import inspect +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import PipelineFailure, PipelineStage, PipelineSuccess, Verdict +from app.schemas.threat_db import GSBMatch, GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import HopRecord, UnchainResult +from app.services.pipeline import run_pipeline + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +def _make_unchain(final_url: str, *, signals: list[str] | None = None) -> UnchainResult: + return UnchainResult( + input_url=final_url, + final_url=final_url, + hops=[], + hop_count=0, + signals=signals or [], + ) + + +def _make_threat(final_url: str) -> ThreatDbResult: + return ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=2, + gsb=GSBResult(checked=True), + urlhaus=URLhausResult(checked=True), + ) + + +def _make_heuristic(domain: str) -> DomainHeuristicResult: + return DomainHeuristicResult( + domain=domain, + score=15, + signals=[DomainHeuristicSignal.HOSTING_PLATFORM], + rdap=None, + rdap_error="not_found", + ) + + +def _make_content(final_url: str, *, score: int = 0) -> ContentAnalysisResult: + return ContentAnalysisResult(final_url=final_url, fetched=True, score=score, signals=[]) + + +def _missing_content(final_url: str, *, status_code: int = 404) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + status_code=status_code, + score=0, + signals=[ContentSignal.FETCH_FAILED], + reason="페이지를 찾을 수 없습니다.", + error=f"http_error_{status_code}", + ) + + +async def _resolve_upstream(value: object) -> object: + if inspect.isawaitable(value): + return await value + return value + + +async def _run_with_scores( + async_session: AsyncSession, + *, + final_url: str, + threat: ThreatDbResult, + heuristic_score: int, + content_score: int = 0, +) -> PipelineSuccess: + """verdict 매핑만 보고 싶을 때 쓰는 헬퍼 — stage 결과는 점수 외엔 의미 없음.""" + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = threat + mock_heuristic.return_value = _heuristic_with_score(heuristic_score) + mock_content.return_value = _make_content(final_url, score=content_score) + + result = await run_pipeline("aid-verdict", final_url, async_session) + assert isinstance(result, PipelineSuccess) + return result + + +@pytest.mark.asyncio +async def test_run_pipeline_includes_domain_heuristic_stage(async_session: AsyncSession) -> None: + """domain_heuristic stage 추가가 응답 스키마에 정상 반영되는지 검증.""" + final_url = "https://example.com/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _make_heuristic("example.com") + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-1", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.analysis_id == "aid-1" + assert result.final_url == final_url + assert result.stages.domain_heuristic.domain == "example.com" + assert result.stages.domain_heuristic.signals == [DomainHeuristicSignal.HOSTING_PLATFORM] + assert result.stages.content_analysis.final_url == final_url + assert result.stages.content_analysis.fetched is True + assert result.timings is not None + assert result.timings.total_seconds >= 0 + assert result.timings.stages.normalize is not None + assert result.timings.stages.unchain is not None + assert result.timings.stages.threat_db is not None + assert result.timings.stages.domain_heuristic is not None + assert result.timings.stages.content_analysis is not None + # threat_db → domain_heuristic → content_analysis 순서 — 모두 unchain.final_url 기준 + mock_heuristic.assert_awaited_once_with(final_url) + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + # _make_heuristic 가 HOSTING_PLATFORM 시그널을 가지므로 그대로 전달돼야 한다 + assert await _resolve_upstream(kwargs["upstream_signals"]) == ("HOSTING_PLATFORM",) + + +@pytest.mark.asyncio +async def test_run_pipeline_normalize_failure_skips_heuristic( + async_session: AsyncSession, +) -> None: + """normalize 단계 실패 시 후속 단계가 호출되지 않아야 한다.""" + from app.core.exceptions import NormalizationError + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock) as mock_h, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_c, + ): + mock_norm.side_effect = NormalizationError("invalid") + + result = await run_pipeline("aid-2", "not a url", async_session) + + assert isinstance(result, PipelineFailure) + assert result.failed_at_stage == PipelineStage.NORMALIZE + assert result.timings is not None + assert result.timings.total_seconds >= 0 + assert result.timings.stages.normalize is not None + assert result.timings.stages.unchain is None + assert result.timings.stages.threat_db is None + assert result.timings.stages.domain_heuristic is None + assert result.timings.stages.content_analysis is None + mock_h.assert_not_awaited() + mock_c.assert_not_awaited() + + +def _malicious_threat(final_url: str) -> ThreatDbResult: + return ThreatDbResult( + final_url=final_url, + is_malicious=True, + sources_checked=2, + gsb=GSBResult(checked=True, is_threat=True, matches=[GSBMatch(threat_type="MALWARE")]), + urlhaus=URLhausResult(checked=True, is_threat=False), + threat_types=["MALWARE"], + ) + + +def _heuristic_with_score(score: int) -> DomainHeuristicResult: + return DomainHeuristicResult( + domain="example.com", + score=score, + signals=[], + rdap=None, + rdap_error=None, + ) + + +@pytest.mark.asyncio +async def test_run_pipeline_skips_content_when_gsb_malicious( + async_session: AsyncSession, +) -> None: + """GSB 매치로 is_malicious=True 면 heuristic 점수와 무관하게 4단계를 skip 한다.""" + final_url = "https://evil.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock), + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + # GSB(+50) + heuristic(20) = 70 >= 61 → 건너뜀 + mock_heuristic.return_value = _heuristic_with_score(20) + + result = await run_pipeline("aid-skip", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # 병렬 선시작 이후에도 최종 응답은 skip 결과로 고정돼야 한다. + content = result.stages.content_analysis + assert content.fetched is False + assert content.error == "skipped_already_danger" + assert ContentSignal.SKIPPED_ALREADY_DANGER in content.signals + assert content.score == 0 + + +@pytest.mark.asyncio +async def test_run_pipeline_runs_content_when_below_danger( + async_session: AsyncSession, +) -> None: + final_url = "https://ok.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + # heuristic 점수만으로 임계 미달 → 콘텐츠 분석 정상 수행 + mock_heuristic.return_value = _heuristic_with_score(10) + mock_content.return_value = _make_content(final_url) + + await run_pipeline("aid-run", final_url, async_session) + + # heuristic 시그널 없는 경우 upstream_signals 는 빈 튜플로 전달돼야 한다 + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + assert await _resolve_upstream(kwargs["upstream_signals"]) == () + + +@pytest.mark.asyncio +async def test_run_pipeline_returns_failure_when_unchain_sees_404( + async_session: AsyncSession, +) -> None: + final_url = "https://missing.test/not-found" + unchain = UnchainResult( + input_url=final_url, + final_url=final_url, + hops=[], + hop_count=0, + signals=[], + ) + unchain.hops.append(HopRecord(url=final_url, status_code=404)) + unchain.hop_count = 1 + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = unchain + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-404", final_url, async_session) + + assert isinstance(result, PipelineFailure) + assert result.failed_at_stage == PipelineStage.UNCHAIN + assert result.error_code == "PAGE_UNAVAILABLE" + assert result.final_url == final_url + assert result.status_code == 404 + assert "페이지를 찾을 수 없습니다" in result.error + mock_threat.assert_awaited_once() + mock_heuristic.assert_awaited_once() + mock_content.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_run_pipeline_returns_failure_when_content_fetch_cannot_connect( + async_session: AsyncSession, +) -> None: + final_url = "https://offline.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(0) + mock_content.return_value = ContentAnalysisResult( + final_url=final_url, + fetched=False, + score=0, + signals=[ContentSignal.FETCH_FAILED], + reason="페이지에 연결할 수 없습니다.", + error="connect_error", + ) + + result = await run_pipeline("aid-connect", final_url, async_session) + + assert isinstance(result, PipelineFailure) + assert result.failed_at_stage == PipelineStage.CONTENT_ANALYSIS + assert result.error_code == "PAGE_UNAVAILABLE" + assert result.final_url == final_url + assert result.status_code is None + assert result.error == "페이지에 연결할 수 없습니다." + + +class TestVerdictAndScore: + """PipelineSuccess.verdict / score 매핑 회귀.""" + + async def test_safe_when_all_clean(self, async_session: AsyncSession) -> None: + final_url = "https://clean.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=0, + content_score=0, + ) + assert result.verdict == Verdict.SAFE + assert result.score == 0 + + async def test_caution_in_middle_band(self, async_session: AsyncSession) -> None: + """heuristic 35 단독 — caution(31~60) 구간.""" + final_url = "https://mid.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=35, + content_score=0, + ) + assert result.verdict == Verdict.CAUTION + assert result.score == 35 + + async def test_danger_by_score_alone(self, async_session: AsyncSession) -> None: + """blacklist 미스라도 합산 점수가 임계(61) 이상이면 danger.""" + final_url = "https://heur-heavy.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=65, + content_score=0, + ) + assert result.verdict == Verdict.DANGER + assert result.score == 65 + + async def test_score_capped_at_100(self, async_session: AsyncSession) -> None: + """단계별 점수 합이 100 을 넘으면 cap. preceding 임계 미만 → 4단계 합산까지 진입.""" + final_url = "https://overflow.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=10, + content_score=95, + ) + assert result.score == 100 + assert result.verdict == Verdict.DANGER + + async def test_danger_when_threat_matches_even_below_threshold( + self, async_session: AsyncSession + ) -> None: + """GSB 매치 시 known malicious 로 보고 score=100, verdict=danger.""" + final_url = "https://blacklist.test/" + # short-circuit 경로 — heuristic 은 placeholder, content skip + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock), + ): + mock_norm.return_value = NormalizeResult( + original_url=final_url, normalized_url=final_url + ) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-blk", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.score == 100 + assert result.verdict == Verdict.DANGER + + async def test_verdict_score_appear_before_stages_in_response( + self, async_session: AsyncSession + ) -> None: + """직렬화 시 verdict/score 가 stages 보다 앞에 나와 클라이언트가 상단에서 바로 읽도록.""" + final_url = "https://order.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=10, + ) + keys = list(result.model_dump().keys()) + assert keys.index("verdict") < keys.index("stages") + assert keys.index("score") < keys.index("stages") + + +@pytest.mark.asyncio +async def test_run_pipeline_passes_upstream_signals_to_content_analysis( + async_session: AsyncSession, +) -> None: + """heuristic 시그널 + threat 매치 플래그가 analyze_content 의 upstream_signals 로 전달된다.""" + final_url = "https://typo-naverr.test/" + + weak_threat = ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=2, + # 매치 자체는 False 라 short-circuit 안 일어나지만, 미래에 약한 매치만 있을 케이스 대비 + gsb=GSBResult(checked=True, is_threat=False), + urlhaus=URLhausResult(checked=True, is_threat=False), + ) + heuristic = DomainHeuristicResult( + domain="typo-naverr.test", + score=20, + signals=[DomainHeuristicSignal.TYPO_DOMAIN, DomainHeuristicSignal.NEW_DOMAIN], + rdap=None, + rdap_error=None, + ) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = weak_threat + mock_heuristic.return_value = heuristic + mock_content.return_value = _make_content(final_url) + + await run_pipeline("aid-up", final_url, async_session) + + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + assert await _resolve_upstream(kwargs["upstream_signals"]) == ("TYPO_DOMAIN", "NEW_DOMAIN") + + +@pytest.mark.asyncio +async def test_run_pipeline_scores_cross_origin_redirect_signal( + async_session: AsyncSession, +) -> None: + final_url = "https://redirected.example.com/login" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult( + original_url="https://short.test/a", + normalized_url="https://short.test/a", + ) + mock_unchain.return_value = _make_unchain( + final_url, + signals=["cross_origin:short.test->redirected.example.com"], + ) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(20) + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-redirect-score", "https://short.test/a", async_session) + + assert isinstance(result, PipelineSuccess) + assert DomainHeuristicSignal.REDIRECT_CROSS_ORIGIN in result.stages.domain_heuristic.signals + assert result.stages.domain_heuristic.score > 20 + args, kwargs = mock_content.await_args + assert args == (final_url,) + assert "REDIRECT_CROSS_ORIGIN" in await _resolve_upstream(kwargs["upstream_signals"]) + + +@pytest.mark.asyncio +async def test_run_pipeline_runs_threat_and_heuristic_in_parallel( + async_session: AsyncSession, +) -> None: + """2·3단계는 서로 독립이므로 gather 로 동시에 진입해야 한다. + + 각 stub 을 상대방의 Event 를 기다리게 만들어서, 순차 실행이면 데드락 → + wait_for 타임아웃으로 실패하도록 유도한다. 병렬이면 둘 다 진입 후 통과. + """ + final_url = "https://ok.test/" + + threat_started = asyncio.Event() + heuristic_started = asyncio.Event() + + async def fake_threat_db(_session: AsyncSession, _url: str) -> ThreatDbResult: + threat_started.set() + await asyncio.wait_for(heuristic_started.wait(), timeout=1.0) + return _make_threat(final_url) + + async def fake_heuristic(_url: str) -> DomainHeuristicResult: + heuristic_started.set() + await asyncio.wait_for(threat_started.wait(), timeout=1.0) + return _heuristic_with_score(10) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=fake_threat_db), + patch("app.services.pipeline.check_domain_heuristic", side_effect=fake_heuristic), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-parallel", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # 둘 다 실행됐고 4단계까지 이어졌는지 확인 + assert threat_started.is_set() and heuristic_started.is_set() + mock_content.assert_awaited_once() + args, _ = mock_content.await_args + assert args == (final_url,) + + +@pytest.mark.asyncio +async def test_run_pipeline_cancelled_error_propagates_from_parallel_stage( + async_session: AsyncSession, +) -> None: + """병렬 단계 중 하나에서 CancelledError 가 나면 degraded 흡수 없이 상위로 올라가야 한다.""" + final_url = "https://ok.test/" + + async def cancelled_threat(_session: AsyncSession, _url: str) -> ThreatDbResult: + raise asyncio.CancelledError + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=cancelled_threat), + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_heuristic.return_value = _heuristic_with_score(10) + + with pytest.raises(asyncio.CancelledError): + await run_pipeline("aid-cancel", final_url, async_session) + + +@pytest.mark.asyncio +async def test_run_pipeline_short_circuits_on_gsb_match_and_cancels_heuristic( + async_session: AsyncSession, +) -> None: + """GSB 매치 시: threat_db 결과가 나오자마자 heuristic 을 cancel 하고 + 4단계도 skip 한 채 즉시 종료해야 한다. verdict 가 이미 danger 로 확정이므로. + + placeholder heuristic 의 domain 은 등록 가능 도메인(파이프라인 정상 경로와 동일)이어야 + 하지 full host 가 들어가면 안 된다 — 다운스트림 버킷팅이 어긋난다. + """ + final_url = "https://signin.evil.example.com/login" + + heuristic_finished = asyncio.Event() + + async def slow_heuristic(_url: str) -> DomainHeuristicResult: + # RDAP 대기로 수 초 떠 있는 상황을 흉내낸다. cancel 되지 않으면 이벤트가 set 되어 + # 테스트가 그 사실을 검출한다. + try: + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + heuristic_finished.set() + return _heuristic_with_score(10) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch("app.services.pipeline.check_domain_heuristic", side_effect=slow_heuristic), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + + result = await run_pipeline("aid-sc", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.score == 100 + mock_content.assert_not_awaited() + # heuristic 은 cancel 되어 본문이 끝까지 돌지 않았어야 한다 + assert heuristic_finished.is_set() is False + # 4단계 응답은 skip + content = result.stages.content_analysis + assert content.fetched is False + assert content.error == "skipped_already_danger" + assert ContentSignal.SKIPPED_ALREADY_DANGER in content.signals + # heuristic placeholder 가 채워져 스키마 유지 — domain 은 registrable domain + heur = result.stages.domain_heuristic + assert heur.score == 0 + assert heur.rdap_error is None + assert heur.skipped_reason == "threat_matched" + assert heur.domain == "example.com" + + +@pytest.mark.asyncio +async def test_short_circuit_uses_placeholder_even_when_heuristic_finishes_first( + async_session: AsyncSession, +) -> None: + """heuristic 이 먼저 끝났더라도 threat malicious 면 heuristic 점수는 placeholder(0). + + 종전엔 task 종료 순서에 따라 heuristic 점수가 placeholder ↔ 실제 점수로 갈려 + 동일 입력에 응답 score 가 비결정적이었다. 두 분기 모두 placeholder 로 통일하는 + 회귀를 박는다 — verdict 는 DANGER 강제라 사용자 영향이 없지만 옵저버빌리티 보호. + """ + final_url = "https://heur-fast-threat-mal.test/" + + async def slow_threat(_session: AsyncSession, _url: str) -> ThreatDbResult: + # heuristic 보다 늦게 끝나도록 인위적 지연 — wait FIRST_COMPLETED 분기를 강제. + await asyncio.sleep(0.05) + return _malicious_threat(final_url) + + async def fast_heuristic(_url: str) -> DomainHeuristicResult: + # heuristic 이 먼저 끝남 — 실제 점수가 있는 결과를 반환 + return _heuristic_with_score(40) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=slow_threat), + patch("app.services.pipeline.check_domain_heuristic", side_effect=fast_heuristic), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock), + ): + mock_norm.return_value = NormalizeResult( + original_url=final_url, normalized_url=final_url + ) + mock_unchain.return_value = _make_unchain(final_url) + + result = await run_pipeline("aid-race", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # heuristic 이 실제 점수(40) 를 가졌어도 placeholder 로 갈아치워져야 한다. + assert result.stages.domain_heuristic.score == 0 + assert result.stages.domain_heuristic.rdap_error is None + assert result.stages.domain_heuristic.skipped_reason == "threat_matched" + assert result.score == 100 + assert result.verdict == Verdict.DANGER + + +@pytest.mark.asyncio +async def test_run_pipeline_short_circuits_on_urlhaus_match( + async_session: AsyncSession, +) -> None: + """URLhaus 단독 매치도 동일하게 조기 종료 — threat_db.is_malicious 가 True 이기 때문.""" + final_url = "https://urlhaus-hit.test/" + + urlhaus_only = ThreatDbResult( + final_url=final_url, + is_malicious=True, + sources_checked=2, + gsb=GSBResult(checked=True, is_threat=False), + urlhaus=URLhausResult(checked=True, is_threat=True), + ) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = urlhaus_only + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-urlhaus", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.score == 100 + mock_content.assert_not_awaited() + assert result.stages.content_analysis.error == "skipped_already_danger" + + +@pytest.mark.asyncio +async def test_run_pipeline_checks_content_when_heuristic_alone_exceeds_threshold( + async_session: AsyncSession, +) -> None: + """위협 DB 미매치인 휴리스틱 danger 는 페이지 존재 확인 후 verdict 를 낸다.""" + final_url = "https://typo-naverr.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(settings.score_danger_threshold) + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-heur", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + mock_content.assert_awaited_once() + assert result.verdict == Verdict.DANGER + assert result.stages.content_analysis.fetched is True + + +@pytest.mark.asyncio +async def test_run_pipeline_returns_verdict_when_page_unavailable_but_url_signal_is_strong( + async_session: AsyncSession, +) -> None: + final_url = "http://xj3kq9vbnm2p7zla.com/login" + unchain = _make_unchain(final_url) + unchain.error = "dns_failure" + heuristic = DomainHeuristicResult( + domain="xj3kq9vbnm2p7zla.com", + score=settings.score_caution_threshold, + signals=[DomainHeuristicSignal.DGA_LIKE], + ) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = unchain + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = heuristic + + result = await run_pipeline("aid-unavailable-url-signal", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.verdict == Verdict.CAUTION + assert result.score == settings.score_caution_threshold + assert result.stages.content_analysis.fetched is False + assert result.stages.content_analysis.error == "page_unavailable" + mock_content.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_run_pipeline_degrades_reputation_error_and_continues_to_content( + async_session: AsyncSession, +) -> None: + final_url = "https://error.test/" + + async def failing_threat(_session: AsyncSession, _url: str) -> ThreatDbResult: + raise RuntimeError("threat db failed") + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=failing_threat), + patch( + "app.services.pipeline.check_domain_heuristic", + new_callable=AsyncMock, + return_value=_heuristic_with_score(0), + ), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-error-cleanup", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.stages.threat_db.gsb.error == "pipeline_timeout" + assert result.stages.domain_heuristic.rdap_error == "pipeline_timeout" + mock_content.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_run_pipeline_degrades_content_error( + async_session: AsyncSession, +) -> None: + final_url = "https://content-error.test/" + + async def failing_content(_url: str, **_kwargs: object) -> ContentAnalysisResult: + raise RuntimeError("content failed") + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", side_effect=failing_content), + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-content-error", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.stages.content_analysis.error == "pipeline_timeout" diff --git a/tests/services/threat_db/__init__.py b/tests/services/threat_db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/threat_db/test_check.py b/tests/services/threat_db/test_check.py new file mode 100644 index 0000000..22e2732 --- /dev/null +++ b/tests/services/threat_db/test_check.py @@ -0,0 +1,151 @@ +"""check_threat_db 병렬 조회·폴백 검증.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from app.schemas.analysis import GSBMatch, GSBResult, URLhausResult +from app.services.threat_db.check import check_threat_db +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _run( + session: AsyncSession, + gsb: GSBResult, + urlhaus: URLhausResult, + url: str = "https://x.test/", +): + with ( + patch("app.services.threat_db.check.check_gsb", AsyncMock(return_value=gsb)), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=urlhaus), + ), + ): + return await check_threat_db(session, url) + + +async def test_both_clean(async_session: AsyncSession) -> None: + result = await _run( + async_session, + GSBResult(checked=True, is_threat=False), + URLhausResult(checked=True, is_threat=False), + ) + assert result.is_malicious is False + assert result.sources_checked == 2 + assert result.threat_types == [] + + +async def test_gsb_hit_marks_malicious(async_session: AsyncSession) -> None: + gsb = GSBResult( + checked=True, + is_threat=True, + matches=[GSBMatch(threat_type="MALWARE")], + ) + result = await _run( + async_session, + gsb, + URLhausResult(checked=True, is_threat=False), + ) + assert result.is_malicious is True + assert "MALWARE" in result.threat_types + + +async def test_urlhaus_hit_with_gsb_error(async_session: AsyncSession) -> None: + """GSB 실패 + URLhaus 히트 → 여전히 malicious 판정.""" + result = await _run( + async_session, + GSBResult(checked=False, is_threat=False, error="timeout"), + URLhausResult( + checked=True, + is_threat=True, + match_type="host", + matched_key="evil.test", + threat="malware_download", + ), + ) + assert result.is_malicious is True + assert result.sources_checked == 1 + assert "malware_download" in result.threat_types + + +async def test_both_fail_returns_clean_zero_sources( + async_session: AsyncSession, +) -> None: + result = await _run( + async_session, + GSBResult(checked=False, is_threat=False, error="timeout"), + URLhausResult(checked=False, is_threat=False, error="db_error"), + ) + assert result.is_malicious is False + assert result.sources_checked == 0 + + +async def test_exception_is_caught(async_session: AsyncSession) -> None: + with ( + patch( + "app.services.threat_db.check.check_gsb", + AsyncMock(side_effect=RuntimeError("boom")), + ), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + ), + ): + result = await check_threat_db(async_session, "https://x.test/") + assert result.gsb.checked is False + assert result.gsb.error == "unexpected" + assert result.sources_checked == 1 + + +async def test_cancelled_error_propagates(async_session: AsyncSession) -> None: + # 상위 shutdown/timeout 신호(CancelledError)는 절대 삼키지 않는다. + with ( + patch( + "app.services.threat_db.check.check_gsb", + AsyncMock(side_effect=asyncio.CancelledError()), + ), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + ), + pytest.raises(asyncio.CancelledError), + ): + await check_threat_db(async_session, "https://x.test/") + + +async def test_checks_original_and_final_url_candidates(async_session: AsyncSession) -> None: + clean = GSBResult(checked=True, is_threat=False) + hit = URLhausResult( + checked=True, + is_threat=True, + match_type="host", + matched_key="phish-origin.test", + threat="phishing", + ) + + with ( + patch("app.services.threat_db.check.check_gsb", AsyncMock(return_value=clean)) as mock_gsb, + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(side_effect=[URLhausResult(checked=True, is_threat=False), hit]), + ) as mock_urlhaus, + ): + result = await check_threat_db( + async_session, + "https://benign-final.test/", + original_url="https://phish-origin.test/login", + ) + + assert result.is_malicious is True + assert result.urlhaus.matched_key == "phish-origin.test" + assert [call.args[0] for call in mock_gsb.await_args_list] == [ + "https://benign-final.test/", + "https://phish-origin.test/login", + ] + assert [call.args[1] for call in mock_urlhaus.await_args_list] == [ + "https://benign-final.test/", + "https://phish-origin.test/login", + ] diff --git a/tests/services/threat_db/test_gsb.py b/tests/services/threat_db/test_gsb.py new file mode 100644 index 0000000..1b26f30 --- /dev/null +++ b/tests/services/threat_db/test_gsb.py @@ -0,0 +1,146 @@ +"""GSB check_gsb 단위 테스트 — httpx 모킹으로 각 실패 분기 검증.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.core.config import settings +from app.services.threat_db import gsb as gsb_module +from app.services.threat_db.gsb import check_gsb + + +@pytest.fixture(autouse=True) +def _with_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "gsb_api_key", "test-key") + # "키 값별 1회 경고" throttle 초기화 — 테스트 간 상태 누수 방지. + gsb_module.reset_missing_key_warning() + + +def _mock_post_response(resp: httpx.Response) -> AsyncMock: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(return_value=resp) + return client + + +async def test_no_api_key_returns_not_configured(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "gsb_api_key", None) + result = await check_gsb("https://example.com") + assert result.checked is False + assert result.error == "not_configured" + + +async def test_match_returns_is_threat() -> None: + body = { + "matches": [ + { + "threatType": "MALWARE", + "platformType": "ANY_PLATFORM", + "cacheDuration": "300s", + } + ] + } + resp = httpx.Response(200, json=body, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://evil.test/") + assert result.checked is True + assert result.is_threat is True + assert result.matches[0].threat_type == "MALWARE" + + +async def test_no_match_returns_safe() -> None: + resp = httpx.Response(200, json={}, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://safe.test/") + assert result.checked is True + assert result.is_threat is False + assert result.matches == [] + + +async def test_timeout_mapped_to_error() -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(side_effect=httpx.TimeoutException("timeout")) + with patch("app.services.threat_db.gsb.httpx.AsyncClient", return_value=client): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "timeout" + + +@pytest.mark.parametrize( + "status,expected", + [ + (400, "auth_error"), + (401, "auth_error"), + (403, "auth_error"), + (429, "rate_limited"), + (500, "server_error_500"), + (503, "server_error_503"), + ], +) +async def test_http_status_errors(status: int, expected: str) -> None: + resp = httpx.Response(status, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == expected + + +async def test_invalid_json() -> None: + resp = httpx.Response(200, content=b"not json", request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "invalid_response" + + +async def test_unmapped_non_200_returns_http_error() -> None: + # 3xx/기타 비매핑 상태코드는 스펙대로 "http_error" 로 평탄화된다. + resp = httpx.Response(301, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "http_error" + + +async def test_missing_key_warning_throttled_per_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 같은 키 상태로 두 번 호출해도 경고는 1회. 키 값이 바뀌면 다시 1회. + warnings: list[str] = [] + monkeypatch.setattr(settings, "gsb_api_key", None) + gsb_module.reset_missing_key_warning() + monkeypatch.setattr(gsb_module.logger, "warning", lambda event, **kw: warnings.append(event)) + await check_gsb("https://x.test/") + await check_gsb("https://y.test/") + assert warnings.count("gsb.api_key_not_configured") == 1 + + +async def test_generic_http_error() -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(side_effect=httpx.HTTPError("boom")) + with patch("app.services.threat_db.gsb.httpx.AsyncClient", return_value=client): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "http_error" diff --git a/tests/services/threat_db/test_match_keys.py b/tests/services/threat_db/test_match_keys.py new file mode 100644 index 0000000..61cd3d1 --- /dev/null +++ b/tests/services/threat_db/test_match_keys.py @@ -0,0 +1,46 @@ +"""match_keys.derive_keys 단위 테스트.""" + +from __future__ import annotations + +from app.services.threat_db.match_keys import derive_keys + + +def test_host_only_for_standard_domain() -> None: + assert derive_keys("https://example.com/path/deep") == ["example.com"] + + +def test_github_returns_host_path_only() -> None: + keys = derive_keys("https://github.com/alice/repo/blob/main/x.exe") + assert keys == ["github.com/alice/repo"] + + +def test_raw_githubusercontent() -> None: + keys = derive_keys("https://raw.githubusercontent.com/alice/repo/main/x.sh") + assert keys == ["raw.githubusercontent.com/alice/repo"] + + +def test_github_short_path_does_not_fallback_to_host() -> None: + # 다중 테넌트 호스트는 특정 사용자/리포 단위 이하로는 매칭하지 않는다. + assert derive_keys("https://github.com/alice") == [] + + +def test_dropbox_root_does_not_fallback_to_host() -> None: + assert derive_keys("https://www.dropbox.com/") == [] + + +def test_dropbox_shared_file_uses_path_prefix() -> None: + keys = derive_keys("https://www.dropbox.com/scl/fi/abc/report.exe?dl=0") + assert keys == ["www.dropbox.com/scl/fi"] + + +def test_dropboxusercontent_download_host_uses_path_prefix() -> None: + keys = derive_keys("https://dl.dropboxusercontent.com/scl/fi/abc/report.exe") + assert keys == ["dl.dropboxusercontent.com/scl/fi"] + + +def test_empty_host_returns_empty_list() -> None: + assert derive_keys("not a url") == [] + + +def test_host_case_is_normalized() -> None: + assert derive_keys("https://Example.COM/x") == ["example.com"] diff --git a/tests/services/threat_db/test_urlhaus.py b/tests/services/threat_db/test_urlhaus.py new file mode 100644 index 0000000..c546055 --- /dev/null +++ b/tests/services/threat_db/test_urlhaus.py @@ -0,0 +1,138 @@ +"""URLhaus 조회 단위 테스트 — 인메모리 SQLite 시딩.""" + +from __future__ import annotations + +from datetime import datetime +from unittest.mock import AsyncMock, patch + +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db.urlhaus import check_urlhaus +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _seed(session: AsyncSession, **kwargs) -> None: + defaults = { + "threat": "malware_download", + "tags": "exe,emotet", + "url_status": "online", + "date_added": datetime(2026, 4, 10, 0, 0, 0), + "last_online": None, + "urlhaus_link": "https://urlhaus.abuse.ch/url/1/", + "reporter": "tester", + "synced_at": datetime(2026, 4, 15, 0, 0, 0), + } + defaults.update(kwargs) + session.add(URLhausEntry(**defaults)) + await session.flush() + + +async def test_exact_url_match(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=1, + url="https://evil.test/malware.exe", + host="evil.test", + match_key="evil.test", + ) + result = await check_urlhaus(async_session, "https://evil.test/malware.exe") + assert result.is_threat is True + assert result.match_type == "url" + assert result.threat == "malware_download" + assert result.tags == ["exe", "emotet"] + + +async def test_exact_url_match_tolerates_scheme_and_trailing_slash_variants( + async_session: AsyncSession, +) -> None: + await _seed( + async_session, + id=11, + url="http://evil.test/login", + host="evil.test", + match_key="evil.test", + ) + + result = await check_urlhaus(async_session, "https://evil.test/login/") + + assert result.is_threat is True + assert result.match_type == "url" + assert result.matched_key == "http://evil.test/login" + + +async def test_host_match(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=2, + url="https://evil.test/other.bin", + host="evil.test", + match_key="evil.test", + ) + result = await check_urlhaus(async_session, "https://evil.test/something-else") + assert result.is_threat is True + assert result.match_type == "host" + assert result.matched_key == "evil.test" + + +async def test_host_path_match_github(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=3, + url="https://github.com/bad/repo/raw/main/x.exe", + host="github.com", + match_key="github.com/bad/repo", + ) + result = await check_urlhaus(async_session, "https://github.com/bad/repo/issues/1") + assert result.is_threat is True + assert result.match_type == "host_path" + assert result.matched_key == "github.com/bad/repo" + + +async def test_multitenant_host_does_not_match_host_only_entry( + async_session: AsyncSession, +) -> None: + await _seed( + async_session, + id=6, + url="https://www.dropbox.com/scl/fi/bad/payload.exe", + host="www.dropbox.com", + match_key="www.dropbox.com", + ) + + result = await check_urlhaus(async_session, "https://www.dropbox.com/") + + assert result.checked is True + assert result.is_threat is False + + +async def test_multitenant_host_path_match_dropbox(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=7, + url="https://www.dropbox.com/scl/fi/bad/payload.exe", + host="www.dropbox.com", + match_key="www.dropbox.com/scl/fi", + ) + + result = await check_urlhaus( + async_session, + "https://www.dropbox.com/scl/fi/bad/readme.txt", + ) + + assert result.is_threat is True + assert result.match_type == "host_path" + assert result.matched_key == "www.dropbox.com/scl/fi" + + +async def test_no_match(async_session: AsyncSession) -> None: + result = await check_urlhaus(async_session, "https://clean.test/") + assert result.checked is True + assert result.is_threat is False + + +async def test_db_error_returns_checked_false(async_session: AsyncSession) -> None: + from sqlalchemy.exc import SQLAlchemyError + + with patch.object(async_session, "execute", AsyncMock(side_effect=SQLAlchemyError("boom"))): + result = await check_urlhaus(async_session, "https://x.test/") + assert result.checked is False + assert result.error == "db_error" diff --git a/tests/services/threat_db/test_urlhaus_sync.py b/tests/services/threat_db/test_urlhaus_sync.py new file mode 100644 index 0000000..8daf807 --- /dev/null +++ b/tests/services/threat_db/test_urlhaus_sync.py @@ -0,0 +1,120 @@ +"""URLhaus 동기화 단위 테스트 — 인메모리 SQLite + httpx 픽스처.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.db.base import Base +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db import urlhaus_sync as sync_module +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +SAMPLE_CSV = """# URLhaus recent URLs +# id, dateadded, url, url_status, last_online, threat, tags, urlhaus_link, reporter +1,2026-04-14 00:00:00,https://evil.test/a.exe,online,,malware_download,"exe,emotet",https://urlhaus.abuse.ch/url/1/,tester +2,2026-04-14 00:05:00,https://github.com/bad/repo/raw/main/x.sh,online,,malware_download,"sh",https://urlhaus.abuse.ch/url/2/,tester +3,2026-04-14 00:10:00,https://www.dropbox.com/scl/fi/bad/payload.exe,online,,malware_download,"exe",https://urlhaus.abuse.ch/url/3/,tester +""" + + +@pytest.fixture +async def sync_engine_patch(monkeypatch: pytest.MonkeyPatch): + """SessionLocal 을 인메모리 DB 로 치환.""" + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + maker = async_sessionmaker(bind=engine, expire_on_commit=False) + monkeypatch.setattr(sync_module, "SessionLocal", maker) + yield maker + await engine.dispose() + + +def _mock_csv_response(text: str, status: int = 200) -> AsyncMock: + resp = httpx.Response( + status, + text=text, + request=httpx.Request("GET", "https://x/"), + ) + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.get = AsyncMock(return_value=resp) + return client + + +async def test_sync_inserts_rows(sync_engine_patch) -> None: + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(SAMPLE_CSV), + ): + stats = await sync_module.sync_urlhaus() + + # 최초 실행 — 모두 insert, update 는 0 이어야 한다(C1 회귀 방지). + assert stats["total"] == 3 + assert stats["inserted"] == 3 + assert stats["updated"] == 0 + assert stats["failed"] == 0 + + async with sync_engine_patch() as session: + rows = (await session.execute(select(URLhausEntry))).scalars().all() + assert len(rows) == 3 + by_id = {r.id: r for r in rows} + assert by_id[1].host == "evil.test" + assert by_id[1].match_key == "evil.test" + assert by_id[2].match_key == "github.com/bad/repo" + assert by_id[3].match_key == "www.dropbox.com/scl/fi" + + +async def test_sync_idempotent(sync_engine_patch) -> None: + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(SAMPLE_CSV), + ): + await sync_module.sync_urlhaus() + stats2 = await sync_module.sync_urlhaus() + + # 두 번째 실행은 모두 update 여야 한다 — insert/update 분류가 맞는지 검증. + async with sync_engine_patch() as session: + rows = (await session.execute(select(URLhausEntry))).scalars().all() + assert len(rows) == 3 + assert stats2["total"] == 3 + assert stats2["inserted"] == 0 + assert stats2["updated"] == 3 + assert stats2["failed"] == 0 + + +async def test_sync_short_row_counts_failed(sync_engine_patch) -> None: + # 필드 수가 모자라는 행은 스킵하되 failed 로 카운트되어야 한다(I6). + bad_csv = ( + "# header comment\n" + "1,2026-04-14 00:00:00,https://evil.test/a.exe\n" # 필드 부족 + '2,2026-04-14 00:05:00,https://good.test/b,online,,malware_download,"",' + "https://urlhaus.abuse.ch/url/2/,tester\n" + ) + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(bad_csv), + ): + stats = await sync_module.sync_urlhaus() + assert stats["total"] == 2 + assert stats["failed"] == 1 + assert stats["inserted"] == 1 + + +async def test_sync_network_failure_returns_empty_stats( + sync_engine_patch, +) -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.get = AsyncMock(side_effect=httpx.HTTPError("net")) + with patch("app.services.threat_db.urlhaus_sync.httpx.AsyncClient", return_value=client): + stats = await sync_module.sync_urlhaus() + assert stats["total"] == 0 + assert stats["inserted"] == 0 diff --git a/tests/services/unchainer/__init__.py b/tests/services/unchainer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py new file mode 100644 index 0000000..b3539e6 --- /dev/null +++ b/tests/services/unchainer/test_unchain.py @@ -0,0 +1,696 @@ +"""unchain_url 단위 테스트. + +httpx.AsyncClient를 모킹해서 네트워크 요청 없이 리다이렉트 체인 로직을 검증. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.services.unchainer.unchain import unchain_url + + +def _make_response( + status_code: int = 200, + headers: dict[str, str] | None = None, +) -> httpx.Response: + """테스트용 httpx.Response 생성.""" + resp = httpx.Response( + status_code=status_code, + headers=headers or {}, + request=httpx.Request("HEAD", "https://example.com"), + ) + resp.aclose = AsyncMock() # 스트리밍 close 호출 안전하게 처리 + return resp + + +def _mock_client( + responses: list[httpx.Response] | None = None, + *, + side_effect=None, +) -> AsyncMock: + """httpx.AsyncClient mock 생성. 반복되는 보일러플레이트 제거용. + + build_request + send(stream=True) 방식에 맞춰 mock 구성. + side_effect 함수는 기존과 동일하게 (method, *args, **kwargs) 시그니처. + """ + mock = AsyncMock() + + # build_request는 동기 메서드 — 실제 httpx.Request 반환 + mock.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + + if side_effect is not None: + if isinstance(side_effect, BaseException): + mock.send = AsyncMock(side_effect=side_effect) + else: + _original = side_effect + + async def _send_wrapper(req, **kwargs): + if asyncio.iscoroutinefunction(_original): + return await _original(req.method, str(req.url)) + return _original(req.method, str(req.url)) + + mock.send = AsyncMock(side_effect=_send_wrapper) + elif responses is not None: + mock.send = AsyncMock(side_effect=responses) + else: + mock.send = AsyncMock(return_value=_make_response(200)) + + mock.__aenter__ = AsyncMock(return_value=mock) + mock.__aexit__ = AsyncMock(return_value=False) + return mock + + +_PATCH_TARGET = "app.services.unchainer.unchain.httpx.AsyncClient" + + +@pytest.fixture(autouse=True) +def _bypass_host_safety(monkeypatch): + """기본적으로 SSRF 검사를 우회 — SSRF 전용 테스트에서만 직접 패치. + + unchain 의 모듈 싱글턴 _client 도 매 테스트에서 None 으로 리셋해, 패치된 + httpx.AsyncClient 가 새로 호출되어 mock client 가 _client 에 박히도록 보장한다. + """ + from app.services.unchainer import unchain as unchain_module + + monkeypatch.setattr( + "app.services.unchainer.unchain._check_host_safety", + AsyncMock(return_value=None), + ) + monkeypatch.setattr(unchain_module, "_client", None) + + +class TestBasicChain: + """리다이렉트 없는 경우 / 단일·다중 hop 체인.""" + + @pytest.mark.asyncio + async def test_no_redirect(self) -> None: + client = _mock_client([_make_response(200)]) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.final_url == "https://example.com/" + assert result.hop_count == 1 + assert result.hops[0].status_code == 200 + assert result.error is None + assert not result.signals + + @pytest.mark.asyncio + async def test_single_redirect(self) -> None: + responses = [ + _make_response(301, {"location": "https://example.com/new"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/old") + + assert result.final_url == "https://example.com/new" + assert result.hop_count == 2 + assert result.hops[0].status_code == 301 + assert result.hops[0].location == "https://example.com/new" + assert result.hops[1].status_code == 200 + + @pytest.mark.asyncio + async def test_multi_hop_chain(self) -> None: + responses = [ + _make_response(302, {"location": "https://a.com/2"}), + _make_response(302, {"location": "https://a.com/3"}), + _make_response(302, {"location": "https://a.com/final"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://a.com/1") + + assert result.final_url == "https://a.com/final" + assert result.hop_count == 4 + + @pytest.mark.asyncio + async def test_schemeless_prefers_https_when_https_is_analyzable(self) -> None: + def _side_effect(method: str, url: str) -> httpx.Response: + assert method == "HEAD" + if url == "https://example.com/path": + return _make_response(200, {"content-type": "text/html; charset=utf-8"}) + raise AssertionError(f"unexpected request: {method} {url}") + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url( + "http://example.com/path", + prefer_https_when_schemeless=True, + ) + + assert result.final_url == "https://example.com/path" + assert "schemeless_https_upgrade" in result.signals + + @pytest.mark.asyncio + async def test_schemeless_falls_back_to_http_when_https_is_not_analyzable(self) -> None: + requests: list[tuple[str, str]] = [] + + def _side_effect(method: str, url: str) -> httpx.Response: + requests.append((method, url)) + if url == "https://example.com/path": + return _make_response(404) + if url == "http://example.com/path": + return _make_response(200) + raise AssertionError(f"unexpected request: {method} {url}") + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url( + "http://example.com/path", + prefer_https_when_schemeless=True, + ) + + assert result.final_url == "http://example.com/path" + assert result.hops[0].status_code == 200 + assert "schemeless_https_upgrade" not in result.signals + assert requests == [ + ("HEAD", "https://example.com/path"), + ("HEAD", "http://example.com/path"), + ] + + +class TestRedirectLoop: + """무한 루프 감지.""" + + @pytest.mark.asyncio + async def test_loop_detected(self) -> None: + """A → B → A 루프 감지 시 redirect_loop 신호.""" + responses = [ + _make_response(302, {"location": "https://b.com/"}), + _make_response(302, {"location": "https://a.com/"}), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://a.com/") + + assert "redirect_loop" in result.signals + assert result.hop_count == 2 + + +class TestMaxHops: + """hop 수 제한.""" + + @pytest.mark.asyncio + async def test_max_hops_reached(self) -> None: + hop_count = 0 + + def _side_effect(*args: object, **kwargs: object) -> httpx.Response: + nonlocal hop_count + hop_count += 1 + return _make_response( + 302, + {"location": f"https://example.com/{hop_count}"}, + ) + + client = _mock_client(side_effect=_side_effect) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch("app.services.unchainer.unchain.settings") as mock_settings, + ): + mock_settings.unchain_max_hops = 5 + mock_settings.unchain_timeout_seconds = 5.0 + mock_settings.unchain_connect_timeout_seconds = 3.0 + mock_settings.unchain_chain_timeout_seconds = 20.0 + mock_settings.unchain_user_agent = "test-agent" + result = await unchain_url("https://example.com/start") + + assert "max_hops_reached" in result.signals + assert result.hop_count == 5 + + +class TestSchemeDowngrade: + """https → http 스킴 다운그레이드 감지.""" + + @pytest.mark.asyncio + async def test_scheme_downgrade_signal(self) -> None: + responses = [ + _make_response(302, {"location": "http://example.com/insecure"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert "scheme_downgrade" in result.signals + + @pytest.mark.asyncio + async def test_no_signal_on_http_to_https(self) -> None: + responses = [ + _make_response(302, {"location": "https://example.com/secure"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("http://example.com/") + + assert "scheme_downgrade" not in result.signals + + +class TestCrossOrigin: + """크로스 오리진 호스트 변화 감지.""" + + @pytest.mark.asyncio + async def test_cross_origin_signal(self) -> None: + responses = [ + _make_response(302, {"location": "https://evil.com/phish"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://safe.com/link") + + assert any(s.startswith("cross_origin:") for s in result.signals) + assert "cross_origin:safe.com->evil.com" in result.signals + + @pytest.mark.asyncio + async def test_same_registered_domain_redirect_is_not_cross_origin(self) -> None: + responses = [ + _make_response(301, {"location": "https://github.com/"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://www.github.com/") + + assert not any(s.startswith("cross_origin:") for s in result.signals) + assert result.final_url == "https://github.com/" + + +class TestRelativeLocation: + """상대 경로 Location 해석.""" + + @pytest.mark.asyncio + async def test_relative_path_resolved(self) -> None: + responses = [ + _make_response(302, {"location": "/new-path"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/old-path") + + assert result.final_url == "https://example.com/new-path" + assert result.hops[0].location == "https://example.com/new-path" + + @pytest.mark.asyncio + async def test_raw_location_preserved(self) -> None: + """상대 경로 원본 값이 raw_location에 보존되는지 확인.""" + responses = [ + _make_response(302, {"location": "/new-path"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/old-path") + + assert result.hops[0].raw_location == "/new-path" + assert result.hops[0].location == "https://example.com/new-path" + + +class TestHeadFallback: + """HEAD 실패 시 GET 폴백.""" + + @pytest.mark.asyncio + async def test_head_405_falls_back_to_get(self) -> None: + responses = [ + _make_response(405), # HEAD 실패 + _make_response(200), # GET 성공 + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + @pytest.mark.asyncio + async def test_head_timeout_falls_back_to_get(self) -> None: + """HEAD에서 타임아웃 → GET으로 폴백해서 성공.""" + call_count = 0 + + def _side_effect(method: str, *args: object, **kwargs: object) -> httpx.Response: + nonlocal call_count + call_count += 1 + if method == "HEAD": + raise httpx.ReadTimeout("timed out") + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + @pytest.mark.asyncio + async def test_head_redirect_without_location_falls_back_to_get(self) -> None: + """HEAD가 리다이렉트인데 Location 누락 → GET으로 폴백해서 체인 추적.""" + + def _side_effect(method: str, url: str) -> httpx.Response: + if url == "https://example.com/start": + # HEAD는 301인데 Location 헤더 생략, GET은 정상 응답 + if method == "HEAD": + return _make_response(301) + return _make_response(301, {"location": "https://example.com/final"}) + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/start") + + assert result.error is None + assert result.final_url == "https://example.com/final" + assert result.hops[0].method == "GET" + assert result.hops[0].status_code == 301 + + @pytest.mark.asyncio + async def test_head_connect_error_falls_back_to_get(self) -> None: + """HEAD에서 ConnectError → GET으로 폴백해서 성공.""" + + def _side_effect(method: str, *args: object, **kwargs: object) -> httpx.Response: + if method == "HEAD": + raise httpx.ConnectError("Connection refused") + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + +class TestErrorHandling: + """네트워크·서버 에러 처리.""" + + @pytest.mark.asyncio + async def test_timeout_error(self) -> None: + """HEAD·GET 모두 타임아웃이면 최종 에러.""" + client = _mock_client(side_effect=httpx.ReadTimeout("timed out")) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://slow.example.com/") + + assert result.timed_out is True + assert result.error == "timeout" + + @pytest.mark.asyncio + async def test_dns_failure_via_connect_error(self) -> None: + client = _mock_client( + side_effect=httpx.ConnectError("[Errno -2] Name or service not known"), + ) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://nonexistent.invalid/") + + assert result.error == "dns_failure" + + @pytest.mark.asyncio + async def test_connection_refused(self) -> None: + client = _mock_client( + side_effect=httpx.ConnectError("Connection refused"), + ) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://refused.example.com/") + + assert result.error is not None + assert "connection_refused" in result.error + + @pytest.mark.asyncio + async def test_server_error_5xx(self) -> None: + client = _mock_client([_make_response(503)]) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://down.example.com/") + + assert result.error == "server_error_503" + assert result.hop_count == 1 + assert result.hops[0].status_code == 503 + + @pytest.mark.asyncio + async def test_missing_location_header(self) -> None: + # HEAD·GET 모두 Location 헤더 누락 — 최종적으로 missing_location_header + client = _mock_client([_make_response(302), _make_response(302)]) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://broken.example.com/") + + assert result.error == "missing_location_header" + + +class TestChainTimeout: + """전체 체인 총 timeout.""" + + @pytest.mark.asyncio + async def test_chain_timeout(self) -> None: + """hop별로는 안 터지지만 전체 체인이 총 timeout을 초과하면 중단.""" + + async def _slow_request(*args: object, **kwargs: object) -> httpx.Response: + await asyncio.sleep(10) + return _make_response(302, {"location": "https://example.com/next"}) + + client = _mock_client(side_effect=_slow_request) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch("app.services.unchainer.unchain.settings") as mock_settings, + ): + mock_settings.unchain_max_hops = 20 + mock_settings.unchain_timeout_seconds = 5.0 + mock_settings.unchain_connect_timeout_seconds = 3.0 + mock_settings.unchain_chain_timeout_seconds = 0.1 + mock_settings.unchain_user_agent = "test-agent" + result = await unchain_url("https://example.com/start") + + assert result.timed_out is True + assert result.error == "chain_timeout" + + +class TestUnsafeScheme: + """javascript:, data: 등 비허용 스킴 리다이렉트 차단.""" + + @pytest.mark.asyncio + async def test_javascript_scheme_blocked(self) -> None: + responses = [ + _make_response(302, {"location": "javascript:alert(1)"}), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.error == "unsafe_redirect_scheme:javascript" + assert "unsafe_scheme:javascript" in result.signals + + @pytest.mark.asyncio + async def test_data_scheme_blocked(self) -> None: + responses = [ + _make_response(302, {"location": "data:text/html,

phish

"}), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.error == "unsafe_redirect_scheme:data" + assert "unsafe_scheme:data" in result.signals + + +class TestHopRecording: + """hop 기록 정확성.""" + + @pytest.mark.asyncio + async def test_hop_records_method_and_url(self) -> None: + responses = [ + _make_response(302, {"location": "https://b.com/"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://a.com/") + + assert result.hops[0].url == "https://a.com/" + assert result.hops[0].method == "HEAD" + assert result.hops[1].url == "https://b.com/" + + @pytest.mark.asyncio + async def test_all_redirect_status_codes(self) -> None: + """301, 302, 303, 307, 308 모두 리다이렉트로 처리.""" + from app.services.unchainer import unchain as unchain_module + + for code in (301, 302, 303, 307, 308): + responses = [ + _make_response(code, {"location": "https://example.com/dest"}), + _make_response(200), + ] + client = _mock_client(responses) + + # 모듈 싱글턴 — iteration 마다 None 으로 리셋해 새 mock client 가 박히도록. + unchain_module._client = None + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/src") + + assert result.hops[0].status_code == code, f"Failed for status {code}" + assert result.final_url == "https://example.com/dest" + + +class TestSsrfProtection: + """SSRF·DNS Rebinding 방어.""" + + @pytest.mark.asyncio + async def test_loopback_blocked(self) -> None: + """127.0.0.1 등 루프백 주소 차단.""" + with ( + patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ), + patch("app.services.unchainer.unchain._build_client") as build_client, + ): + result = await unchain_url("http://127.0.0.1/admin") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + assert result.hop_count == 0 + build_client.assert_not_called() + + @pytest.mark.asyncio + async def test_private_ip_blocked(self) -> None: + """사설 IP 대역(10.x, 172.16.x, 192.168.x) 차단.""" + for url in ( + "http://10.0.0.1/", + "http://172.16.0.1/", + "http://192.168.1.1/", + ): + with patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ): + result = await unchain_url(url) + + assert result.error == "ssrf_blocked", f"Failed for {url}" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_metadata_endpoint_blocked(self) -> None: + """클라우드 메타데이터 엔드포인트(169.254.169.254) 차단.""" + with patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ): + result = await unchain_url("http://169.254.169.254/latest/meta-data/") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_redirect_to_internal_blocked(self) -> None: + """외부 URL이 내부 IP로 리다이렉트할 때 두 번째 hop에서 차단.""" + safety_call_count = 0 + + async def _safety_per_hop(url: str) -> str | None: + nonlocal safety_call_count + safety_call_count += 1 + # 첫 번째 hop은 통과, 두 번째(내부 IP)는 차단 + if "internal" in url or "127.0.0.1" in url: + return "ssrf_blocked" + return None + + responses = [ + _make_response(302, {"location": "http://127.0.0.1/secret"}), + ] + client = _mock_client(responses) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch( + "app.services.unchainer.unchain._check_host_safety", + side_effect=_safety_per_hop, + ), + ): + result = await unchain_url("https://evil.com/redirect") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_public_url_passes(self) -> None: + """공개 IP는 정상 통과.""" + client = _mock_client([_make_response(200)]) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value=None, + ), + ): + result = await unchain_url("https://example.com/") + + assert result.error is None + assert "ssrf_blocked" not in result.signals + + +class TestIsPrivateIp: + """_is_private_ip 단위 테스트.""" + + def test_loopback(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("127.0.0.1") is True + assert _is_private_ip("::1") is True + + def test_private_ranges(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("10.0.0.1") is True + assert _is_private_ip("172.16.0.1") is True + assert _is_private_ip("192.168.1.1") is True + + def test_link_local(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("169.254.169.254") is True + + def test_public_ip(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("8.8.8.8") is False + assert _is_private_ip("1.1.1.1") is False + + def test_invalid_ip(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("not-an-ip") is True diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..db4760b --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import subprocess +import sys + + +def _run_python(code: str) -> str: + result = subprocess.run( + [sys.executable, "-c", code], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def test_select_ai_provider_always_uses_openai_even_when_disabled(): + output = _run_python( + "import sys\n" + "import app.main as main\n" + "main.settings.ai_provider = 'null'\n" + "main.settings.openai_api_key = None\n" + "provider = main._select_ai_provider()\n" + "print(type(provider).__name__, 'openai' in sys.modules)\n" + ) + + assert output == "OpenAIProvider False" + + +def test_select_ai_provider_with_key_defers_openai_sdk_import(): + output = _run_python( + "import sys\n" + "import app.main as main\n" + "main.settings.ai_provider = 'auto'\n" + "main.settings.openai_api_key = 'test-key'\n" + "provider = main._select_ai_provider()\n" + "print(type(provider).__name__, 'openai' in sys.modules)\n" + ) + + assert output == "OpenAIProvider False" diff --git a/tests/test_makefile.py b/tests/test_makefile.py new file mode 100644 index 0000000..3d24ac6 --- /dev/null +++ b/tests/test_makefile.py @@ -0,0 +1,18 @@ +from pathlib import Path + + +def test_run_target_does_not_use_reload_watcher() -> None: + makefile = Path("Makefile").read_text() + run_block = makefile.split("run: ## Run the API locally", 1)[1].split("\n\n", 1)[0] + + assert "--reload" not in run_block + + +def test_dev_target_reloads_app_only() -> None: + makefile = Path("Makefile").read_text() + dev_block = makefile.split("dev: ## Run the API locally with hot reload", 1)[1].split( + "\n\n", 1 + )[0] + + assert "--reload-dir app" in dev_block + assert "--reload-exclude .venv" in dev_block