diff --git a/.cursor/Dockerfile b/.cursor/Dockerfile new file mode 100644 index 00000000..ae842d0f --- /dev/null +++ b/.cursor/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV ES_VERSION=7.10.1 + +# System dependencies: Python 3.12 (matches CI/dev), build toolchain for +# mysqlclient/numpy/pandas, MySQL server, and utilities. +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.12 python3.12-venv python3.12-dev python3-pip \ + build-essential pkg-config \ + default-libmysqlclient-dev libmagic1 libcairo2-dev \ + mysql-server \ + git curl wget unzip ca-certificates tzdata sudo procps \ + && rm -rf /var/lib/apt/lists/* + +# Provide a plain `python` pointing at 3.12 +RUN ln -sf /usr/bin/python3.12 /usr/local/bin/python + +# Elasticsearch 7.10.1 (matches docker-compose.yml + CI service) +RUN wget -q "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz" -O /tmp/es.tgz \ + && tar xzf /tmp/es.tgz -C /opt \ + && mv "/opt/elasticsearch-${ES_VERSION}" /opt/elasticsearch \ + && rm /tmp/es.tgz \ + && useradd -r -m -d /opt/elasticsearch-home elasticsearch \ + && mkdir -p /var/lib/elasticsearch /var/log/elasticsearch \ + && chown -R elasticsearch:elasticsearch /opt/elasticsearch /var/lib/elasticsearch /var/log/elasticsearch + +COPY .cursor/elasticsearch.yml /opt/elasticsearch/config/elasticsearch.yml +RUN sed -i 's/^-Xms.*/-Xms512m/; s/^-Xmx.*/-Xmx512m/' /opt/elasticsearch/config/jvm.options \ + && chown elasticsearch:elasticsearch /opt/elasticsearch/config/elasticsearch.yml + +# Ubuntu 24.04 ships a `ubuntu` user (uid 1000); grant passwordless sudo so +# start.sh can manage MySQL and Elasticsearch services. +RUN if ! id ubuntu >/dev/null 2>&1; then useradd -m -s /bin/bash ubuntu; fi \ + && echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu \ + && chmod 0440 /etc/sudoers.d/ubuntu + +USER ubuntu +WORKDIR /workspace diff --git a/.cursor/elasticsearch.yml b/.cursor/elasticsearch.yml new file mode 100644 index 00000000..92181da6 --- /dev/null +++ b/.cursor/elasticsearch.yml @@ -0,0 +1,10 @@ +cluster.name: ror-local +node.name: ror-node +path.data: /var/lib/elasticsearch +path.logs: /var/log/elasticsearch +network.host: 127.0.0.1 +http.port: 9200 +discovery.type: single-node +xpack.security.enabled: false +http.cors.enabled: true +http.cors.allow-origin: "*" diff --git a/.cursor/env.example b/.cursor/env.example new file mode 100644 index 00000000..9e55e3dc --- /dev/null +++ b/.cursor/env.example @@ -0,0 +1,9 @@ +ELASTIC7_HOST=localhost +ELASTIC7_PORT=9200 +ELASTIC_PASSWORD=changeme +ROR_BASE_URL=http://localhost +DB_HOST=127.0.0.1 +DB_PORT=3306 +DB_NAME=rorapi +DB_USER=ror_user +DB_PASSWORD=password diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 00000000..45ff2cba --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,17 @@ +{ + "name": "ROR API", + "user": "ubuntu", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "install": "bash .cursor/install.sh", + "start": "bash .cursor/start.sh", + "terminals": [ + { + "name": "ror-api server", + "command": "cd /workspace && source .venv/bin/activate && python manage.py runserver 0.0.0.0:9292" + } + ], + "ports": [9292, 9200, 3306] +} diff --git a/.cursor/install.sh b/.cursor/install.sh new file mode 100755 index 00000000..9a9d1c29 --- /dev/null +++ b/.cursor/install.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Repository bootstrap for the ROR API Cloud Agent environment. +# Idempotent: safe to run repeatedly. Prepares the Python virtualenv, +# installs dependencies and collects static files. Runtime services +# (MySQL, Elasticsearch) are handled by start.sh. +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Local dev configuration (settings.py loads this via python-dotenv). +if [ ! -f .env ]; then + cp .cursor/env.example .env +fi + +# Python virtualenv +if [ ! -x .venv/bin/python ]; then + python3 -m venv .venv +fi +./.venv/bin/pip install --upgrade pip +./.venv/bin/pip install -r requirements.txt + +# Static assets can be built without a database connection. +DJANGO_SKIP_DB_CHECK=True ./.venv/bin/python manage.py collectstatic --noinput + +echo "install.sh: dependencies installed" diff --git a/.cursor/mysql-init.sql b/.cursor/mysql-init.sql new file mode 100644 index 00000000..026ae355 --- /dev/null +++ b/.cursor/mysql-init.sql @@ -0,0 +1,8 @@ +-- Idempotent MySQL bootstrap for local ROR API development. +-- Mirrors the credentials used by the CI test workflow. +CREATE DATABASE IF NOT EXISTS rorapi CHARACTER SET utf8mb4; +CREATE USER IF NOT EXISTS 'ror_user'@'localhost' IDENTIFIED BY 'password'; +CREATE USER IF NOT EXISTS 'ror_user'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON *.* TO 'ror_user'@'localhost' WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON *.* TO 'ror_user'@'%' WITH GRANT OPTION; +FLUSH PRIVILEGES; diff --git a/.cursor/start.sh b/.cursor/start.sh new file mode 100755 index 00000000..b2d57eb5 --- /dev/null +++ b/.cursor/start.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Per-boot service startup for the ROR API Cloud Agent environment. +# Starts MySQL and Elasticsearch, applies migrations and ensures the +# v2 Elasticsearch index exists. Idempotent and safe to re-run. +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Ensure local dev config exists even if the checkout dropped it. +if [ ! -f .env ]; then + cp .cursor/env.example .env +fi + +echo "start.sh: starting MySQL..." +# Initialize the data directory if the image did not already do so. +# (/var/lib/mysql is mode 700 mysql:mysql, so probe it with sudo. MySQL 8 +# stores the system schema in mysql.ibd rather than a mysql/ directory.) +if ! sudo test -f /var/lib/mysql/mysql.ibd; then + sudo mkdir -p /var/lib/mysql + sudo chown -R mysql:mysql /var/lib/mysql + sudo mysqld --initialize-insecure --user=mysql +fi +sudo mkdir -p /var/run/mysqld +sudo chown mysql:mysql /var/run/mysqld +sudo service mysql start || true +for i in $(seq 1 60); do + if sudo mysqladmin ping >/dev/null 2>&1; then break; fi + sleep 1 +done +sudo mysql < .cursor/mysql-init.sql +echo "start.sh: MySQL ready" + +echo "start.sh: starting Elasticsearch..." +if ! curl -s -m 3 http://127.0.0.1:9200 >/dev/null 2>&1; then + # Detach all standard fds so the daemon does not hold this command's + # stdout/stderr pipe open (otherwise `start` never returns and boot stalls). + sudo -u elasticsearch ES_PATH_CONF=/opt/elasticsearch/config \ + /opt/elasticsearch/bin/elasticsearch -d -p /tmp/es.pid \ + /tmp/elasticsearch-console.log 2>&1 +fi +for i in $(seq 1 90); do + if curl -s -m 3 http://127.0.0.1:9200 >/dev/null 2>&1; then break; fi + sleep 2 +done +echo "start.sh: Elasticsearch ready" + +# Database migrations (idempotent) and v2 index creation. +./.venv/bin/python manage.py migrate --noinput +if [ "$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9200/organizations-v2)" != "200" ]; then + ./.venv/bin/python manage.py createindex || true +fi + +echo "start.sh: ROR API services ready" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 68605a7f..f3bc27cb 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -10,3 +10,20 @@ jobs: test-pr: uses: ./.github/workflows/run_tests.yml secrets: inherit + ruff: + runs-on: ubuntu-latest + # Soft-fail until Wave 5 (quickwin/ruff-fix) cleans the tree and removes this. + continue-on-error: true + steps: + - name: Checkout ror-api code + uses: actions/checkout@v2 + - name: Set up Python environment + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Install ruff + run: pip install ruff + - name: Ruff check + run: ruff check . + - name: Ruff format check + run: ruff format --check . diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 28fe53c1..142a0833 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -6,6 +6,7 @@ jobs: test: runs-on: ubuntu-latest env: + SECRET_KEY: "django-insecure-github-actions-test-key" ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" diff --git a/.gitignore b/.gitignore index d4525307..7fbdad9e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ esdata/** .venv310/ .uv-python/ .uv-bin/ + +# Cloud Agent env +.venv/ +/static/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..34558c4f --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,483 @@ +# ROR API Architecture + +This document describes the architecture of the [Research Organization Registry (ROR) API](https://ror.org) codebase. It is intended as context for coding agents and contributors working on this repository. + +Public API documentation lives at https://ror.readme.io. Operational data workflows are managed in sibling repos: [ror-data](https://github.com/ror-community/ror-data) (full dumps), [ror-records](https://github.com/ror-community/ror-records) (incremental updates), and [ror-schema](https://github.com/ror-community/ror-schema) (JSON schema). + +--- + +## Purpose + +The ROR API provides: + +1. **Public read API** — search, filter, retrieve, and affiliation-match research organizations stored in Elasticsearch. +2. **Internal write helpers** — generate schema-valid ROR record JSON (single record, bulk CSV) for the separate data-release pipeline. These endpoints do **not** write directly to the search index. +3. **Indexing operations** — management commands and authenticated HTTP endpoints to load organization data from GitHub releases or AWS S3 into Elasticsearch. + +--- + +## High-Level Architecture + +``` + ┌─────────────────────────────────────────┐ + │ Clients / Integrators │ + │ (ror-app UI, Crossref, publishers…) │ + └────────────────────┬────────────────────┘ + │ HTTPS + ▼ + ┌─────────────────────────────────────────┐ + │ Nginx + Phusion Passenger (port 80) │ + │ WSGI entry: rorapi/wsgi.py │ + └────────────────────┬────────────────────┘ + │ + ┌────────────────────▼────────────────────┐ + │ Django 5.2 + DRF (rorapi/) │ + │ views → queries/matching/create_update │ + └───────┬──────────────────────┬───────────┘ + │ │ + ┌─────────────▼──────────┐ ┌───────▼────────────┐ + │ Elasticsearch 7 │ │ MySQL 8 │ + │ index: organizations-v2│ │ Client model only │ + └────────────────────────┘ └────────────────────┘ + ▲ + │ bulk index / reindex + ┌─────────────┴──────────┐ + │ Management commands │ + │ setup, indexror, … │ + └─────────────┬──────────┘ + │ + ┌──────────────────┼──────────────────┐ + ▼ ▼ ▼ + GitHub (ror-data) AWS S3 (DATA_STORE) Geonames (update_address) +``` + +**Key design choice:** Organization records live in **Elasticsearch**, not in Django ORM models. Django models under `rorapi/v2/models.py` are plain Python classes that wrap ES hit objects for serialization — except `Client`, which is a real Django model backed by MySQL. + +--- + +## Technology Stack + +| Layer | Technology | +|-------|------------| +| Language | Python 3.12 | +| Web framework | Django 5.2.17 (LTS) | +| REST API | Django REST Framework 3.18.1 (URL path versioning, v2 only) | +| Search | Elasticsearch 7.10.1 via `elasticsearch` + `elasticsearch_dsl` 7.4.1 | +| Relational DB | MySQL 8 (client registration only) | +| App server | Phusion Passenger + Nginx (`vendor/docker/webapp.conf`) | +| Container | Docker (`Dockerfile` based on `phusion/passenger-python312:3.2.0`) | +| Observability | Sentry (`sentry-sdk` 1.45.1), django-prometheus 2.4.1 | +| Feature flags | LaunchDarkly (`rorapi/common/features.py`; `launchdarkly-server-sdk` 7.6.1) | +| Email | django-ses 4.8.0 (client ID registration emails) | +| External packages | `update_address` (Geonames enrichment), `jsonschema` 3.2.0, `rapidfuzz` 3.6.1, `boto3` (unpinned), `pandas` 2.2.3 | + +Pins above match `[requirements.txt](requirements.txt)` on `dev` at the time of writing. + +--- + +## Repository Layout + +``` +ror-api/ +├── manage.py # Django entry point +├── requirements.txt +├── Dockerfile +├── docker-compose.yml # Local: web + elasticsearch7 + mysql +├── rorapi/ +│ ├── settings.py # Central config, ES client, AWS, env vars +│ ├── wsgi.py +│ ├── common/ # Shared logic (most application code) +│ │ ├── urls.py # All HTTP routes +│ │ ├── views.py # ViewSets and APIViews +│ │ ├── queries.py # Search/retrieve param validation + ES queries +│ │ ├── es_utils.py # ESQueryBuilder +│ │ ├── matching.py # Multi-search affiliation matching +│ │ ├── matching_single_search.py # Single-search affiliation matching +│ │ ├── create_update.py # Record JSON generation (POST/PUT) +│ │ ├── csv_bulk.py # Bulk CSV processing +│ │ ├── csv_create.py # CSV → new record +│ │ ├── csv_update.py # CSV → updated record +│ │ ├── record_utils.py # Schema validation, language codes +│ │ ├── serializers.py # Shared serializers (Errors, buckets) +│ │ └── models.py # Errors, Entity, aggregation buckets +│ ├── v2/ # v2 schema-specific code +│ │ ├── models.py # Organization, ListResult, MatchingResult, Client (ORM) +│ │ ├── serializers.py # Response + Client registration serializers +│ │ ├── record_constants.py +│ │ ├── record_template.json +│ │ ├── ror_schema_v2_1.json # Vendored JSON schema for write validation +│ │ └── index_template_es7.json # ES index template + mappings +│ ├── management/commands/ # CLI indexing and legacy GRID tools +│ ├── migrations/ # Django migrations (Client model) +│ └── tests/ # Unit, integration, functional, affiliation suites +└── vendor/docker/ # Nginx, env, Terraform var templates for deploy +``` + +There is **no v1 API code path** in active use. `REST_FRAMEWORK['ALLOWED_VERSIONS']` is `['v2']` only. + +--- + +## HTTP API Surface + +Routes are defined in `rorapi/common/urls.py`. Version prefix `v2/` is optional for some legacy paths. + +### Public read endpoints (GET, no auth) + +| Route | Handler | Purpose | +|-------|---------|---------| +| `GET /v2/organizations` | `OrganizationViewSet.list` | Search, filter, or affiliation match | +| `GET /v2/organizations/{id}` | `OrganizationViewSet.retrieve` | Fetch one org by ROR ID (URL or bare ID) | +| `GET /heartbeat`, `GET /v2/heartbeat` | `HeartbeatView` | Health check (ES index exists) | +| `GET /validate-client-id/{client_id}/` | `ValidateClientView` | Check if Client-Id is registered | + +### Authenticated write / admin endpoints (POST/PUT; `Token` + `Route-User` headers) + +| Route | Handler | Purpose | +|-------|---------|---------| +| `POST /v2/organizations` | `OrganizationViewSet.create` | Generate new record JSON (does **not** index) | +| `PUT /v2/organizations/{id}` | `OrganizationViewSet.update` | Generate updated record JSON (does **not** index) | +| `POST /v2/bulkupdate` | `BulkUpdate` | CSV bulk create/update → zip on S3 | +| `POST /v2/indexdata/{branch}` | `IndexData` | Incremental index from S3 directory | +| `POST /v2/indexdatadump/{filename}/{test\|prod}` | `IndexDataDump` | Full reindex from GitHub data dump | +| `GET /generateid` | `GenerateId` | Generate unused ROR ID | +| `GET /v2/generateaddress/{geonamesid}` | `GenerateAddress` | Geonames location enrichment | + +### Client registration (POST, no Token auth) + +| Route | Handler | Purpose | +|-------|---------|---------| +| `POST /v2/register` | `ClientRegistrationView` | Register for rate-limit Client-Id; sends email via SES | + +### Auth model + +- **`OurTokenPermission`** (`views.py`): GET is always allowed. Mutating methods require `Token` and `Route-User` headers matching env vars `TOKEN` and `ROUTE_USER` (compared with `hmac.compare_digest` when both sides are present). Access is denied if either env var or either header is missing/empty. +- **`Client-Id` header**: Not authentication. Registered clients get higher rate limits (enforced outside this repo, typically at the edge). CORS allows the `Client-Id` header. + +--- + +## Request Flow: Search & Retrieve + +``` +GET /v2/organizations?query=…&filter=…&page=… + │ + ▼ +OrganizationViewSet.list + │ + ├─ affiliation param? ──► matching.py OR matching_single_search.py + │ (see Affiliation Matching below) + │ + └─ else ──► queries.search_organizations(params) + │ + ├─ validate(params) # allowed keys, filters, page range + ├─ build_search_query(params) # ESQueryBuilder + ├─ execute ES search on organizations-v2 + └─ wrap hits → ListResultV2 → ListResultSerializerV2 +``` + +``` +GET /v2/organizations/{pk} + │ + ▼ +get_ror_id(pk) → retrieve_organization(ror_id) → OrganizationSerializerV2 +``` + +**Search parameters** (validated in `queries.py`): + +- `query` — free-text search against nested `names_ids` field +- `query.advanced` — Elasticsearch query_string syntax with field allowlist +- `filter` — comma-separated `field:value` filters (types, status, country, geonames fields, continent) +- `page` — pagination (page size = `ES_VARS['BATCH_SIZE']` = 20, max page = 500) +- `all_status` — include inactive records when not filtering by status + +**Default behavior:** Active records only (`status:active` filter applied unless `all_status` or explicit status filter). + +--- + +## Request Flow: Affiliation Matching + +Triggered by `?affiliation=…` on `GET /v2/organizations`. + +Two strategies exist: + +| Strategy | Module | Selection | +|----------|--------|-----------| +| Multi-search (legacy default) | `common/matching.py` | Default unless `SINGLE_SEARCH_DEFAULT=True` or `single_search` param | +| Single-search (Marple-derived) | `common/matching_single_search.py` | `SINGLE_SEARCH_DEFAULT=True`, or `?single_search`, or when multisearch not requested | + +Both: + +1. Parse country hints from affiliation string using `countries.txt` + geonamescache +2. Query ES nested `affiliation_match.names` (indexed at ingest time) +3. Score candidates with fuzzy matching (`rapidfuzz`) +4. Return `MatchingResult` with chosen match, score, matching_type, substring + +Response uses `MatchingResultSerializerV2`. + +--- + +## Request Flow: Record Generation (Not Indexing) + +**Important for agents:** `POST`/`PUT` on `/organizations` and `POST` on `/bulkupdate` produce **files for the ror-records release workflow**. They validate against the vendored schema at `rorapi/v2/ror_schema_v2_1.json` (loaded on first write via `get_v2_schema()` in `create_update.py`) and enrich Geonames data, but do **not** call Elasticsearch bulk index. + +``` +POST /v2/organizations (JSON body) + │ + ▼ +new_record_from_json() in create_update.py + ├─ add optional field defaults + ├─ update_address.new_geonames_v2() per location + ├─ generaterorid.check_ror_id() — collision-safe ID + ├─ jsonschema validation (vendored v2.1 schema) + └─ sort fields → OrganizationSerializerV2 response + +PUT /v2/organizations/{id} + │ + ▼ +retrieve existing from ES → update_record_from_json() +``` + +Bulk CSV path: `BulkUpdate` → `validate_csv()` → `process_csv()` → per-row `new_record_from_csv` / `update_record_from_csv` → zip uploaded to S3 (`DATA_STORE` / `PUBLIC_STORE`). + +--- + +## Elasticsearch Data Model + +- **Index name:** `organizations-v2` (`ES_VARS['INDEX_V2']`) +- **Template:** `rorapi/v2/index_template_es7.json` +- **Document ID:** Organization `id` (full URL, e.g. `https://ror.org/01an7q238`) + +Each indexed document is the ROR v2 JSON record plus **derived fields** added at index time (in `indexror.py` and `indexrordump.py`): + +| Derived field | Purpose | +|---------------|---------| +| `names_ids` (nested) | Searchable names + external IDs for general query | +| `affiliation_match` (nested) | Optimized structure for affiliation matching | + +Index creation: `python manage.py createindex` +Deletion: `python manage.py deleteindex` + +Local/docker ES uses basic auth `(elastic, ELASTIC_PASSWORD)`. Remote AWS OpenSearch uses `AWS4Auth`. + +--- + +## Indexing Pipelines + +### Full reindex from GitHub data dump + +Used for local dev, CI test setup, and disaster recovery. + +``` +python manage.py setup {dump-filename} -s 2 [-t] +``` + +Orchestration (`setup.py`): + +1. Verify dump exists in [ror-data](https://github.com/ror-community/ror-data) or [ror-data-test](https://github.com/ror-community/ror-data-test) via GitHub API (`GITHUB_TOKEN` required) +2. `getrordump` — download zip +3. `deleteindex` — drop existing index +4. `createindex` — apply template, create index +5. `indexrordump` — bulk load all records with derived fields + +HTTP equivalent: `POST /v2/indexdatadump/{filename}/{test|prod}` + +### Incremental index from S3 + +Used in production data deployment (triggered from [ror-records](https://github.com/ror-community/ror-records) GitHub Actions). + +``` +python manage.py indexror {s3-directory} +``` + +Or: `POST /v2/indexdata/{branch}` + +Flow (`indexror.py`): + +1. Download `{dir}/files.zip` from S3 bucket `DATA_STORE` +2. Extract JSON record files +3. **Backup current index** via reindex to `organizations-v2-tmp` +4. Bulk upsert changed records (with derived fields) +5. On `TransportError`, reindex backup back to restore; delete tmp index + +Requires AWS credentials and `DATA_STORE` env var. + +--- + +## Django / MySQL Usage + +MySQL stores **only** the `Client` model (`rorapi/v2/models.py`): + +- Registration fields: email, name, institution, country, ror_use +- System: `client_id` (32-char random), timestamps, request counters + +Run migrations after DB is up: + +```bash +python manage.py migrate +``` + +All other "models" (`Organization`, `ListResult`, etc.) are in-memory wrappers around Elasticsearch responses, not Django ORM entities. + +--- + +## Configuration & Environment Variables + +Loaded from environment and optional root `.env` file (`python-dotenv`). + +| Variable | Purpose | +|----------|---------| +| `ELASTIC7_HOST`, `ELASTIC7_PORT`, `ELASTIC_PASSWORD` | Elasticsearch connection | +| `DB_HOST`, `DB_NAME`, `DB_USER`, `DB_PASSWORD`, `DB_PORT` | MySQL for Client model | +| `DATA_STORE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | S3 indexing and bulk upload | +| `PUBLIC_STORE` | Public URL base for bulk update zip downloads | +| `GITHUB_TOKEN` | Fetch data dumps from GitHub | +| `ROUTE_USER`, `TOKEN` | Admin API authentication | +| `ROR_BASE_URL` | Base URL configuration | +| `SENTRY_DSN` | Error reporting | +| `LAUNCH_DARKLY_KEY` | Feature flags | +| `SINGLE_SEARCH_DEFAULT` | Default affiliation matcher (`True`/`False`) | +| `ENABLE_BEHAVIORAL_LIMITING` | Rate limiting toggle (edge behavior) | +| `SECRET_KEY` | Django secret (falls back to a hardcoded default if unset; `DEBUG` is always `False`) | +| `ALLOWED_HOSTS` | Currently hardcoded to `['*']` in settings | +| `AWS_REGION` | SES + AWS auth region | + +See `README.md` for local Docker `.env` template. + +--- + +## Local Development + +```bash +docker-compose up -d +docker-compose exec web python manage.py migrate +docker-compose exec web python manage.py setup v1.0-2022-03-17-ror-data -s 2 +# API at http://localhost:9292/v2/organizations +``` + +- Port **9292** maps to container port 80 +- `./rorapi` is volume-mounted for live code edits +- Services: `web`, `elasticsearch7`, `db` + +--- + +## Testing + +| Suite | Path | CI status | +|-------|------|-----------| +| Unit | `rorapi.tests.tests_unit` | Runs in GitHub Actions | +| Integration | `rorapi.tests.tests_integration` | Commented out in CI (TODO) | +| Functional | `rorapi.tests.tests_functional` | Commented out in CI | +| Affiliation eval | `rorapi.tests.tests_affiliations` | Manual / local | + +CI workflow (`.github/workflows/run_tests.yml`): + +1. Spin up ES 7.10 + MySQL service containers +2. `pip install -r requirements.txt` +3. `python manage.py setup $TEST_DATA_DUMP_FILE -t` (test data from ror-data-test) +4. `python manage.py test rorapi.tests.tests_unit` + +Run locally (with services up and data indexed): + +```bash +docker-compose exec web python manage.py test rorapi.tests.tests_unit +docker-compose exec web python manage.py test rorapi.tests.tests_integration +docker-compose exec web python manage.py test rorapi.tests.tests_functional +``` + +--- + +## Deployment & CI/CD + +| Branch / event | Workflow | Docker tag | Deploy target | +|----------------|----------|------------|---------------| +| Push to `dev` | `dev.yml` | `rorcommunity/ror-api:dev` | Terraform in [new-deployment](https://github.com/ror-community/new-deployment) | +| Push to `staging` | `staging.yml` | `:staging` | Same pattern | +| GitHub Release | `release.yml` | release tag | Production | +| Pull request | `pull-request.yml` | — | Tests only | + +Deploy mechanism: GitHub Action updates `_ror-api-*.auto.tfvars` in the `new-deployment` repo (from templates in `vendor/docker/`), triggering Terraform infrastructure rollout. Images are pushed to Docker Hub (`rorcommunity/ror-api`). + +--- + +## Legacy Code + +Commands prefixed with `legacy*` (GRID conversion, old upgrade paths) are **non-functional** — referenced data was moved to ror-data. GRID-based generation ended March 2022. Do not extend or rely on these unless explicitly reviving historical tooling. + +`settings.py` still contains commented GRID/ROR_DUMP version history for reference. `GRID_REMOVED_IDS` is an empty list retained for a check in `retrieve_organization`. + +--- + +## External Dependencies & Related Repos + +| Repo / service | Role | +|----------------|------| +| [ror-community/ror-data](https://github.com/ror-community/ror-data) | Production full data dumps (zip) | +| [ror-community/ror-data-test](https://github.com/ror-community/ror-data-test) | Test dumps for CI/local | +| [ror-community/ror-records](https://github.com/ror-community/ror-records) | Record PR workflow → S3 → `indexror` | +| [ror-community/ror-schema](https://github.com/ror-community/ror-schema) | Upstream JSON schema (vendored copy lives in-repo) | +| [ror-community/update_address](https://github.com/ror-community/update_address) | Geonames location enrichment | +| [ror-community/ror-app](https://github.com/ror-community/ror-app) | Public search UI (separate repo) | +| [ror-community/new-deployment](https://github.com/ror-community/new-deployment) | Terraform / infra | + +--- + +## Agent Guidance: Common Tasks + +### Add or change a search/filter parameter + +1. Update allowlists in `rorapi/common/queries.py` (`ALLOWED_*` constants) +2. Extend `build_search_query()` and/or `validate()` +3. If ES mapping changes are needed, edit `rorapi/v2/index_template_es7.json` and plan a reindex +4. Add tests under `rorapi/tests/tests_unit/` or `tests_integration/` + +### Change affiliation matching behavior + +- Multi-search logic: `rorapi/common/matching.py` +- Single-search logic: `rorapi/common/matching_single_search.py` +- ES query shape: `ESQueryBuilder.add_affiliation_query()` in `es_utils.py` +- Indexed field shape: `get_affiliation_match_doc()` in `indexror.py` / `indexrordump.py` +- Default strategy: `SINGLE_SEARCH_DEFAULT` env var or request params in `OrganizationViewSet.list` + +### Change record JSON generation / validation + +- Core logic: `create_update.py`, `record_utils.py`, `v2/record_constants.py` +- Schema is the vendored `rorapi/v2/ror_schema_v2_1.json`, loaded lazily by `get_v2_schema()` on first write +- CSV paths: `csv_create.py`, `csv_update.py`, `csv_utils.py`, `csv_bulk.py` + +### Add a new authenticated admin endpoint + +1. Add view in `rorapi/common/views.py` with `OurTokenPermission` +2. Register route in `rorapi/common/urls.py` +3. Add unit tests in `tests_views_v2.py` + +### Reindex after mapping changes + +1. Update `index_template_es7.json` +2. Run full `setup` command (deletes and recreates index) — not incremental `indexror` +3. Verify with `GET /v2/heartbeat` and search tests + +### What NOT to assume + +- POST/PUT organizations **do not** update Elasticsearch; production updates go through ror-records → S3 → `indexror` +- v1 schema/index support has been removed +- Django ORM is not used for organization records +- Rate limiting for public API is largely enforced outside this application (Client-Id registration is stored here; enforcement is at infrastructure layer) + +--- + +## Key Files Quick Reference + +| Concern | Primary file(s) | +|---------|-----------------| +| Routes | `rorapi/common/urls.py` | +| HTTP handlers | `rorapi/common/views.py` | +| Search/retrieve | `rorapi/common/queries.py` | +| ES query building | `rorapi/common/es_utils.py` | +| Affiliation match | `matching.py`, `matching_single_search.py` | +| Record create/update | `create_update.py`, `csv_*.py` | +| Vendored JSON schema | `rorapi/v2/ror_schema_v2_1.json` | +| ES index template | `rorapi/v2/index_template_es7.json` | +| Settings / clients | `rorapi/settings.py` | +| Full reindex CLI | `management/commands/setup.py` | +| Incremental index CLI | `management/commands/indexror.py` | +| ROR ID generation | `management/commands/generaterorid.py` | +| Response serializers | `rorapi/v2/serializers.py` | diff --git a/docker-compose.yml b/docker-compose.yml index e123173f..84e5b6cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,8 @@ services: web: container_name: rorapiweb env_file: .env + environment: + SECRET_KEY: "django-insecure-docker-compose-dummy-key" build: . #image: rorcommunity/ror-api ports: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c6757284 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.ruff] +target-version = "py312" +line-length = 88 +src = ["rorapi"] +exclude = [ + ".git", + ".venv", + ".venv310", + ".uv-python", + "venv", + "__pycache__", + "vendor", + "esdata", + "rorapi/data", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "F", # Pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear +] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +line-ending = "auto" diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..15285a66 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +statsmodels==0.14.4 diff --git a/requirements.txt b/requirements.txt index 73c07405..51664707 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ base32_crockford==0.3.0 bleach==6.4.0 -boto3 +boto3==1.43.100 Django==5.2.17 django-cors-headers==4.9.0 django-prometheus==2.4.1 @@ -11,11 +11,7 @@ elasticsearch_dsl==7.4.1 geonamescache==1.3.0 iso639-lang jsonschema==3.2.0 -launchdarkly-server-sdk==7.6.1 -mock==3.0.5 mysqlclient==2.2.7 -numpy==1.26.4 -pandas==2.2.3 pycountry==22.3.5 python-dotenv==1.2.3 python-magic @@ -24,7 +20,6 @@ requests==2.33.0 requests-aws4auth==0.9 sentry-sdk==1.45.1 setuptools>=40,<81 -statsmodels==0.14.4 titlecase==2.3 unidecode==1.1.1 update_address @ git+https://github.com/ror-community/update_address.git diff --git a/rorapi/common/matching.py b/rorapi/common/matching.py index 1e772ea1..2ef74c9a 100644 --- a/rorapi/common/matching.py +++ b/rorapi/common/matching.py @@ -1,11 +1,15 @@ import geonamescache import os import re -import unicodedata -import unidecode from rorapi.common.models import Errors from rorapi.common.es_utils import ESQueryBuilder +from rorapi.common.matching_common import ( + check_latin_chars, + get_country_codes, + normalize, + to_region, +) from rorapi.v2.models import MatchingResult as MatchingResultV2 from collections import namedtuple @@ -74,44 +78,6 @@ def load_countries(): COUNTRIES = load_countries() -def to_region(c): - """Map country code to "region" string. - - This effectively groups countries often confused in the data to make sure - the scoring functions do not reject potential matching candidates.""" - - return { - "GB": "GB-UK", - "UK": "GB-UK", - "CN": "CN-HK-TW", - "HK": "CN-HK-TW", - "TW": "CN-HK-TW", - "PR": "US-PR", - "US": "US-PR", - }.get(c, c) - - -def get_country_codes(string): - """Extract the country codes from the string, - if the country names are mentioned.""" - - string = unidecode.unidecode(string).strip() - lower = re.sub(r"\s+", " ", string.lower()) - lower_alpha = re.sub(r"\s+", " ", re.sub("[^a-z]", " ", string.lower())) - alpha = re.sub(r"\s+", " ", re.sub("[^a-zA-Z]", " ", string)) - codes = [] - for code, name in COUNTRIES: - if re.search("[^a-z]", name): - score = fuzz.partial_ratio(name, lower) - elif len(name) == 2: - score = max([fuzz.ratio(name.upper(), t) for t in alpha.split()]) - else: - score = max([fuzz.ratio(name, t) for t in lower_alpha.split()]) - if score >= 90: - codes.append(code.upper()) - return list(set(codes)) - - def get_countries(string): """Extract country codes the the string and map to regions.""" @@ -124,48 +90,6 @@ def get_countries(string): ##################################################################### -def check_latin_chars(s): - for ch in s: - if ch.isalpha(): - if "LATIN" not in unicodedata.name(ch): - return False - return True - - -def normalize(s): - """Normalize string for matching.""" - - if check_latin_chars(s): - s = re.sub(r"\s+", " ", unidecode.unidecode(s).strip().lower()) - else: - s = re.sub(r"\s+", " ", s.strip().lower()) - s = re.sub( - "(?= 90: + codes.append(code.upper()) + return list(set(codes)) + + +def check_latin_chars(s): + for ch in s: + if ch.isalpha(): + if "LATIN" not in unicodedata.name(ch): + return False + return True + + +def normalize(s): + """Normalize string for matching.""" + + if check_latin_chars(s): + s = re.sub(r"\s+", " ", unidecode.unidecode(s).strip().lower()) + else: + s = re.sub(r"\s+", " ", s.strip().lower()) + s = re.sub( + "(?= 90: - codes.append(code.upper()) - return list(set(codes)) - def get_countries(string): """Extract country codes the the string and map to regions.""" @@ -77,52 +31,6 @@ def get_countries(string): return [to_region(c) for c in codes] -def check_latin_chars(s): - for ch in s: - if ch.isalpha(): - if "LATIN" not in unicodedata.name(ch): - return False - return True - - -def normalize(s): - """Normalize string for matching.""" - - if check_latin_chars(s): - s = re.sub(r"\s+", " ", unidecode.unidecode(s).strip().lower()) - else: - s = re.sub(r"\s+", " ", s.strip().lower()) - s = re.sub( - "(?