feat(drugs_com): add drugs.com mirror site (port 40016)

.assets-revision

@@ -4,5 +4,12 @@
 # fetch_assets.sh uses the `hf download` CLI. The pin below
 # is a git revision (branch name like `main`, a tag, or a specific commit
 # sha). Override at runtime with the ASSETS_REVISION env var.
+#
+# TEMPORARY: pinned to the resolved commit from refs/pr/39 (the drugs_com.tar.gz PR discussion at
+# https://huggingface.co/datasets/ChilleD/WebHarbor/discussions/39) because
+# drugs_com's assets aren't on `main` yet. refs/pr/39 is a superset of
+# `main` (all 16 existing tarballs, unchanged, plus drugs_com.tar.gz) so this
+# is safe for every other site. Once PR 39 merges, re-pin to `main` (or the
+# resulting merge commit SHA) and drop this note.
 repo: ChilleD/WebHarbor
-revision: 54882a6a66a17a3e43455057e7c9e0d103cd8b81
+revision: 1e4ccf0f8110c783f826b38f47c0066a5def822b

.dockerignore

@@ -8,10 +8,12 @@ sites/*/instance/
 # Don't ship — runtime doesn't need scrape intermediate (data is in instance_seed/*.db).
 sites/*/scraped_data/
 
-# Don't ship — bytecode / venvs.
+# Don't ship — bytecode / venvs. `uv` creates `.venv`, so cover both names.
 sites/*/__pycache__/
 **/__pycache__/
 sites/*/venv/
+sites/*/.venv/
+**/.venv/
 sites/*/*.pyc
 
 # Don't ship — HF asset tarballs are build-time inputs (fetched/extracted by

.gitignore

@@ -94,4 +94,9 @@ secrets.json
 # ============================================================
 # Agent demo results
 # =============================================================
-agent_demo/runs/
+agent_demo/runs/.claude/ralph-loop.local.md
+.claude/ralph-loop.local.md
+sites/*/.claude/
+sites/*/download_pill_images.py
+sites/drugs_com/instance/
+sites/drugs_com/scraped_data/

AGENTS.md

@@ -4,7 +4,7 @@ A coding agent (Claude Code, Cursor, Aider, Codex, ...) is reading this. Read on
 
 ## What it is
 
-15 Flask mirror websites (Amazon, GitHub, BBC News, ...) packaged into one Docker image, plus a control plane on `:8101` for resetting per-site state. Used as a deterministic offline environment for web-agent benchmarks. ~3 GB image.
+17 Flask mirror websites (Amazon, GitHub, BBC News, ...) packaged into one Docker image, plus a control plane on `:8101` for resetting per-site state. Used as a deterministic offline environment for web-agent benchmarks. ~6 GB image.
 
 Two repos:
 - **code** (this one) — Flask apps, control plane, scripts.
@@ -48,17 +48,17 @@ Inside the image, sites live at `/opt/WebSyn/<site>/`. The path predates the ren
 # fresh clone
 ./scripts/fetch_assets.sh                     # pulls assets from HF
 ./scripts/build.sh                            # docker build -t webharbor:dev .
-docker run -d -p 8101:8101 -p 40000-40014:40000-40014 webharbor:dev
+docker run -d -p 8101:8101 -p 40000-40016:40000-40016 webharbor:dev
 ```
 
 Or use the published image directly:
 
 ```bash
-docker run -d -p 8101:8101 -p 40000-40014:40000-40014 \
+docker run -d -p 8101:8101 -p 40000-40016:40000-40016 \
   battalion7244/webharbor:latest
 ```
 
-Sites are on `40000`-`40014` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane:
+Sites are on `40000`-`40016` in the order declared by `SITES=( ... )` in `websyn_start.sh`. Control plane:
 
 | Method | Path                | Purpose                                   |
 |--------|---------------------|-------------------------------------------|
@@ -136,13 +136,13 @@ python3 -m py_compile sites/<site>/app.py
 
 # 3. run on alt ports (don't collide with anything you already have running)
 docker run -d --rm --name wh-test \
-  -p 8201:8101 -p 41000-41014:40000-40014 webharbor:dev
+  -p 8201:8101 -p 41000-41016:40000-40016 webharbor:dev
 
 # 4. control plane healthy, all sites alive
 curl -s http://localhost:8201/health | python3 -m json.tool | head
 
 # 5. every site renders 200
-for p in $(seq 41000 41014); do
+for p in $(seq 41000 41016); do
   curl -so /dev/null -w "$p:%{http_code}\n" http://localhost:$p/
 done
 

CLAUDE.md

@@ -16,4 +16,4 @@ The full agent guide is loaded above via `@AGENTS.md`. The notes below apply onl
 
 ## Existing containers
 
-If a container is already running on `:8101` / `:40000-40014`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41014`).
+If a container is already running on `:8101` / `:40000-40016`, treat it as the user's working environment — don't `docker stop` or `docker rm` it without explicit confirmation. Spin up your test container under a different name on alt ports (`:8201`, `:41000-41016`).

CONTRIBUTING.md

@@ -24,7 +24,7 @@ git clone https://github.com/<you>/webharbor && cd webharbor
 ./scripts/fetch_assets.sh                       # pull current assets
 ./scripts/new_site.py mywebsite                 # OR edit an existing site
 ./scripts/build.sh && docker run -d --rm \
-  -p 8101:8101 -p 40000-40014:40000-40014 webharbor:dev
+  -p 8101:8101 -p 40000-40016:40000-40016 webharbor:dev
 # iterate locally...
 
 ./scripts/extract_assets.sh ../webharbor-static-pr/   # split assets out
@@ -277,4 +277,4 @@ Sites must not import from one another. The image launches each as an independen
 
 ### Don't hard-code secrets
 
-Each site sets `SECRET_KEY` to a deterministic dev value. Acceptable for a benchmark image (resets blow away sessions anyway). If a contrib ever needs real secrets, raise it in an issue first.
+Each site sets `SECRET_KEY` to a deterministic dev value. Acceptable for a benchmark image (resets blow away sessions anyway). If a contrib ever needs real secrets, raise it in an issue first.

Dockerfile

@@ -1,5 +1,5 @@
 # WebHarbor — slim, self-contained image.
-# 16 Flask mirror sites + control plane on :8101.
+# 17 Flask mirror sites + control plane on :8101.
 
 FROM python:3.12-slim-bookworm
 
@@ -19,7 +19,8 @@ RUN pip3 install --no-cache-dir \
     SQLAlchemy==2.0.36 \
     WTForms==3.2.1 \
     email-validator==2.2.0 \
-    Pillow==11.0.0
+    Pillow==11.0.0 \
+    requests==2.32.3
 
 WORKDIR /opt/WebSyn
 
@@ -33,6 +34,6 @@ COPY control_server.py  /opt/control_server.py
 COPY site_runner.py     /opt/site_runner.py
 RUN chmod +x /opt/websyn_start.sh
 
-EXPOSE 8101 40000-40015
+EXPOSE 8101 40000-40016
 
 CMD ["/opt/websyn_start.sh"]

README.md

@@ -36,17 +36,17 @@ WebHarbor takes a different approach. We leverage coding agent (e.g., Claude Cod
 - **Deep features unlocked** — carts, checkouts, accounts, all fully testable
 - **Evolving** — harder tasks drive richer mirrors; the environment grows with agents
 - **RL-ready** — sub-second database resets between rollouts
-- **Community-driven** — 15 sites today, scaling to 100+ together
+- **Community-driven** — 17 sites today, scaling to 100+ together
 
 ## 🚀 Quickstart
 
 One command to run all web environments:
 
 ```bash
-docker run -p 8101:8101 -p 40000-40014:40000-40014 battalion7244/webharbor:latest
+docker run -p 8101:8101 -p 40000-40016:40000-40016 battalion7244/webharbor:latest
 ```
 
-Then point your agent at `http://localhost:40000` through `http://localhost:40014` to explore 15 local mirrors of webvoyager sites: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, and ESPN`.
+Then point your agent at `http://localhost:40000` through `http://localhost:40016` to explore 17 local mirrors: `Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, GitHub, Google Flights, Google Maps, Google Search, Hugging Face, Wolfram Alpha, Cambridge Dictionary, Coursera, ESPN, Merriam-Webster, and Drugs.com`.
 
 For sub-second reset between rollouts, expose the control plane and call `/reset/<site>`:
 
@@ -65,7 +65,7 @@ git clone https://github.com/aiming-lab/WebHarbor && cd WebHarbor
 
 ## 🤝 Contribute
 
-We have built 15 high-quality mirrors covering the [WebVoyager](https://github.com/MinorJerry/WebVoyager) benchmark. The next goal is **100+ sites**, covering everything in [Online-Mind2Web](https://huggingface.co/datasets/osunlp/Online-Mind2Web). We are inviting the community to build this together.
+We have built 17 high-quality mirrors, including the original [WebVoyager](https://github.com/MinorJerry/WebVoyager) coverage. The next goal is **100+ sites**, covering everything in [Online-Mind2Web](https://huggingface.co/datasets/osunlp/Online-Mind2Web). We are inviting the community to build this together.
 
 There are two ways to join the author list:
 
@@ -111,4 +111,4 @@ WebHarbor is initiated by UNC-Chapel Hill and Microsoft, with contributions from
   url          = {https://aiming-lab.github.io/webharbor.github.io},
   note         = {Project website.}
 }
-```
+```

control_server.py

@@ -26,7 +26,7 @@
     'allrecipes', 'amazon', 'apple', 'arxiv', 'bbc_news', 'booking',
     'github', 'google_flights', 'google_map', 'google_search',
     'huggingface', 'wolfram_alpha', 'cambridge_dictionary',
-    'coursera', 'espn', 'merriam_webster',
+    'coursera', 'espn', 'merriam_webster', 'drugs_com',
 ]
 BASE_PORT = 40000
 WEBSYN_DIR = '/opt/WebSyn'

scripts/extract_assets.sh

@@ -42,7 +42,15 @@ for site_dir in sites/*/; do
     fi
 
     out="$TARGET/$site.tar.gz"
-    tar -czf "$out" -C sites "${members[@]}"
+    # Drop any macOS junk already on disk, then pack with COPYFILE_DISABLE=1 so
+    # Apple's bsdtar does not synthesize ._* AppleDouble sidecars from extended
+    # attributes (e.g. com.apple.provenance on Sonoma+). GNU tar ignores the env
+    # var, so this stays correct on Linux. Keeps every <site>.tar.gz free of
+    # ._* / .DS_Store cruft regardless of which machine packs it.
+    for m in "${members[@]}"; do
+        find "sites/$m" \( -name '._*' -o -name '.DS_Store' \) -delete 2>/dev/null || true
+    done
+    COPYFILE_DISABLE=1 tar -czf "$out" -C sites "${members[@]}"
     sz=$(du -sh "$out" 2>/dev/null | cut -f1)
     printf "  %-22s -> %-30s %s\n" "$site" "$site.tar.gz" "$sz"
     count=$((count + 1))