From 81f0eeba37c15bbaceb6520211bd77d786890724 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:14:18 -0400 Subject: [PATCH 1/4] docs(api): channels api page, and the once-per-window call limits on zones and scopes --- docs/channels-api.md | 106 +++++++++++++++++++++++++++++++++++++++++++ docs/scopes-api.md | 26 ++++++++--- docs/zones-api.md | 26 ++++++++--- mkdocs.yml | 1 + 4 files changed, 146 insertions(+), 13 deletions(-) create mode 100644 docs/channels-api.md diff --git a/docs/channels-api.md b/docs/channels-api.md new file mode 100644 index 0000000..84506f4 --- /dev/null +++ b/docs/channels-api.md @@ -0,0 +1,106 @@ +# Channels API + +The Channels API lists the hashtag channels a region's admins have set up for it. That's the same list the MeshMapper app listens on for passive RX in that region. Use it to show a region's local channels, or to set them up in your own client, without scraping the map. No key is needed. + +``` +GET https://yow.meshmapper.net/get_channels.php +``` + +- On a **region** site: that region's own channels. +- On a **group** site (e.g. `pnw.meshmapper.net`): every enabled member region's channels, combined. A channel set up in several members is listed once. + +There are no parameters. + +!!! warning "One call per day, or you're banned" + Each region site's answer can be fetched **once every 23.5 hours per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. + +## Response + +```json +{ + "generated_at": "2026-09-30T18:00:00Z", + "region": "YOW", + "zones": ["YOW"], + "count": 2, + "channels": [ + { "name": "#ottawa", "key": "7871ec72b45617696c35c970bddd8124", "hash": "21", "zones": ["YOW"] }, + { "name": "#yow-mesh", "key": "f371e82cb45c1f1d55f97f917055c1a2", "hash": "6c", "zones": ["YOW"] } + ] +} +``` + +### Top-level fields + +| Field | Description | +| --- | --- | +| `generated_at` | When this answer was built. | +| `region` | The region code this page serves, uppercase. On a group site this is the group's own code. | +| `zones` | The region codes included in this answer: one code on a region site, every enabled member on a group site. | +| `count` | How many channels are listed. | +| `channels` | One entry per channel, sorted by name. Empty when the region hasn't set any up. | + +### Channel fields + +| Field | Description | +| --- | --- | +| `name` | The channel name with its `#`, in lowercase. | +| `key` | The channel's 16-byte secret as 32 hex characters: the first 16 bytes of the SHA-256 of `name` (including the `#`). Every MeshCore client derives the same key from the name; it's here so you don't have to. | +| `hash` | The channel's one-byte hash as 2 hex characters: the first byte of the SHA-256 of the key. This is the byte that identifies the channel in packets on air. | +| `zones` | The region codes that set up this channel. Always a list, even on a region site. | + +The **Public** channel isn't listed, since every region has it and its key is fixed. Neither is the app's own wardriving channel, which isn't regional. + +Hashtag channels are public by design: anyone who knows the name can derive the key. Nothing private is in this response. + +To check a `key` and `hash` yourself: + +```python +import hashlib + +def channel_key(name): # name includes the '#' + return hashlib.sha256(name.lower().encode()).digest()[:16] + +k = channel_key("#ottawa") +print(k.hex(), hashlib.sha256(k).hexdigest()[:2]) # 7871ec72b45617696c35c970bddd8124 21 +``` + +## Call limits + +- **One call per region site every 23.5 hours per IP address.** A group site counts as its own site, separate from its members. +- **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. +- **Only a successful answer counts.** A `200` or a `304` uses your call. A `404` or `503` doesn't, so retrying after one of those is safe. +- **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. +- **The window is 23.5 hours, not 24**, so a scheduled job that runs at the same time every day always has room. A job that runs more often than once a day will get banned. +- **While testing, don't open the URL twice.** Save the response to a file once and work from the file. + +## Caching + +Sends `Cache-Control: public, max-age=84600` (23.5 hours) and a strong `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next daily call, send the `ETag` back in `If-None-Match` to get `304 Not Modified` with no body when nothing changed. `generated_at` isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. + +`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). + +Responses are gzip-compressed. + +## Errors + +Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. + +| Status | `error` | Meaning | +| --- | --- | --- | +| 404 | `zone_not_found` | The region is unknown, pending or turned off. | +| 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | +| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | + +## Example + +Run this once a day from a scheduled job, never in a loop: + +```python +import json, httpx + +r = httpx.get("https://yow.meshmapper.net/get_channels.php").json() +json.dump(r, open("yow-channels.json", "w")) +for c in r["channels"]: + print(c["name"], c["key"]) +``` diff --git a/docs/scopes-api.md b/docs/scopes-api.md index 49387ca..7764796 100644 --- a/docs/scopes-api.md +++ b/docs/scopes-api.md @@ -11,8 +11,8 @@ GET https://yow.meshmapper.net/get_scopes.php There are no parameters. -!!! note "Fair use" - Public, cached for 5 minutes, and rate limited to 60 requests per minute per IP address. Cache the results, respect `ETag`, and don't poll more than once every few minutes. Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "One call per hour, or you're banned" + Each region site's answer can be fetched **once every 55 minutes per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for an hour**. Every repeat doubles the ban, up to a week. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. ## Response @@ -54,24 +54,38 @@ There are no parameters. Names and counts only. There is no per-repeater list, no repeater ID, and no key of any kind in this response. +## Call limits + +- **One call per region site every 55 minutes per IP address.** A group site counts as its own site, separate from its members. +- **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 hour. Each repeat doubles the ban: 2 hours, 4 hours, 8 hours, up to a week. +- **Only a successful answer counts.** A `200` or a `304` uses your call. A `404` or `503` doesn't, so retrying after one of those is safe. +- **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. +- **The window is 55 minutes, not 60**, so a scheduled job that runs once an hour always has room. A job that runs more often than hourly will get banned. +- **While testing, don't open the URL twice.** Save the response to a file once and work from the file. + ## Caching -Sends `Cache-Control: public, max-age=300` and a strong `ETag`. Send the `ETag` back in `If-None-Match` to get `304 Not Modified` with no body when nothing changed. `generated_at` isn't part of the `ETag`. A `304` still counts against the rate limit. +Sends `Cache-Control: public, max-age=3300` (55 minutes) and a strong `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next call, send the `ETag` back in `If-None-Match` to get `304 Not Modified` with no body when nothing changed. `generated_at` isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. + +`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). Responses are gzip-compressed. ## Errors -Errors are JSON: `{"error": ""}`. +Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. | Status | `error` | Meaning | | --- | --- | --- | | 404 | `zone_not_found` | The region is unknown, pending or turned off. | -| 429 | `rate_limited` | Over 60 requests in a minute. Wait for the `Retry-After` header (seconds). | -| 503 | `unavailable` | Temporary server problem. Try again later. | +| 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | +| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 55 minutes. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | ## Example +Run this at most once an hour from a scheduled job, never in a loop: + ```python import httpx diff --git a/docs/zones-api.md b/docs/zones-api.md index a755f4c..a16c209 100644 --- a/docs/zones-api.md +++ b/docs/zones-api.md @@ -9,8 +9,8 @@ Two endpoints work together: Call the first one, then fetch `url + "get_geojson.php"` for each region you want. -!!! note "Fair use" - Both endpoints are public, cached for an hour, and rate limited to 60 requests per minute per IP address (shared between the two). Cache the results, respect `ETag`, and don't poll more than once an hour. Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "One call per day, or you're banned" + Each answer can be fetched **once every 23.5 hours per IP address**: `get_zones.php` once per country, and `get_geojson.php` once per region site. A second request for the same country or region inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. ## List regions @@ -114,27 +114,39 @@ Returns a GeoJSON `FeatureCollection` ([RFC 7946](https://datatracker.ietf.org/d - `center` is `[longitude, latitude]` too. `radius_km` can be `null`. - If the response is cut off mid-stream, the JSON is left unclosed on purpose. Treat a parse error as "retry". +## Call limits + +- **One call per target every 23.5 hours per IP address.** For `get_zones.php` the target is the country (`?country=CA` and `?country=US` are separate calls). For `get_geojson.php` it's the region site, and a group site counts as its own site, separate from its members. Fetching every region of a country in one run is fine: each region is its own call. +- **A second call for the same target inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. +- **Only a successful answer counts.** A `200` or a `304` uses your call. A `400`, `404` or `503` doesn't, so retrying after one of those is safe. +- **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. +- **The window is 23.5 hours, not 24**, so a scheduled job that runs at the same time every day always has room. A job that runs more often than once a day will get banned. +- **While testing, don't open a URL twice.** Save the response to a file once and work from the file. + ## Caching -Both endpoints send `Cache-Control: public, max-age=3600` and an `ETag`. To check for changes cheaply, send the `ETag` back in `If-None-Match`; you'll get `304 Not Modified` with no body if nothing changed. `generated_at` changes on every response and isn't part of the `ETag`. A `304` still counts against the rate limit. +Both endpoints send `Cache-Control: public, max-age=84600` (23.5 hours) and an `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next daily call, send the `ETag` back in `If-None-Match`; you'll get `304 Not Modified` with no body if nothing changed. `generated_at` changes on every response and isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. + +`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). Responses are gzip-compressed. ## Errors -Errors are JSON: `{"error": ""}`. +Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. | Status | `error` | Meaning | | --- | --- | --- | | 400 | `country_required` | `get_zones.php` was called without `country`. | | 400 | `invalid_country` | `country` isn't two letters. | | 404 | `zone_not_found` | The region is unknown, pending or turned off. | -| 429 | `rate_limited` | Over 60 requests in a minute. Wait for the `Retry-After` header (seconds). | -| 503 | `unavailable` | Temporary server problem. Try again later. | +| 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | +| 429 | `rate_limited` | This country or region was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | ## Example -Fetch every Canadian region with a boundary into one GeoJSON file: +Run this once a day from a scheduled job, never in a loop. It fetches every Canadian region with a boundary into one GeoJSON file (one call to `get_zones.php`, then one call per region): ```python import json, httpx diff --git a/mkdocs.yml b/mkdocs.yml index 4133a81..8df4545 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -51,6 +51,7 @@ nav: - Coverage API: coverage-api.md - Zones API: zones-api.md - Scopes API: scopes-api.md + - Channels API: channels-api.md - Data Upload: dataupload.md - Support: From 2a3d0ff5f29cc277911e58f2e982d9f3533f27fb Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 23:32:37 -0400 Subject: [PATCH 2/4] docs(api): a server-side stream failure hands the call back --- docs/zones-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zones-api.md b/docs/zones-api.md index a16c209..0d11226 100644 --- a/docs/zones-api.md +++ b/docs/zones-api.md @@ -112,7 +112,7 @@ Returns a GeoJSON `FeatureCollection` ([RFC 7946](https://datatracker.ietf.org/d - The geometry is always a single `Polygon`. - A region with no drawn boundary has `"geometry": null` and `has_boundary: false`. MeshMapper doesn't draw a circle in its place, but `center` and `radius_km` are there if you want one. - `center` is `[longitude, latitude]` too. `radius_km` can be `null`. -- If the response is cut off mid-stream, the JSON is left unclosed on purpose. Treat a parse error as "retry". +- If the server hits a problem mid-stream, the JSON is left unclosed on purpose and your call is handed back, so treat a parse error as "retry". A response that your own network cut off after the server finished sending still counts as your call. ## Call limits From 61165c071d7b625c7a90ba4fc4c6b68a74d98c87 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Thu, 1 Oct 2026 00:17:07 -0400 Subject: [PATCH 3/4] docs(api): only JSON errors hand the call back, track your last call, call from your server --- docs/channels-api.md | 31 +++++++++++++++++++++---------- docs/scopes-api.md | 31 ++++++++++++++++++++++--------- docs/zones-api.md | 43 +++++++++++++++++++++++++++++++------------ 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/docs/channels-api.md b/docs/channels-api.md index 84506f4..adb43e1 100644 --- a/docs/channels-api.md +++ b/docs/channels-api.md @@ -14,6 +14,9 @@ There are no parameters. !!! warning "One call per day, or you're banned" Each region site's answer can be fetched **once every 23.5 hours per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "Call this from your server, not your visitors' browsers" + Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. + ## Response ```json @@ -68,16 +71,17 @@ print(k.hex(), hashlib.sha256(k).hexdigest()[:2]) # 7871ec72b45617696c35c970bd - **One call per region site every 23.5 hours per IP address.** A group site counts as its own site, separate from its members. - **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. -- **Only a successful answer counts.** A `200` or a `304` uses your call. A `404` or `503` doesn't, so retrying after one of those is safe. +- **Only certain JSON errors hand your call back.** A `404` `zone_not_found` or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. -- **The window is 23.5 hours, not 24**, so a scheduled job that runs at the same time every day always has room. A job that runs more often than once a day will get banned. +- **Don't rely on the clock, track your last call.** Store the time of your last served call for each region site and skip the call if it was less than 23.5 hours ago. Schedule in UTC: a local-time cron job gets a 23-hour day at the DST change, which is enough to trip the limit. +- **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. - **While testing, don't open the URL twice.** Save the response to a file once and work from the file. ## Caching Sends `Cache-Control: public, max-age=84600` (23.5 hours) and a strong `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next daily call, send the `ETag` back in `If-None-Match` to get `304 Not Modified` with no body when nothing changed. `generated_at` isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. -`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). +`ETag` and `Retry-After` are readable from browser JavaScript (`Access-Control-Expose-Headers`), for server-side tools and your own debugging, not so you can embed this call directly in a page your visitors load; see the warning above. Responses are gzip-compressed. @@ -94,13 +98,20 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. ## Example -Run this once a day from a scheduled job, never in a loop: +Run this once a day from a scheduled job on your own server, never in a loop and never from a browser. It tracks the last served call so an early rerun skips instead of risking a ban: ```python -import json, httpx - -r = httpx.get("https://yow.meshmapper.net/get_channels.php").json() -json.dump(r, open("yow-channels.json", "w")) -for c in r["channels"]: - print(c["name"], c["key"]) +import json, os, time, httpx + +STATE = "meshmapper-state.json" +last = json.load(open(STATE)) if os.path.exists(STATE) else {} + +url = "https://yow.meshmapper.net/get_channels.php" +if time.time() - last.get(url, 0) >= 23.5 * 3600: + r = httpx.get(url, timeout=120) + if r.status_code == 200: + last[url] = time.time() + json.dump(r.json(), open("yow-channels.json", "w")) + # anything else: keep your previous copy, don't retry here +json.dump(last, open(STATE, "w")) ``` diff --git a/docs/scopes-api.md b/docs/scopes-api.md index 7764796..1d44966 100644 --- a/docs/scopes-api.md +++ b/docs/scopes-api.md @@ -14,6 +14,9 @@ There are no parameters. !!! warning "One call per hour, or you're banned" Each region site's answer can be fetched **once every 55 minutes per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for an hour**. Every repeat doubles the ban, up to a week. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "Call this from your server, not your visitors' browsers" + Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. + ## Response ```json @@ -58,16 +61,17 @@ Names and counts only. There is no per-repeater list, no repeater ID, and no key - **One call per region site every 55 minutes per IP address.** A group site counts as its own site, separate from its members. - **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 hour. Each repeat doubles the ban: 2 hours, 4 hours, 8 hours, up to a week. -- **Only a successful answer counts.** A `200` or a `304` uses your call. A `404` or `503` doesn't, so retrying after one of those is safe. +- **Only certain JSON errors hand your call back.** A `404` `zone_not_found` or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. -- **The window is 55 minutes, not 60**, so a scheduled job that runs once an hour always has room. A job that runs more often than hourly will get banned. +- **Don't rely on the clock, track your last call.** Store the time of your last served call for each region site and skip the call if it was less than 55 minutes ago. Schedule in UTC: a local-time cron's hour mapping shifts at the DST change, which can land two runs closer together than you expect. +- **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. - **While testing, don't open the URL twice.** Save the response to a file once and work from the file. ## Caching Sends `Cache-Control: public, max-age=3300` (55 minutes) and a strong `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next call, send the `ETag` back in `If-None-Match` to get `304 Not Modified` with no body when nothing changed. `generated_at` isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. -`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). +`ETag` and `Retry-After` are readable from browser JavaScript (`Access-Control-Expose-Headers`), for server-side tools and your own debugging, not so you can embed this call directly in a page your visitors load; see the warning above. Responses are gzip-compressed. @@ -84,12 +88,21 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. ## Example -Run this at most once an hour from a scheduled job, never in a loop: +Run this at most once an hour from a scheduled job on your own server, never in a loop and never from a browser. It tracks the last served call so an early rerun skips instead of risking a ban: ```python -import httpx - -r = httpx.get("https://yow.meshmapper.net/get_scopes.php").json() -for s in r["scopes"]: - print(s["name"], s["repeaters"], "of", r["repeaters"]) +import json, os, time, httpx + +STATE = "meshmapper-state.json" +last = json.load(open(STATE)) if os.path.exists(STATE) else {} + +url = "https://yow.meshmapper.net/get_scopes.php" +if time.time() - last.get(url, 0) >= 55 * 60: + r = httpx.get(url, timeout=120) + if r.status_code == 200: + last[url] = time.time() + for s in r.json()["scopes"]: + print(s["name"], s["repeaters"]) + # anything else: keep your previous copy, don't retry here +json.dump(last, open(STATE, "w")) ``` diff --git a/docs/zones-api.md b/docs/zones-api.md index 0d11226..2ff01d3 100644 --- a/docs/zones-api.md +++ b/docs/zones-api.md @@ -12,6 +12,9 @@ Call the first one, then fetch `url + "get_geojson.php"` for each region you wan !!! warning "One call per day, or you're banned" Each answer can be fetched **once every 23.5 hours per IP address**: `get_zones.php` once per country, and `get_geojson.php` once per region site. A second request for the same country or region inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "Call this from your server, not your visitors' browsers" + Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. + ## List regions ``` @@ -112,22 +115,23 @@ Returns a GeoJSON `FeatureCollection` ([RFC 7946](https://datatracker.ietf.org/d - The geometry is always a single `Polygon`. - A region with no drawn boundary has `"geometry": null` and `has_boundary: false`. MeshMapper doesn't draw a circle in its place, but `center` and `radius_km` are there if you want one. - `center` is `[longitude, latitude]` too. `radius_km` can be `null`. -- If the server hits a problem mid-stream, the JSON is left unclosed on purpose and your call is handed back, so treat a parse error as "retry". A response that your own network cut off after the server finished sending still counts as your call. +- If the server hits a problem mid-stream, the JSON is deliberately left unclosed, so you get a truncated, unparseable body, not a clean JSON error. A parse error does **not** by itself mean your call was handed back; see [Call limits](#call-limits) for exactly which errors are safe to retry. ## Call limits - **One call per target every 23.5 hours per IP address.** For `get_zones.php` the target is the country (`?country=CA` and `?country=US` are separate calls). For `get_geojson.php` it's the region site, and a group site counts as its own site, separate from its members. Fetching every region of a country in one run is fine: each region is its own call. - **A second call for the same target inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. -- **Only a successful answer counts.** A `200` or a `304` uses your call. A `400`, `404` or `503` doesn't, so retrying after one of those is safe. +- **Only certain JSON errors hand your call back.** A `400` (missing or bad `country` on `get_zones.php`), a `404` `zone_not_found`, or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. -- **The window is 23.5 hours, not 24**, so a scheduled job that runs at the same time every day always has room. A job that runs more often than once a day will get banned. +- **Don't rely on the clock, track your last call.** Store the time of your last served call for each target (country or region) and skip the call if it was less than 23.5 hours ago. Schedule in UTC: a local-time cron job gets a 23-hour day at the DST change, which is enough to trip the limit. +- **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. - **While testing, don't open a URL twice.** Save the response to a file once and work from the file. ## Caching Both endpoints send `Cache-Control: public, max-age=84600` (23.5 hours) and an `ETag`, so a standard HTTP cache won't ask again before your next call is allowed. On your next daily call, send the `ETag` back in `If-None-Match`; you'll get `304 Not Modified` with no body if nothing changed. `generated_at` changes on every response and isn't part of the `ETag`. A `304` uses your call for that window, just like a `200`. -`ETag` and `Retry-After` are readable from browser JavaScript on other sites (`Access-Control-Expose-Headers`). +`ETag` and `Retry-After` are readable from browser JavaScript (`Access-Control-Expose-Headers`), for server-side tools and your own debugging, not so you can embed this call directly in a page your visitors load; see the warning above. Responses are gzip-compressed. @@ -146,16 +150,31 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. ## Example -Run this once a day from a scheduled job, never in a loop. It fetches every Canadian region with a boundary into one GeoJSON file (one call to `get_zones.php`, then one call per region): +Run this from a scheduled job on your own server, never in a loop and never from a browser. It fetches every Canadian region with a boundary into one GeoJSON file (one call to `get_zones.php`, then one call per region), tracking the last served call per URL so an early rerun skips instead of risking a ban: ```python -import json, httpx +import json, os, time, httpx + +STATE = "meshmapper-state.json" +last = json.load(open(STATE)) if os.path.exists(STATE) else {} + +def fetch(url, **params): + if time.time() - last.get(url, 0) < 23.5 * 3600: + return None # already used this call recently + r = httpx.get(url, params=params, timeout=120) + if r.status_code != 200: + return None # keep the previous copy, don't retry here + last[url] = time.time() + return r.json() -zones = httpx.get("https://meshmapper.net/get_zones.php", params={"country": "CA"}).json() +zones = fetch("https://meshmapper.net/get_zones.php", country="CA") features = [] -for z in zones["zones"]: - if z["has_boundary"]: - fc = httpx.get(z["url"] + "get_geojson.php").json() - features.extend(fc["features"]) -json.dump({"type": "FeatureCollection", "features": features}, open("meshmapper-ca.geojson", "w")) +if zones: + for z in zones["zones"]: + if z["has_boundary"]: + fc = fetch(z["url"] + "get_geojson.php") + if fc: + features.extend(fc["features"]) + json.dump({"type": "FeatureCollection", "features": features}, open("meshmapper-ca.geojson", "w")) +json.dump(last, open(STATE, "w")) ``` From 9dee89e540cec6ec429d4d16f27bc5f33bc59644 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Thu, 1 Oct 2026 04:18:47 -0400 Subject: [PATCH 4/4] docs(api): the 3rd refusal in the window bans, 30 calls a minute across the APIs, zones example tracks per country --- docs/channels-api.md | 14 ++++++++------ docs/scopes-api.md | 14 ++++++++------ docs/zones-api.md | 20 ++++++++++++-------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/channels-api.md b/docs/channels-api.md index adb43e1..9af4bd6 100644 --- a/docs/channels-api.md +++ b/docs/channels-api.md @@ -11,8 +11,8 @@ GET https://yow.meshmapper.net/get_channels.php There are no parameters. -!!! warning "One call per day, or you're banned" - Each region site's answer can be fetched **once every 23.5 hours per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "One call per day; keep calling early and you're banned" + Each region site's answer can be fetched **once every 23.5 hours per IP address**. A second request to the same region site inside that window is refused with `429`, and **the third refused request within the window bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. !!! warning "Call this from your server, not your visitors' browsers" Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. @@ -70,12 +70,13 @@ print(k.hex(), hashlib.sha256(k).hexdigest()[:2]) # 7871ec72b45617696c35c970bd ## Call limits - **One call per region site every 23.5 hours per IP address.** A group site counts as its own site, separate from its members. -- **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. -- **Only certain JSON errors hand your call back.** A `404` `zone_not_found` or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. +- **An early call is refused with `429`; the third refusal within the window bans your IP address.** Your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. +- **At most 30 calls a minute from one IP address, across all these APIs together.** A call over that gets `503` with `{"error":"slow_down","retry_after":N}`. It doesn't use your call and never counts toward a ban: wait `Retry-After` seconds and carry on. Pausing 2 seconds between calls keeps you under it. +- **Only certain JSON errors hand your call back.** A `404` `zone_not_found`, a `503` `unavailable`, or a `503` `slow_down` (after waiting `Retry-After`) doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. - **Don't rely on the clock, track your last call.** Store the time of your last served call for each region site and skip the call if it was less than 23.5 hours ago. Schedule in UTC: a local-time cron job gets a 23-hour day at the DST change, which is enough to trip the limit. - **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. -- **While testing, don't open the URL twice.** Save the response to a file once and work from the file. +- **While testing, don't open the URL twice.** Save the response to a file once and work from the file; every reload after the first is refused and counts toward a ban. ## Caching @@ -93,8 +94,9 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. | --- | --- | --- | | 404 | `zone_not_found` | The region is unknown, pending or turned off. | | 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | -| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Three of these within the window ban your IP address; see [Call limits](#call-limits). | | 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | +| 503 | `slow_down` | More than 30 calls in the last minute from your IP address, across all these APIs. `Retry-After` and `retry_after` give the seconds left. It doesn't use your call and never counts toward a ban. | ## Example diff --git a/docs/scopes-api.md b/docs/scopes-api.md index 1d44966..2f1a7d4 100644 --- a/docs/scopes-api.md +++ b/docs/scopes-api.md @@ -11,8 +11,8 @@ GET https://yow.meshmapper.net/get_scopes.php There are no parameters. -!!! warning "One call per hour, or you're banned" - Each region site's answer can be fetched **once every 55 minutes per IP address**. A second request to the same region site inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for an hour**. Every repeat doubles the ban, up to a week. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "One call per hour; keep calling early and you're banned" + Each region site's answer can be fetched **once every 55 minutes per IP address**. A second request to the same region site inside that window is refused with `429`, and **the third refused request within the window bans your IP address from all of meshmapper.net for an hour**. Every repeat doubles the ban, up to a week. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. !!! warning "Call this from your server, not your visitors' browsers" Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. @@ -60,12 +60,13 @@ Names and counts only. There is no per-repeater list, no repeater ID, and no key ## Call limits - **One call per region site every 55 minutes per IP address.** A group site counts as its own site, separate from its members. -- **A second call inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 hour. Each repeat doubles the ban: 2 hours, 4 hours, 8 hours, up to a week. -- **Only certain JSON errors hand your call back.** A `404` `zone_not_found` or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. +- **An early call is refused with `429`; the third refusal within the window bans your IP address.** Your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 hour. Each repeat doubles the ban: 2 hours, 4 hours, 8 hours, up to a week. +- **At most 30 calls a minute from one IP address, across all these APIs together.** A call over that gets `503` with `{"error":"slow_down","retry_after":N}`. It doesn't use your call and never counts toward a ban: wait `Retry-After` seconds and carry on. Pausing 2 seconds between calls keeps you under it. +- **Only certain JSON errors hand your call back.** A `404` `zone_not_found`, a `503` `unavailable`, or a `503` `slow_down` (after waiting `Retry-After`) doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. - **Don't rely on the clock, track your last call.** Store the time of your last served call for each region site and skip the call if it was less than 55 minutes ago. Schedule in UTC: a local-time cron's hour mapping shifts at the DST change, which can land two runs closer together than you expect. - **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. -- **While testing, don't open the URL twice.** Save the response to a file once and work from the file. +- **While testing, don't open the URL twice.** Save the response to a file once and work from the file; every reload after the first is refused and counts toward a ban. ## Caching @@ -83,8 +84,9 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. | --- | --- | --- | | 404 | `zone_not_found` | The region is unknown, pending or turned off. | | 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | -| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 55 minutes. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 429 | `rate_limited` | This region site was already fetched from your IP address in the last 55 minutes. `Retry-After` and `retry_after` give the seconds left. Three of these within the window ban your IP address; see [Call limits](#call-limits). | | 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | +| 503 | `slow_down` | More than 30 calls in the last minute from your IP address, across all these APIs. `Retry-After` and `retry_after` give the seconds left. It doesn't use your call and never counts toward a ban. | ## Example diff --git a/docs/zones-api.md b/docs/zones-api.md index 2ff01d3..46be1dd 100644 --- a/docs/zones-api.md +++ b/docs/zones-api.md @@ -9,8 +9,8 @@ Two endpoints work together: Call the first one, then fetch `url + "get_geojson.php"` for each region you want. -!!! warning "One call per day, or you're banned" - Each answer can be fetched **once every 23.5 hours per IP address**: `get_zones.php` once per country, and `get_geojson.php` once per region site. A second request for the same country or region inside that window is refused with `429` **and bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. +!!! warning "One call per day; keep calling early and you're banned" + Each answer can be fetched **once every 23.5 hours per IP address**: `get_zones.php` once per country, and `get_geojson.php` once per region site. A second request for the same country or region inside that window is refused with `429`, and **the third refused request within the window bans your IP address from all of meshmapper.net for a day**. Every repeat doubles the ban, up to 30 days. See [Call limits](#call-limits). Accessing anything that isn't a published API, or scraping pages for data, is not allowed; see the warning on the [Coverage API](coverage-api.md) page. !!! warning "Call this from your server, not your visitors' browsers" Fetch this API from your own backend, store the result, and serve your own copy to visitors. Don't call it from client-side JavaScript in a visitor's browser: everyone behind the same home router or mobile carrier shares one public IP address, so one visitor's fetch uses the call and the next visitor's fetch bans that whole IP address from all of meshmapper.net, including every MeshMapper app user on that network. @@ -120,12 +120,13 @@ Returns a GeoJSON `FeatureCollection` ([RFC 7946](https://datatracker.ietf.org/d ## Call limits - **One call per target every 23.5 hours per IP address.** For `get_zones.php` the target is the country (`?country=CA` and `?country=US` are separate calls). For `get_geojson.php` it's the region site, and a group site counts as its own site, separate from its members. Fetching every region of a country in one run is fine: each region is its own call. -- **A second call for the same target inside the window bans your IP address.** The request is refused with `429` and your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. -- **Only certain JSON errors hand your call back.** A `400` (missing or bad `country` on `get_zones.php`), a `404` `zone_not_found`, or a `503` `unavailable` doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. +- **An early call for the same target is refused with `429`; the third refusal within the window bans your IP address.** Your IP address is banned from all of meshmapper.net (the map included, and every device behind that IP address) for 1 day. Each repeat doubles the ban: 2 days, 4 days, 8 days, up to 30 days. +- **At most 30 calls a minute from one IP address, across all these APIs together.** A call over that gets `503` with `{"error":"slow_down","retry_after":N}`. It doesn't use your call and never counts toward a ban: wait `Retry-After` seconds and carry on. Pausing 2 seconds between calls keeps you under it. +- **Only certain JSON errors hand your call back.** A `400` (missing or bad `country` on `get_zones.php`), a `404` `zone_not_found`, a `503` `unavailable`, or a `503` `slow_down` (after waiting `Retry-After`) doesn't use your call, so retrying after one of those is safe. Anything else, a cut-off or unparseable body, a timeout on your side, an HTML `5xx` error page, a connection reset, may already have used your call: keep your previous copy and wait for your next scheduled run instead of retrying. - **Only `GET` counts.** A browser's CORS preflight (`OPTIONS`) doesn't. `HEAD` and other methods get `405` and don't count either. - **Don't rely on the clock, track your last call.** Store the time of your last served call for each target (country or region) and skip the call if it was less than 23.5 hours ago. Schedule in UTC: a local-time cron job gets a 23-hour day at the DST change, which is enough to trip the limit. - **Use a generous client timeout, 60 to 120 seconds.** A short timeout on your side can cut the connection before the server finishes, and that counts against you as an "anything else" error above, not a safe-to-retry one. -- **While testing, don't open a URL twice.** Save the response to a file once and work from the file. +- **While testing, don't open a URL twice.** Save the response to a file once and work from the file; every reload after the first is refused and counts toward a ban. ## Caching @@ -145,8 +146,9 @@ Errors are JSON: `{"error": ""}`. A `429` also carries `retry_after`. | 400 | `invalid_country` | `country` isn't two letters. | | 404 | `zone_not_found` | The region is unknown, pending or turned off. | | 405 | `method_not_allowed` | Only `GET` (and the `OPTIONS` preflight) are answered. | -| 429 | `rate_limited` | This country or region was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Your IP address is now banned; see [Call limits](#call-limits). | +| 429 | `rate_limited` | This country or region was already fetched from your IP address in the last 23.5 hours. `Retry-After` and `retry_after` give the seconds left. Three of these within the window ban your IP address; see [Call limits](#call-limits). | | 503 | `unavailable` | Temporary server problem. Try again later; it doesn't use your call. | +| 503 | `slow_down` | More than 30 calls in the last minute from your IP address, across all these APIs. `Retry-After` and `retry_after` give the seconds left. It doesn't use your call and never counts toward a ban. | ## Example @@ -159,12 +161,13 @@ STATE = "meshmapper-state.json" last = json.load(open(STATE)) if os.path.exists(STATE) else {} def fetch(url, **params): - if time.time() - last.get(url, 0) < 23.5 * 3600: + key = str(httpx.URL(url, params=params)) # the target: for get_zones this includes ?country=.. + if time.time() - last.get(key, 0) < 23.5 * 3600: return None # already used this call recently r = httpx.get(url, params=params, timeout=120) if r.status_code != 200: return None # keep the previous copy, don't retry here - last[url] = time.time() + last[key] = time.time() return r.json() zones = fetch("https://meshmapper.net/get_zones.php", country="CA") @@ -175,6 +178,7 @@ if zones: fc = fetch(z["url"] + "get_geojson.php") if fc: features.extend(fc["features"]) + time.sleep(2) # stay under the 30-calls-a-minute cross-API cap json.dump({"type": "FeatureCollection", "features": features}, open("meshmapper-ca.geojson", "w")) json.dump(last, open(STATE, "w")) ```