From e5f5fac467284b0a191c494e464456d4272f9a0c Mon Sep 17 00:00:00 2001 From: Frederik Ramm Date: Wed, 16 Sep 2026 11:25:41 +0200 Subject: [PATCH 1/9] Add tag usage stats script in almost unmodified version --- tag_usage_stats.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 tag_usage_stats.py diff --git a/tag_usage_stats.py b/tag_usage_stats.py new file mode 100755 index 0000000..60f6fb8 --- /dev/null +++ b/tag_usage_stats.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python3 +# Author: Frederik Ramm +# originally published in https://github.com/shortbread-tiles/shortbread-docs/issues/153#issuecomment-5476984543 +# +# Program to determine a list of values for a given key +# where each value is in the list of N most frequently used values +# across a list of regions. + +import requests +from collections import defaultdict + +N = 100 +tag = 'shop' +regions = ['europe', 'north-america', 'asia', 'south-america', 'africa'] + +globalitem = defaultdict(int) + +for region in regions: + r = requests.get("https://taginfo.geofabrik.de/{}/api/4/key/values?key={}&sortname=count&sortorder=desc&rp={}&page=1".format(region, tag, N)) + for item in r.json()['data']: + globalitem[item['value']] += 1 + +for item in globalitem: + if globalitem[item] == len(regions): + print (item) From e65337c2d9b663fdc34982f6706a0b50b9bce478 Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 11:33:56 +0200 Subject: [PATCH 2/9] Allow users to change regions, key and number --- tag_usage_stats.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index 60f6fb8..0a6d8f3 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -1,22 +1,38 @@ #! /usr/bin/env python3 # Author: Frederik Ramm +# Extended by Michael Reichert # originally published in https://github.com/shortbread-tiles/shortbread-docs/issues/153#issuecomment-5476984543 # # Program to determine a list of values for a given key # where each value is in the list of N most frequently used values # across a list of regions. +import argparse import requests from collections import defaultdict +import sys N = 100 -tag = 'shop' +key = 'shop' regions = ['europe', 'north-america', 'asia', 'south-america', 'africa'] +default_regions = ",".join(regions) + +parser = argparse.ArgumentParser(description="""Program to determine a list of values for a given key +where each value is in the list of N most frequently used values +across a list of regions. +""") +parser.add_argument("-n", "--number", type=int, default=N, help="Change N (defaults to 100)") +parser.add_argument("-k", "--key", type=str, default=key, help="Set OSM key (defaults to 'shop')") +parser.add_argument("-r", "--regions", type=str, default=default_regions, help="Regions to compare. Specify as one string separated by comma. This must match the paths used at taginfo.geofabrik.de") +args = parser.parse_args() +regions = args.regions.split(",") +if len(regions) == 0: + sys.stderr.write("ERROR: No regions provided.\n") globalitem = defaultdict(int) for region in regions: - r = requests.get("https://taginfo.geofabrik.de/{}/api/4/key/values?key={}&sortname=count&sortorder=desc&rp={}&page=1".format(region, tag, N)) + r = requests.get("https://taginfo.geofabrik.de/{}/api/4/key/values?key={}&sortname=count&sortorder=desc&rp={}&page=1".format(region, args.key, args.number)) for item in r.json()['data']: globalitem[item['value']] += 1 From 7dcfc92be2eb22702596f283a5faba86224fd75d Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 11:39:29 +0200 Subject: [PATCH 3/9] Add quoting and escaping --- tag_usage_stats.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index 0a6d8f3..b033e77 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -11,6 +11,7 @@ import requests from collections import defaultdict import sys +import urllib.parse N = 100 key = 'shop' @@ -32,7 +33,8 @@ globalitem = defaultdict(int) for region in regions: - r = requests.get("https://taginfo.geofabrik.de/{}/api/4/key/values?key={}&sortname=count&sortorder=desc&rp={}&page=1".format(region, args.key, args.number)) + url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) + r = requests.get(url, params = { "key": args.key, "sortname": "count", "sortorder": "desc", "rp": str(args.number), "page": "1"}) for item in r.json()['data']: globalitem[item['value']] += 1 From 9e86b8ff01ee69974372627f762f47ecc73ddd2c Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 11:43:19 +0200 Subject: [PATCH 4/9] Log an error message and fail if HTTP status code != 200 --- tag_usage_stats.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index b033e77..a6aabd6 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -35,6 +35,9 @@ for region in regions: url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) r = requests.get(url, params = { "key": args.key, "sortname": "count", "sortorder": "desc", "rp": str(args.number), "page": "1"}) + if r.status_code != 200: + sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) + sys.exit(1) for item in r.json()['data']: globalitem[item['value']] += 1 From 3d2462ccf0e4333f8ed3369d506f6a01819b509c Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 11:45:51 +0200 Subject: [PATCH 5/9] Set user-agent --- tag_usage_stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index a6aabd6..31a43df 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -17,6 +17,7 @@ key = 'shop' regions = ['europe', 'north-america', 'asia', 'south-america', 'africa'] default_regions = ",".join(regions) +headers = {"User-Agent": "shortbread/tag_usage_stats"} parser = argparse.ArgumentParser(description="""Program to determine a list of values for a given key where each value is in the list of N most frequently used values @@ -34,7 +35,7 @@ for region in regions: url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) - r = requests.get(url, params = { "key": args.key, "sortname": "count", "sortorder": "desc", "rp": str(args.number), "page": "1"}) + r = requests.get(url, headers=headers, params={"key": args.key, "sortname": "count", "sortorder": "desc", "rp": str(args.number), "page": "1"}) if r.status_code != 200: sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) sys.exit(1) From 1f106cd2474c70474210bccf358b704ee57c560b Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 14:07:50 +0200 Subject: [PATCH 6/9] Allow to query frequently used keys matching a certain string This is useful to find out which languages are most frequently tagged in various regions. --- tag_usage_stats.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index 31a43df..354f461 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -4,10 +4,23 @@ # originally published in https://github.com/shortbread-tiles/shortbread-docs/issues/153#issuecomment-5476984543 # # Program to determine a list of values for a given key -# where each value is in the list of N most frequently used values +# or keys matching a given search string +# where each value/key is in the list of N most frequently used values/keys # across a list of regions. +# +# Example: +# - `python tag_usage_stats.py -t value -k shop -n 75` +# will return all values of shop=* which are in the list of N most frequently used values of shop=* in all requested regions. +# For example, if shop=outdoor is frequently used in Europe an North America but rarely used in Africa, it will be not be returned. +# However shop=convenience will be returned because it is among the top 75 shop values in all regions. +# - `python tag_usage_stats.py -t key -k name -n 75` +# will return all keys containing the query string "name" if they appear among the 75 most frequently used keys containing +# the substring "name" in all requested regions. +# - `python tag_usage_stats.py -t value -k office -n 100 -r europe/germany,europe/france` +# will return the values of office=* which are in the list of N most freuqently used values in Germany and France. import argparse +import enum import requests from collections import defaultdict import sys @@ -19,12 +32,21 @@ default_regions = ",".join(regions) headers = {"User-Agent": "shortbread/tag_usage_stats"} +class QueryType(enum.Enum): + values = 'value' + keys = 'key' + + def __str__(self): + return self.value + + parser = argparse.ArgumentParser(description="""Program to determine a list of values for a given key where each value is in the list of N most frequently used values across a list of regions. """) -parser.add_argument("-n", "--number", type=int, default=N, help="Change N (defaults to 100)") parser.add_argument("-k", "--key", type=str, default=key, help="Set OSM key (defaults to 'shop')") +parser.add_argument("-n", "--number", type=int, default=N, help="Change N (defaults to 100)") +parser.add_argument("-t", "--type", type=QueryType, choices=list(QueryType), default=QueryType.values, help="Whether to query the N most frequently used values of a given key or the N most frequently used keys matching a given search string.") parser.add_argument("-r", "--regions", type=str, default=default_regions, help="Regions to compare. Specify as one string separated by comma. This must match the paths used at taginfo.geofabrik.de") args = parser.parse_args() regions = args.regions.split(",") @@ -35,12 +57,18 @@ for region in regions: url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) - r = requests.get(url, headers=headers, params={"key": args.key, "sortname": "count", "sortorder": "desc", "rp": str(args.number), "page": "1"}) + params = {"sortname": "count_all", "sortorder": "desc", "rp": str(args.number), "page": "1"} + if args.type == QueryType.keys: + url = "https://taginfo.geofabrik.de/{}/api/4/keys/all".format(urllib.parse.quote(region)) + params["query"] = args.key + elif args.type == QueryType.values: + params["key"] = args.key + r = requests.get(url, headers=headers, params=params) if r.status_code != 200: sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) sys.exit(1) for item in r.json()['data']: - globalitem[item['value']] += 1 + globalitem[item[args.type.value]] += 1 for item in globalitem: if globalitem[item] == len(regions): From f60f1ac0c959647cc09c138e656904e961f71c4e Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 16 Sep 2026 14:44:23 +0200 Subject: [PATCH 7/9] Print summed counts --- tag_usage_stats.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tag_usage_stats.py b/tag_usage_stats.py index 354f461..81dc4c7 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -21,6 +21,7 @@ import argparse import enum +import math import requests from collections import defaultdict import sys @@ -53,9 +54,9 @@ def __str__(self): if len(regions) == 0: sys.stderr.write("ERROR: No regions provided.\n") -globalitem = defaultdict(int) +globalitem = {} -for region in regions: +for i, region in enumerate(regions): url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) params = {"sortname": "count_all", "sortorder": "desc", "rp": str(args.number), "page": "1"} if args.type == QueryType.keys: @@ -68,8 +69,20 @@ def __str__(self): sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) sys.exit(1) for item in r.json()['data']: - globalitem[item[args.type.value]] += 1 + entry_key = item[args.type.value] + entry = globalitem.get(entry_key, [0 for j in range(len(regions))]) + attr = "count" + if args.type == QueryType.keys: + attr = "count_all" + entry[i] = item[attr] + globalitem[entry_key] = entry -for item in globalitem: - if globalitem[item] == len(regions): - print (item) +top_entries = [] +for item, counts in globalitem.items(): + if 0 not in counts: + top_entries.append((item, sum(counts))) +top_entries.sort(key=lambda k: k[1], reverse=True) +max_keylength = max([len(k[0]) for k in top_entries]) +max_valuelength = int(max([math.log(k[1], 10) for k in top_entries])) + 1 +for t in top_entries: + print("{} {}".format(t[0].ljust(max_keylength, " "), str(t[1]).rjust(max_valuelength, " "))) From 4ead496f4d30697582f65d9aeb374369751060ff Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Wed, 23 Sep 2026 17:17:15 +0200 Subject: [PATCH 8/9] Explain script --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6f206e9..563f2d9 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,14 @@ This repository does not have installable software. You will need to use a softw that can create vector tiles (e.g. Tilemaker) and combine that with configuration files designed to output vector tiles according to this schema (e.g. [shortbread-tilemaker](https://github.com/shortbread-tiles/shortbread-tilemaker)). +## Scripts + +This repository contains a [utility script](tag_usage_stats.py) to aid decisions which tags to support. +It retrieves the N most frequently used values per key or N most frequently used keys matching a given search string +from taginfo.geofabrik.de for a couple for regions and returns those who appear in all regions among the top N ranks. +Run `python tag_usage_stats.py --help` for instructions how to use. See the comment a the beginning of +the file for usage examples. + ## Authors The schema has been created for Geofabrik by Thomas Skowron, Christine Karch, From ecb5884967fd97d10c23a2fcd1eb3ae1b8b699a4 Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Thu, 24 Sep 2026 13:47:50 +0200 Subject: [PATCH 9/9] Improve documentation, remove unnecessary import, add license --- README.md | 13 ++++++++++--- tag_usage_stats.py | 21 ++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 563f2d9..78a2035 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,17 @@ designed to output vector tiles according to this schema (e.g. [shortbread-tilem ## Scripts -This repository contains a [utility script](tag_usage_stats.py) to aid decisions which tags to support. +### `tag_usage_stats.py` + +Although [Taginfo](https://taginfo.openstreetmap.org) provides global statistics about usage of tags, +it provides a narrow view on actual usage only. Some tags are used a lot in one region but rarely used elsewhere. +Overview maps provided by Taginfo show some differences. + +For decisions which tags should be included a vector tile schema like Shortbread, a comparison between +multiple regions is more helpful. Therefore this repository contains a [utility script](tag_usage_stats.py) to aid decisions which tags to support. It retrieves the N most frequently used values per key or N most frequently used keys matching a given search string -from taginfo.geofabrik.de for a couple for regions and returns those who appear in all regions among the top N ranks. -Run `python tag_usage_stats.py --help` for instructions how to use. See the comment a the beginning of +from [Geofabrik's Taginfo instances](https://taginfo.geofabrik.de) for a couple for regions and returns those who appear in all regions among the top N ranks. +Run `python tag_usage_stats.py --help` for instructions how to use. See the comment at the beginning of the file for usage examples. ## Authors diff --git a/tag_usage_stats.py b/tag_usage_stats.py index 81dc4c7..5ec55c8 100755 --- a/tag_usage_stats.py +++ b/tag_usage_stats.py @@ -1,4 +1,5 @@ #! /usr/bin/env python3 +# SPDX-License-Identifier: CC0-1.0 # Author: Frederik Ramm # Extended by Michael Reichert # originally published in https://github.com/shortbread-tiles/shortbread-docs/issues/153#issuecomment-5476984543 @@ -10,32 +11,30 @@ # # Example: # - `python tag_usage_stats.py -t value -k shop -n 75` -# will return all values of shop=* which are in the list of N most frequently used values of shop=* in all requested regions. +# will return all values of shop=* which are in the list of 75 most frequently used values of shop=* in all of the following regions: Africa, Asia, Europe, North America, South America. # For example, if shop=outdoor is frequently used in Europe an North America but rarely used in Africa, it will be not be returned. # However shop=convenience will be returned because it is among the top 75 shop values in all regions. # - `python tag_usage_stats.py -t key -k name -n 75` -# will return all keys containing the query string "name" if they appear among the 75 most frequently used keys containing -# the substring "name" in all requested regions. +# will return all keys which appear among the 75 most frequently used keys containing +# the substring "name" in all of the following regions: Africa, Asia, Europe, North America, South America. # - `python tag_usage_stats.py -t value -k office -n 100 -r europe/germany,europe/france` -# will return the values of office=* which are in the list of N most freuqently used values in Germany and France. +# will return the values of office=* which are in the list of 100 most freuqently used values in Germany and France. import argparse import enum import math import requests -from collections import defaultdict import sys import urllib.parse N = 100 -key = 'shop' -regions = ['europe', 'north-america', 'asia', 'south-america', 'africa'] -default_regions = ",".join(regions) +key = "shop" +default_regions = ",".join(["europe", "north-america", "asia", "south-america", "africa"]) headers = {"User-Agent": "shortbread/tag_usage_stats"} class QueryType(enum.Enum): - values = 'value' - keys = 'key' + values = "value" + keys = "key" def __str__(self): return self.value @@ -68,7 +67,7 @@ def __str__(self): if r.status_code != 200: sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) sys.exit(1) - for item in r.json()['data']: + for item in r.json()["data"]: entry_key = item[args.type.value] entry = globalitem.get(entry_key, [0 for j in range(len(regions))]) attr = "count"