diff --git a/README.md b/README.md index 6f206e9..78a2035 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,21 @@ This repository does not have installable software. You will need to use a softw that can create vector tiles (e.g. Tilemaker) and combine that with configuration files designed to output vector tiles according to this schema (e.g. [shortbread-tilemaker](https://github.com/shortbread-tiles/shortbread-tilemaker)). +## Scripts + +### `tag_usage_stats.py` + +Although [Taginfo](https://taginfo.openstreetmap.org) provides global statistics about usage of tags, +it provides a narrow view on actual usage only. Some tags are used a lot in one region but rarely used elsewhere. +Overview maps provided by Taginfo show some differences. + +For decisions which tags should be included a vector tile schema like Shortbread, a comparison between +multiple regions is more helpful. Therefore this repository contains a [utility script](tag_usage_stats.py) to aid decisions which tags to support. +It retrieves the N most frequently used values per key or N most frequently used keys matching a given search string +from [Geofabrik's Taginfo instances](https://taginfo.geofabrik.de) for a couple for regions and returns those who appear in all regions among the top N ranks. +Run `python tag_usage_stats.py --help` for instructions how to use. See the comment at the beginning of +the file for usage examples. + ## Authors The schema has been created for Geofabrik by Thomas Skowron, Christine Karch, diff --git a/tag_usage_stats.py b/tag_usage_stats.py new file mode 100755 index 0000000..5ec55c8 --- /dev/null +++ b/tag_usage_stats.py @@ -0,0 +1,87 @@ +#! /usr/bin/env python3 +# SPDX-License-Identifier: CC0-1.0 +# Author: Frederik Ramm +# Extended by Michael Reichert +# originally published in https://github.com/shortbread-tiles/shortbread-docs/issues/153#issuecomment-5476984543 +# +# Program to determine a list of values for a given key +# or keys matching a given search string +# where each value/key is in the list of N most frequently used values/keys +# across a list of regions. +# +# Example: +# - `python tag_usage_stats.py -t value -k shop -n 75` +# will return all values of shop=* which are in the list of 75 most frequently used values of shop=* in all of the following regions: Africa, Asia, Europe, North America, South America. +# For example, if shop=outdoor is frequently used in Europe an North America but rarely used in Africa, it will be not be returned. +# However shop=convenience will be returned because it is among the top 75 shop values in all regions. +# - `python tag_usage_stats.py -t key -k name -n 75` +# will return all keys which appear among the 75 most frequently used keys containing +# the substring "name" in all of the following regions: Africa, Asia, Europe, North America, South America. +# - `python tag_usage_stats.py -t value -k office -n 100 -r europe/germany,europe/france` +# will return the values of office=* which are in the list of 100 most freuqently used values in Germany and France. + +import argparse +import enum +import math +import requests +import sys +import urllib.parse + +N = 100 +key = "shop" +default_regions = ",".join(["europe", "north-america", "asia", "south-america", "africa"]) +headers = {"User-Agent": "shortbread/tag_usage_stats"} + +class QueryType(enum.Enum): + values = "value" + keys = "key" + + def __str__(self): + return self.value + + +parser = argparse.ArgumentParser(description="""Program to determine a list of values for a given key +where each value is in the list of N most frequently used values +across a list of regions. +""") +parser.add_argument("-k", "--key", type=str, default=key, help="Set OSM key (defaults to 'shop')") +parser.add_argument("-n", "--number", type=int, default=N, help="Change N (defaults to 100)") +parser.add_argument("-t", "--type", type=QueryType, choices=list(QueryType), default=QueryType.values, help="Whether to query the N most frequently used values of a given key or the N most frequently used keys matching a given search string.") +parser.add_argument("-r", "--regions", type=str, default=default_regions, help="Regions to compare. Specify as one string separated by comma. This must match the paths used at taginfo.geofabrik.de") +args = parser.parse_args() +regions = args.regions.split(",") +if len(regions) == 0: + sys.stderr.write("ERROR: No regions provided.\n") + +globalitem = {} + +for i, region in enumerate(regions): + url = "https://taginfo.geofabrik.de/{}/api/4/key/values".format(urllib.parse.quote(region)) + params = {"sortname": "count_all", "sortorder": "desc", "rp": str(args.number), "page": "1"} + if args.type == QueryType.keys: + url = "https://taginfo.geofabrik.de/{}/api/4/keys/all".format(urllib.parse.quote(region)) + params["query"] = args.key + elif args.type == QueryType.values: + params["key"] = args.key + r = requests.get(url, headers=headers, params=params) + if r.status_code != 200: + sys.stderr.write("ERROR: {url} HTTP {status_code}\n".format(**r.__dict__)) + sys.exit(1) + for item in r.json()["data"]: + entry_key = item[args.type.value] + entry = globalitem.get(entry_key, [0 for j in range(len(regions))]) + attr = "count" + if args.type == QueryType.keys: + attr = "count_all" + entry[i] = item[attr] + globalitem[entry_key] = entry + +top_entries = [] +for item, counts in globalitem.items(): + if 0 not in counts: + top_entries.append((item, sum(counts))) +top_entries.sort(key=lambda k: k[1], reverse=True) +max_keylength = max([len(k[0]) for k in top_entries]) +max_valuelength = int(max([math.log(k[1], 10) for k in top_entries])) + 1 +for t in top_entries: + print("{} {}".format(t[0].ljust(max_keylength, " "), str(t[1]).rjust(max_valuelength, " ")))