diff --git a/data/functions.sql b/data/functions.sql index 2f46d0295..90d90ae00 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -83,24 +83,33 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; --- mz_get_rel_network returns a network tag, or NULL, for a --- given way ID. +-- mz_get_rel_networks returns a list of triples of route type, +-- network and ref tags, or NULL, for a given way ID. -- -- it does this by joining onto the relations slim table, so it -- won't work if you dropped the slim tables, or didn't use slim -- mode in osm2pgsql. -- -CREATE OR REPLACE FUNCTION mz_get_rel_network( +CREATE OR REPLACE FUNCTION mz_get_rel_networks( way_id bigint) -RETURNS text AS $$ -BEGIN - RETURN mz_first_dedup(ARRAY( - SELECT mz_rel_get_tag(tags, 'network') - FROM planet_osm_rels - WHERE parts && ARRAY[way_id] - AND parts[way_off+1:rel_off] && ARRAY[way_id])); -END; -$$ LANGUAGE plpgsql STABLE; +RETURNS text[] AS $$ +SELECT + array_agg(unnested) +FROM ( + SELECT + unnest(tags) AS unnested + FROM ( + SELECT + hstore(tags)->ARRAY['route','network','ref'] AS tags + FROM + planet_osm_rels + WHERE + parts && ARRAY[way_id] AND + parts[way_off+1:rel_off] && ARRAY[way_id] AND + hstore(tags) ?& ARRAY['route','network','ref'] + ) inner1 + ) inner2; +$$ LANGUAGE sql STABLE; -- adds the prefix onto every key in an hstore value CREATE OR REPLACE FUNCTION mz_hstore_add_prefix( diff --git a/data/migrations/v1.0.0-cleanup.sql b/data/migrations/v1.0.0-cleanup.sql new file mode 100644 index 000000000..3b1df1f4a --- /dev/null +++ b/data/migrations/v1.0.0-cleanup.sql @@ -0,0 +1,2 @@ +-- drop no longer used single-network version of this function +DROP FUNCTION mz_get_rel_network(bigint); diff --git a/docs/layers.md b/docs/layers.md index 8eab3e34d..d3de12ce3 100644 --- a/docs/layers.md +++ b/docs/layers.md @@ -837,13 +837,14 @@ To improve performance, some road segments are merged at low and mid-zooms. To f * `source`: `openstreetmap` or `naturalearthdata.com` * `kind`: one of High Road's values for `highway`, `major_road`, `minor_road`, `rail`, `path`, `ferry`, `piste`, `aerialway`, `aeroway`, `racetrack`, `portage_way` if `whitewater=portage_way`; or Natural Earth's `featurecla` value. You'll want to look at other tags like `highway` and `railway` for raw OpenStreetMap values. At low zooms, Natural Earth `featurecla` kinds of `Road` and `Ferry` are used. Look to `type` for more fidelity. * `landuse_kind`: See description above, values match values in the `landuse` layer. -* `ref`: Used for road shields. Related, see `symbol` for pistes. +* `ref`: Commonly-used reference for roads, for example "I 90" for Interstate 90. To use with shields, see the common optional properties `network` and `shield_text`. Related, see `symbol` for pistes. * `sort_key`: a suggestion for which order to draw features. The value is an integer where smaller numbers suggest that features should be "behind" features with larger numbers. At zooms >= 15, the `sort_key` is adjusted to realistically model bridge, tunnel, and layer ordering. #### Road properties (common optional): * `aerialway`: See kind list below. * `aeroway`: See kind list below. +* `all_networks` and `all_shield_texts`: All the networks of which this road is a part, and all of the shield texts. See `network` and `shield_text` below. **Note** that these properties will not be present on MVT format tiles, as we cannot currently encode lists as values. * `bicycle_network`: Present if the feature is part of a cycling network. If so, the value will be one of `icn` for International Cycling Network, `ncn` for National Cycling Network, `rcn` for Regional Cycling Network, `lcn` for Local Cycling Network. * `cycleway`: `cycleway` tag from feature. If no `cycleway` tag is present but `cycleway:both` exists, we source from that tag instead. * `cycleway_left`: `cycleway_left` tag from feature @@ -857,13 +858,14 @@ To improve performance, some road segments are merged at low and mid-zooms. To f * `is_tunnel`: `true` if the road is part of a tunnel. The property will not be present if the road is not part of a tunnel. * `leisure`: See kind list below. * `man_made`: See kind list below. -* `network`: eg: `US:I` for the United States Interstate network, useful for shields and road selections. +* `network`: eg: `US:I` for the United States Interstate network, useful for shields and road selections. This only contains _road_ network types. Please see `bicycle_network` and `walking_network` for bicycle and walking networks, respectively. * `oneway_bicycle`: `oneway:bicycle` tag from feature * `oneway`: `yes` or `no` * `piste_type`: See kind list below. * `railway`: the original OSM railway tag value * `segregated`: Set to `true` when a path allows both pedestrian and bicycle traffic, but when pedestrian traffic is segregated from bicycle traffic. * `service`: See value list below, provided for `railway` and `highway=service` roads. +* `shield_text`: Contains text to display on a shield. For example, I 90 would have a `network` of `US:I` and a `shield_text` of `90`. The `ref`, `I 90`, is less useful for shield display without further processing. * `type`: Natural Earth roads and ferry * `walking_network`: Present if the feature is part of a hiking network. If so, the value will be one of `iwn` for International Walking Network, `nwn` for National Walking Network, `rwn` for Regional Walking Network, `lwn` for Local Walking Network. * `kind_detail`: normalized values describing the kind value, see below. diff --git a/integration-test/192-shield-text-ref.py b/integration-test/192-shield-text-ref.py new file mode 100644 index 000000000..36211f455 --- /dev/null +++ b/integration-test/192-shield-text-ref.py @@ -0,0 +1,21 @@ +# US 101, "James Lick Freeway" +# http://www.openstreetmap.org/way/27183379 +# http://www.openstreetmap.org/relation/108619 +assert_has_feature( + 16, 10484, 25334, 'roads', + { 'kind': 'highway', 'network': 'US:US', 'id': 27183379, + 'shield_text': '101' }) + +# I-77, I-81, US-11 & US-52 all in one road West Virginia. +# +# http://www.openstreetmap.org/way/51388984 +# http://www.openstreetmap.org/relation/2309416 +# http://www.openstreetmap.org/relation/2301037 +# http://www.openstreetmap.org/relation/2297359 +# http://www.openstreetmap.org/relation/1027748 +assert_has_feature( + 16, 18022, 25522, 'roads', + { 'kind': 'highway', 'network': 'US:I', 'id': 51388984, + 'shield_text': '77', + 'all_networks': ['US:I', 'US:I', 'US:US', 'US:US'], + 'all_shield_texts': ['77', '81', '11', '52'] }) diff --git a/integration-test/358-merge-same-roads.py b/integration-test/358-merge-same-roads.py index cde36249e..e5eb3123e 100644 --- a/integration-test/358-merge-same-roads.py +++ b/integration-test/358-merge-same-roads.py @@ -8,11 +8,21 @@ # RAW QUERY: way(36.563,-122.377,37.732,-120.844)[highway=primary];>; # RAW QUERY: way(36.563,-122.377,37.732,-120.844)[highway=trunk];>; # + +def _freeze(thing): + if isinstance(thing, dict): + return frozenset([(_freeze(k), _freeze(v)) for k, v in thing.items()]) + + elif isinstance(thing, list): + return tuple([_freeze(i) for i in thing]) + + return thing + with features_in_tile_layer(8, 41, 99, 'roads') as roads: features = set() for road in roads: - props = frozenset(road['properties'].items()) + props = frozenset(_freeze(road['properties'])) if props in features: raise Exception("Duplicate properties %r in roads layer, but " "properties should be unique." diff --git a/integration-test/647-cycle-route.py b/integration-test/647-cycle-route.py index 4b81d139f..cc8fb7f31 100644 --- a/integration-test/647-cycle-route.py +++ b/integration-test/647-cycle-route.py @@ -3,17 +3,17 @@ # http://www.openstreetmap.org/relation/32386 assert_has_feature( 16, 10487, 25327, 'roads', - { 'kind': 'major_road', 'cycleway': 'lane', 'network': 'lcn', 'bicycle_network': 'lcn' }) + { 'kind': 'major_road', 'cycleway': 'lane', 'bicycle_network': 'lcn' }) # Way: King Street (8920394) http://www.openstreetmap.org/way/8920394 assert_has_feature( 16, 10487, 25329, 'roads', - { 'kind': 'major_road', 'cycleway_left': 'lane', 'network': 'lcn', 'bicycle_network': 'lcn'}) + { 'kind': 'major_road', 'cycleway_left': 'lane', 'bicycle_network': 'lcn'}) # Way: King Street (397270776) http://www.openstreetmap.org/way/397270776 assert_has_feature( 16, 10487, 25329, 'roads', - { 'kind': 'major_road', 'cycleway_right': 'lane', 'network': 'lcn', 'bicycle_network': 'lcn'}) + { 'kind': 'major_road', 'cycleway_right': 'lane', 'bicycle_network': 'lcn'}) # Way: Clara-Immerwahr-Straße (287167007) http://www.openstreetmap.org/way/287167007 assert_has_feature( diff --git a/queries.yaml b/queries.yaml index 19e6ac0c3..9a2149e5c 100644 --- a/queries.yaml +++ b/queries.yaml @@ -89,6 +89,7 @@ layers: - vectordatasource.transform.normalize_aerialways - vectordatasource.transform.normalize_cycleway - vectordatasource.transform.add_is_bicycle_related + - vectordatasource.transform.choose_most_important_network - vectordatasource.transform.road_trim_properties - vectordatasource.transform.remove_feature_id - vectordatasource.transform.tags_remove @@ -567,7 +568,7 @@ post_process: source_layer: roads start_zoom: 0 end_zoom: 14 - properties: [name, ref, network] + properties: [name, ref, network, shield_text] where: >- (kind == 'rail' and zoom < 15) or (kind == 'minor_road' and zoom < 14) or @@ -580,7 +581,7 @@ post_process: source_layer: roads start_zoom: 7 end_zoom: 10 - properties: [name, network] + properties: [name, network, shield_text] where: >- kind == 'major_road' # this is a patch to get rid of name, but keep ref & network, for highways diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 04148dbea..714f29390 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -2764,6 +2764,34 @@ def add_uic_ref(shape, properties, fid, zoom): return shape, properties, fid +def _freeze(thing): + """ + Freezes something to a hashable item. + """ + + if isinstance(thing, dict): + return frozenset([(_freeze(k), _freeze(v)) for k, v in thing.items()]) + + elif isinstance(thing, list): + return tuple([_freeze(i) for i in thing]) + + return thing + + +def _thaw(thing): + """ + Reverse of the freeze operation. + """ + + if isinstance(thing, frozenset): + return dict([_thaw(i) for i in thing]) + + elif isinstance(thing, tuple): + return list([_thaw(i) for i in thing]) + + return thing + + def merge_features(ctx): """ Merge (linear) features with the same properties together, attempting to @@ -2814,7 +2842,7 @@ def merge_features(ctx): # because dicts are mutable and therefore not hashable, we have to # transform their items into a frozenset instead. - frozen_props = frozenset(props.items()) + frozen_props = _freeze(props) if frozen_props in features_by_property: features_by_property[frozen_props][2].append(shape) @@ -2833,7 +2861,7 @@ def merge_features(ctx): multi = MultiLineString(list_of_linestrings) # thaw the frozen properties to use in the new feature. - props = dict(frozen_props) + props = _thaw(frozen_props) # restore any 'id' property. if p_id is not None: @@ -3676,3 +3704,69 @@ def normalize_operator_values(shape, properties, fid, zoom): return (shape, properties, fid) return (shape, properties, fid) + + +def network_importance(route_type, network, ref): + """ + Returns an integer representing the numeric importance of the network, + where lower numbers are more important. + + This is to handle roads which are part of many networks, and ensuring + that the most important one is displayed. For example, in the USA many + roads can be part of both interstate (US:I) and "US" (US:US) highways, + and possibly state ones as well (e.g: US:NY:xxx). In addition, there + are international conventions around the use of "CC:national" and + "CC:regional:*" where "CC" is an ISO 2-letter country code. + + Here we treat national-level roads as more important than regional or + lower, and assume that the deeper the network is in the hierarchy, the + less important the road. Roads with lower "ref" numbers are considered + more important than higher "ref" numbers, if they are part of the same + network. + """ + + if network == 'US:I' or ':national' in network: + network_code = 1 + elif network == 'US:US' or ':regional' in network: + network_code = 2 + else: + network_code = len(network.split(':')) + 3 + + try: + ref = max(int(ref), 0) + except ValueError: + ref = 0 + + return network_code * 10000 + min(ref, 9999) + + +def choose_most_important_network(shape, properties, fid, zoom): + """ + Use the `network_importance` function to select any road networks from + `mz_networks` and take the most important one. + """ + + networks = properties.pop('mz_networks', None) + + if networks is not None: + # take the list and make triples out of it + itr = iter(networks) + triples = zip(itr, itr, itr) + triples = [t for t in triples if t[0] == 'road'] + + if len(triples) > 0: + def network_key(t): + return network_importance(*t) + + networks = sorted(triples, key=network_key) + + # expose first network as network/shield_text + route_type, network, ref = networks[0] + properties['network'] = network + properties['shield_text'] = ref + + # expose all networks as well. + properties['all_networks'] = [n[1] for n in networks] + properties['all_shield_texts'] = [n[2] for n in networks] + + return (shape, properties, fid) diff --git a/yaml/roads.yaml b/yaml/roads.yaml index 1715b9ffa..bd2ac8e39 100644 --- a/yaml/roads.yaml +++ b/yaml/roads.yaml @@ -51,7 +51,7 @@ global: CASE WHEN tags->'crossing' <> 'no' THEN tags->'crossing' END sidewalk: {col: tags->sidewalk} - &osm_network_from_relation - network: {expr: "mz_get_rel_network(osm_id)"} + mz_networks: {expr: "mz_get_rel_networks(osm_id)"} - &osm_network_from_tags network: {col: tags->network} - &osm_piste_properties