From 19294258cc170f6e148504a7d8e26ec2031798ed Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 07:51:39 +0200 Subject: [PATCH 01/60] Move bounding box calculation into nigiri --- .pkg | 2 +- include/nigiri/loader/gtfs/shape_prepare.h | 2 + include/nigiri/shape.h | 6 ++ src/loader/gtfs/load_timetable.cc | 5 ++ src/loader/gtfs/shape_prepare.cc | 81 ++++++++++++++++++++++ src/shape.cc | 24 ++++++- test/shape_test.cc | 30 +++++++- 7 files changed, 146 insertions(+), 4 deletions(-) diff --git a/.pkg b/.pkg index ee8a8bb5..9c0e28ea 100644 --- a/.pkg +++ b/.pkg @@ -9,7 +9,7 @@ [geo] url=git@github.com:motis-project/geo.git branch=master - commit=0a14addf42e91b267906a156c9c2564935c03eaf + commit=463c4f97dde00ddbf0f3f8fafd93559a2027d61e [utl] url=git@github.com:motis-project/utl.git branch=master diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index cab62590..ea12b171 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -12,4 +12,6 @@ void calculate_shape_offsets(timetable const&, vector_map const&, shape_loader_state const&); +void calculate_shape_boxes(timetable const&, shapes_storage&); + } // namespace nigiri::loader::gtfs diff --git a/include/nigiri/shape.h b/include/nigiri/shape.h index f69b3394..e9fe5448 100644 --- a/include/nigiri/shape.h +++ b/include/nigiri/shape.h @@ -5,6 +5,7 @@ #include "cista/containers/pair.h" +#include "geo/box.h" #include "geo/latlng.h" #include "nigiri/types.h" @@ -22,11 +23,16 @@ struct shapes_storage { shape_offset_idx_t add_offsets(std::vector const&); void add_trip_shape_offsets( trip_idx_t, cista::pair const&); + geo::box get_bounding_box(route_idx_t) const; + geo::box get_bounding_box_or_else(route_idx_t, + std::size_t, + std::function const&) const; mm_vecvec data_; mm_vecvec offsets_; mm_vec_map> trip_offset_indices_; + mm_vecvec boxes_; }; } // namespace nigiri \ No newline at end of file diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 42c1339e..cac66b2f 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -367,6 +367,11 @@ void load_timetable(loader_config const& config, progress_tracker->increment(); } + // Build bounding boxes + if (shapes_data != nullptr) { + calculate_shape_boxes(tt, *shapes_data); + } + // Build location_routes map for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) { tt.location_routes_.emplace_back(location_routes[location_idx_t{l}]); diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 104dfc3b..2dd3ef92 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -12,6 +12,8 @@ #include "utl/get_or_create.h" #include "utl/progress_tracker.h" +#include "nigiri/rt/frun.h" +#include "nigiri/shape.h" #include "nigiri/stop.h" #include "nigiri/types.h" @@ -132,4 +134,83 @@ void calculate_shape_offsets(timetable const& tt, } } +void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { + auto shape_segment_boxes = + hash_map, + std::vector>{}; + // Create bounding boxes for all shape segments + for (auto const key : shapes_data.trip_offset_indices_) { + if (key.first == shape_idx_t::invalid() || + key.second == shape_offset_idx_t::invalid()) { + continue; + } + utl::get_or_create(shape_segment_boxes, key, [&]() { + auto const shape = shapes_data.get_shape(key.first); + auto const& offsets = shapes_data.offsets_[key.second]; + auto segment_boxes = std::vector(offsets.size() - 1); + for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { + auto& box = segment_boxes[i]; + auto const& [from, to] = pair; + for (auto const point : + shape.subspan(cista::to_idx(from), + cista::to_idx(to) - cista::to_idx(from) + 1)) { + box.extend(point); + } + } + return segment_boxes; + }); + } + // Create bounding boxes for all routes + for (auto const [i, claszes] : utl::enumerate(tt.route_section_clasz_)) { + auto const r = route_idx_t{i}; + auto const seq = tt.route_location_seq_[r]; + assert(seq.size() > 0U); + auto segment_boxes = std::vector(seq.size()); + auto nontrivial = 0U; + // 0: bounding box for trip, 1-N: bounding box for segment + auto& bounding_box = segment_boxes[0U]; + auto const stop_indices = + interval{stop_idx_t{0U}, static_cast(seq.size())}; + for (auto const transport_idx : tt.route_transport_ranges_[r]) { + auto const frun = rt::frun{tt, nullptr, + rt::run{.t_ = transport{transport_idx}, + .stop_range_ = stop_indices, + .rt_ = rt_transport_idx_t::invalid()}}; + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const absolute_range) { + auto shape_boxes = static_cast const*>(nullptr); + auto it = shape_segment_boxes.find( + shapes_data.trip_offset_indices_[trip_idx]); + if (it != end(shape_segment_boxes)) { + shape_boxes = &it->second; + } + auto prev_pos = tt.locations_.coordinates_.at( + stop{seq[absolute_range.from_]}.location_idx()); + bounding_box.extend(prev_pos); + for (auto const [from, to] : utl::pairwise(absolute_range)) { + auto const next_pos = + tt.locations_.coordinates_.at(stop{seq[to]}.location_idx()); + auto& box = segment_boxes[cista::to_idx(to)]; + bounding_box.extend(next_pos); + box.extend(prev_pos); + box.extend(next_pos); + if (shape_boxes != nullptr) { + auto const& shape_box = (*shape_boxes)[static_cast( + cista::to_idx(from) - cista::to_idx(absolute_range.from_))]; + if (!box.contains(shape_box)) { + bounding_box.extend(shape_box); + box.extend(shape_box); + nontrivial = std::max(nontrivial, from + 1U); + } + } + prev_pos = next_pos; + } + }); + } + // 0: bounding box for trip, 1-N: bounding box for segment + segment_boxes.resize(nontrivial + 1); + shapes_data.boxes_.emplace_back(segment_boxes); + } +} + } // namespace nigiri::loader::gtfs \ No newline at end of file diff --git a/src/shape.cc b/src/shape.cc index b0285d5b..cf44c20d 100644 --- a/src/shape.cc +++ b/src/shape.cc @@ -4,6 +4,8 @@ #include "fmt/core.h" +#include "utl/verify.h" + #include "nigiri/types.h" namespace nigiri { @@ -49,7 +51,8 @@ shapes_storage::shapes_storage(std::filesystem::path const& path, create_storage_vector, trip_idx_t>( fmt::format("{}_offset_indices.bin", path.generic_string()), - mode)} {} + mode)}, + boxes_{create_storage(path, "boxes", mode)} {} std::span shapes_storage::get_shape( shape_idx_t const shape_idx) const { @@ -92,4 +95,23 @@ void shapes_storage::add_trip_shape_offsets( trip_offset_indices_.emplace_back(offset_idx); } +geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { + utl::verify(route_idx < boxes_.size(), "Route index {} is out of bounds", + route_idx); + // 0: bounding box for trip + return boxes_[route_idx][0]; +} + +geo::box shapes_storage::get_bounding_box_or_else( + nigiri::route_idx_t const route_idx, + std::size_t const segment, + std::function const& callback) const { + + utl::verify(route_idx < boxes_.size(), "Route index {} is out of bounds", + route_idx); + auto const& boxes = boxes_[route_idx]; + // 1-N: bounding box for segment + return segment + 1 < boxes.size() ? boxes[segment + 1] : callback(); +} + } // namespace nigiri \ No newline at end of file diff --git a/test/shape_test.cc b/test/shape_test.cc index 6bc1ed42..94e98887 100644 --- a/test/shape_test.cc +++ b/test/shape_test.cc @@ -1,5 +1,6 @@ #include "gtest/gtest.h" +#include "geo/box.h" #include "geo/polyline.h" #include "nigiri/loader/gtfs/load_timetable.h" @@ -76,6 +77,7 @@ Last,5.5,2.5,2 Last,5.5,3.0,3 Last,6.0,3.0,5 Last,5.0,2.0,8 +Last,4.0,1.9,11 Last,4.0,2.0,13 # stop_times.txt @@ -125,8 +127,8 @@ TEST(shape, single_trip_with_shape) { auto const shape_by_shape_idx = shapes_data.get_shape(shape_idx_t{3}); auto const expected_shape = geo::polyline{ - {4.0f, 5.0f}, {5.5f, 2.5f}, {5.5f, 3.0f}, - {6.0f, 3.0f}, {5.0f, 2.0f}, {4.0f, 2.0f}, + {4.0, 5.0}, {5.5, 2.5}, {5.5, 3.0}, {6.0, 3.0}, + {5.0, 2.0}, {4.0, 1.9}, {4.0, 2.0}, }; EXPECT_EQ(expected_shape, shape_by_trip_idx); EXPECT_EQ(expected_shape, shape_by_shape_idx); @@ -157,4 +159,28 @@ TEST(shape, single_trip_with_shape) { EXPECT_TRUE(shape_by_invalid_trip_idx.empty()); EXPECT_TRUE(shape_by_invalid_shape_idx.empty()); } + + // Testing bounding boxes + { + // Full shape in bounding box included + EXPECT_EQ((geo::make_box({{0.0, 2.0}, {1.0, 4.0}})), + shapes_data.get_bounding_box(route_idx_t{0U})); + // Shape in bounding box included + EXPECT_EQ((geo::make_box({{4.0, 1.9}, {6.0, 5.0}})), + shapes_data.get_bounding_box(route_idx_t{2U})); + // Bounding boxes for segments + auto const kTestBox = geo::make_box({{90.0, 45.0}}); + auto const fallback = [&]() { return kTestBox; }; + // Bounding box extended by shape + EXPECT_EQ( + (geo::make_box({{5.0, 2.0}, {6.0, 3.0}})), + shapes_data.get_bounding_box_or_else(route_idx_t{2}, 2, fallback)); + EXPECT_EQ( + (geo::make_box({{4.0, 1.9}, {5.0, 2.0}})), + shapes_data.get_bounding_box_or_else(route_idx_t{2}, 3, fallback)); + + // Shape contained in bounding box + EXPECT_EQ(kTestBox, shapes_data.get_bounding_box_or_else(route_idx_t{2}, 4, + fallback)); + } } From 21c6bdea0273d1acecf3260ffad64f3f5a5f69c5 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 08:37:29 +0200 Subject: [PATCH 02/60] Make bounding boxes for segments optional Clients are now required to provide a fallback when needed. --- include/nigiri/shape.h | 5 ++--- src/loader/gtfs/shape_prepare.cc | 6 +++--- src/shape.cc | 9 ++++----- test/shape_test.cc | 26 +++++++++++++++----------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/include/nigiri/shape.h b/include/nigiri/shape.h index e9fe5448..6dc43965 100644 --- a/include/nigiri/shape.h +++ b/include/nigiri/shape.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "cista/containers/pair.h" @@ -24,9 +25,7 @@ struct shapes_storage { void add_trip_shape_offsets( trip_idx_t, cista::pair const&); geo::box get_bounding_box(route_idx_t) const; - geo::box get_bounding_box_or_else(route_idx_t, - std::size_t, - std::function const&) const; + std::optional get_bounding_box(route_idx_t, std::size_t) const; mm_vecvec data_; mm_vecvec offsets_; diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 2dd3ef92..e53c6d26 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -166,7 +166,7 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto segment_boxes = std::vector(seq.size()); - auto nontrivial = 0U; + auto last_extend = 0U; // 0: bounding box for trip, 1-N: bounding box for segment auto& bounding_box = segment_boxes[0U]; auto const stop_indices = @@ -200,7 +200,7 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { if (!box.contains(shape_box)) { bounding_box.extend(shape_box); box.extend(shape_box); - nontrivial = std::max(nontrivial, from + 1U); + last_extend = std::max(last_extend, from + 1U); } } prev_pos = next_pos; @@ -208,7 +208,7 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { }); } // 0: bounding box for trip, 1-N: bounding box for segment - segment_boxes.resize(nontrivial + 1); + segment_boxes.resize(last_extend + 1); shapes_data.boxes_.emplace_back(segment_boxes); } } diff --git a/src/shape.cc b/src/shape.cc index cf44c20d..309674be 100644 --- a/src/shape.cc +++ b/src/shape.cc @@ -102,16 +102,15 @@ geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { return boxes_[route_idx][0]; } -geo::box shapes_storage::get_bounding_box_or_else( - nigiri::route_idx_t const route_idx, - std::size_t const segment, - std::function const& callback) const { +std::optional shapes_storage::get_bounding_box( + nigiri::route_idx_t const route_idx, std::size_t const segment) const { utl::verify(route_idx < boxes_.size(), "Route index {} is out of bounds", route_idx); auto const& boxes = boxes_[route_idx]; // 1-N: bounding box for segment - return segment + 1 < boxes.size() ? boxes[segment + 1] : callback(); + return segment + 1 < boxes.size() ? boxes[segment + 1] + : std::optional{}; } } // namespace nigiri \ No newline at end of file diff --git a/test/shape_test.cc b/test/shape_test.cc index 94e98887..d3911d3c 100644 --- a/test/shape_test.cc +++ b/test/shape_test.cc @@ -169,18 +169,22 @@ TEST(shape, single_trip_with_shape) { EXPECT_EQ((geo::make_box({{4.0, 1.9}, {6.0, 5.0}})), shapes_data.get_bounding_box(route_idx_t{2U})); // Bounding boxes for segments - auto const kTestBox = geo::make_box({{90.0, 45.0}}); - auto const fallback = [&]() { return kTestBox; }; // Bounding box extended by shape - EXPECT_EQ( - (geo::make_box({{5.0, 2.0}, {6.0, 3.0}})), - shapes_data.get_bounding_box_or_else(route_idx_t{2}, 2, fallback)); - EXPECT_EQ( - (geo::make_box({{4.0, 1.9}, {5.0, 2.0}})), - shapes_data.get_bounding_box_or_else(route_idx_t{2}, 3, fallback)); - + { + auto const extended_by_shape = + shapes_data.get_bounding_box(route_idx_t{2}, 3); + ASSERT_TRUE(extended_by_shape.has_value()); + EXPECT_EQ((geo::make_box({{4.0, 1.9}, {5.0, 2.0}})), *extended_by_shape); + + auto const before_last_extend = + shapes_data.get_bounding_box(route_idx_t{2}, 2); + ASSERT_TRUE(before_last_extend.has_value()); + EXPECT_EQ((geo::make_box({{5.0, 2.0}, {6.0, 3.0}})), *before_last_extend); + } // Shape contained in bounding box - EXPECT_EQ(kTestBox, shapes_data.get_bounding_box_or_else(route_idx_t{2}, 4, - fallback)); + { + + EXPECT_FALSE(shapes_data.get_bounding_box(route_idx_t{2}, 4).has_value()); + } } } From ceee6db893f533e82cf950504f3080bd37f277c0 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:07:40 +0200 Subject: [PATCH 03/60] Fix duplicated entries --- src/loader/gtfs/shape_prepare.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index e53c6d26..8624f870 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -161,8 +161,11 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { }); } // Create bounding boxes for all routes - for (auto const [i, claszes] : utl::enumerate(tt.route_section_clasz_)) { - auto const r = route_idx_t{i}; + for (auto const r : tt.transport_route_) { + // Skip routes added with previous timetables + if (r < shapes_data.boxes_.size()) { + continue; + } auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto segment_boxes = std::vector(seq.size()); From 67b6bac27dfcbd20eaa56b7819b4d880dded9748 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:21:48 +0200 Subject: [PATCH 04/60] Make filtering more explicit --- src/loader/gtfs/shape_prepare.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 8624f870..0bd43cd3 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -160,12 +160,11 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { return segment_boxes; }); } - // Create bounding boxes for all routes - for (auto const r : tt.transport_route_) { - // Skip routes added with previous timetables - if (r < shapes_data.boxes_.size()) { - continue; - } + // Create bounding boxes for all routes not already added + for (auto const r : tt.transport_route_ | + std::views::filter([&](route_idx_t const route_idx) { + return route_idx >= shapes_data.boxes_.size(); + })) { auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto segment_boxes = std::vector(seq.size()); From 3fe690515e0818d8f6b58b3a2e201489520122bb Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:18:14 +0200 Subject: [PATCH 05/60] Delay value calculation until needed This avoids duplicated and unused entries for repeated calls. --- src/loader/gtfs/shape_prepare.cc | 57 ++++++++++++++------------------ test/shape_test.cc | 1 - 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 0bd43cd3..041dd5ea 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -135,32 +135,9 @@ void calculate_shape_offsets(timetable const& tt, } void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { - auto shape_segment_boxes = + auto cached_shape_boxes = hash_map, std::vector>{}; - // Create bounding boxes for all shape segments - for (auto const key : shapes_data.trip_offset_indices_) { - if (key.first == shape_idx_t::invalid() || - key.second == shape_offset_idx_t::invalid()) { - continue; - } - utl::get_or_create(shape_segment_boxes, key, [&]() { - auto const shape = shapes_data.get_shape(key.first); - auto const& offsets = shapes_data.offsets_[key.second]; - auto segment_boxes = std::vector(offsets.size() - 1); - for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { - auto& box = segment_boxes[i]; - auto const& [from, to] = pair; - for (auto const point : - shape.subspan(cista::to_idx(from), - cista::to_idx(to) - cista::to_idx(from) + 1)) { - box.extend(point); - } - } - return segment_boxes; - }); - } - // Create bounding boxes for all routes not already added for (auto const r : tt.transport_route_ | std::views::filter([&](route_idx_t const route_idx) { return route_idx >= shapes_data.boxes_.size(); @@ -180,12 +157,28 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { .rt_ = rt_transport_idx_t::invalid()}}; frun.for_each_trip([&](trip_idx_t const trip_idx, interval const absolute_range) { - auto shape_boxes = static_cast const*>(nullptr); - auto it = shape_segment_boxes.find( - shapes_data.trip_offset_indices_[trip_idx]); - if (it != end(shape_segment_boxes)) { - shape_boxes = &it->second; - } + auto const key = shapes_data.trip_offset_indices_[trip_idx]; + auto const& shape_boxes = + utl::get_or_create(cached_shape_boxes, key, [&]() { + if (key.first == shape_idx_t::invalid() || + key.second == shape_offset_idx_t::invalid()) { + return std::vector{}; + } + auto const shape = shapes_data.get_shape(key.first); + auto const& offsets = shapes_data.offsets_[key.second]; + auto boxes = std::vector(offsets.size() - 1); + for (auto const [i, pair] : + utl::enumerate(utl::pairwise(offsets))) { + auto& box = boxes[i]; + auto const& [from, to] = pair; + for (auto const point : shape.subspan( + cista::to_idx(from), + cista::to_idx(to) - cista::to_idx(from) + 1)) { + box.extend(point); + } + } + return boxes; + }); auto prev_pos = tt.locations_.coordinates_.at( stop{seq[absolute_range.from_]}.location_idx()); bounding_box.extend(prev_pos); @@ -196,8 +189,8 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { bounding_box.extend(next_pos); box.extend(prev_pos); box.extend(next_pos); - if (shape_boxes != nullptr) { - auto const& shape_box = (*shape_boxes)[static_cast( + if (!shape_boxes.empty()) { + auto const& shape_box = shape_boxes[static_cast( cista::to_idx(from) - cista::to_idx(absolute_range.from_))]; if (!box.contains(shape_box)) { bounding_box.extend(shape_box); diff --git a/test/shape_test.cc b/test/shape_test.cc index d3911d3c..1d673078 100644 --- a/test/shape_test.cc +++ b/test/shape_test.cc @@ -183,7 +183,6 @@ TEST(shape, single_trip_with_shape) { } // Shape contained in bounding box { - EXPECT_FALSE(shapes_data.get_bounding_box(route_idx_t{2}, 4).has_value()); } } From a70323c5249bbe7264c17ea15edfb90f6f10ddb1 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:14:58 +0200 Subject: [PATCH 06/60] Simplify route iteration --- src/loader/gtfs/shape_prepare.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 041dd5ea..fddbc6ca 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -138,10 +138,10 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { auto cached_shape_boxes = hash_map, std::vector>{}; - for (auto const r : tt.transport_route_ | - std::views::filter([&](route_idx_t const route_idx) { - return route_idx >= shapes_data.boxes_.size(); - })) { + auto const new_routes = + interval{static_cast(shapes_data.boxes_.size()), + static_cast(tt.route_transport_ranges_.size())}; + for (auto const r : new_routes) { auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto segment_boxes = std::vector(seq.size()); From 1aa404a581436d39a6f11a57a01dbe20ba226eab Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:58:39 +0200 Subject: [PATCH 07/60] Update geo dependency --- .pkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pkg b/.pkg index 9c0e28ea..c64dfd8a 100644 --- a/.pkg +++ b/.pkg @@ -9,7 +9,7 @@ [geo] url=git@github.com:motis-project/geo.git branch=master - commit=463c4f97dde00ddbf0f3f8fafd93559a2027d61e + commit=cee17208dba98f5f3ddaf78d0bfff9c083557ba3 [utl] url=git@github.com:motis-project/utl.git branch=master From 2a9b6bd41aa5bc17ae614d1f2fd71333aa90e8df Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 24 Oct 2024 08:49:07 +0200 Subject: [PATCH 08/60] Fix out of bounds shape offset --- src/loader/gtfs/shape_prepare.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index fddbc6ca..ffdd4f1d 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -68,9 +68,13 @@ std::vector get_offsets_by_dist_traveled( auto offsets = std::vector{}; offsets.reserve(dist_traveled_stops_times.size()); auto remaining_shape_begin = begin(dist_traveled_shape_edges); + // Ensure final offset maps to valid shape point + auto const shape_end = + begin(dist_traveled_shape_edges) + + (end(dist_traveled_shape_edges) - begin(dist_traveled_shape_edges)) - 1; for (auto const& distance : dist_traveled_stops_times) { - remaining_shape_begin = std::lower_bound( - remaining_shape_begin, end(dist_traveled_shape_edges), distance); + remaining_shape_begin = + std::lower_bound(remaining_shape_begin, shape_end, distance); offsets.push_back(shape_offset_t{remaining_shape_begin - begin(dist_traveled_shape_edges)}); } From 121b67d7657498e939e29945149d88c629ef61b2 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 24 Oct 2024 09:31:30 +0200 Subject: [PATCH 09/60] Add test case for invalid offset --- test/rt/frun_shape_test.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc index df116dfc..ed856424 100644 --- a/test/rt/frun_shape_test.cc +++ b/test/rt/frun_shape_test.cc @@ -665,6 +665,22 @@ TEST( }), leg_shape); } + // N -> O + // For O: shape < stop_times => Could be out of bounds + { + leg_shape.clear(); + + full_run.for_each_shape_point( + &shapes_data, interval{stop_idx_t{2U}, stop_idx_t{3U + 1U}}, + plot_point); + + EXPECT_EQ((geo::polyline{ + {3.0F, 3.0F}, + {3.5F, 3.5F}, + {4.0F, 4.0F}, + }), + leg_shape); + } } // Trip with multiple leading 0.0 distances { From c195ca975d27aefd62060c9f069a4217bec0bea4 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:24:06 +0200 Subject: [PATCH 10/60] Improve comment --- test/rt/frun_shape_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc index ed856424..674687bb 100644 --- a/test/rt/frun_shape_test.cc +++ b/test/rt/frun_shape_test.cc @@ -328,7 +328,7 @@ TEST( }), leg_shape); } - // sub trip of a merged trip + // sub trip of a block trip { // Create run transit_realtime::TripDescriptor td; From 6044729431bbdd759446a0fcff3e057188173998 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:29:49 +0100 Subject: [PATCH 11/60] Unify boxes size --- include/nigiri/shapes_storage.h | 2 +- src/shapes_storage.cc | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/nigiri/shapes_storage.h b/include/nigiri/shapes_storage.h index 03c7a261..641bbb6f 100644 --- a/include/nigiri/shapes_storage.h +++ b/include/nigiri/shapes_storage.h @@ -35,7 +35,7 @@ struct shapes_storage { mm_vecvec offsets_; mm_vec_map> trip_offset_indices_; - mm_vecvec boxes_; + mm_vecvec boxes_; }; } // namespace nigiri \ No newline at end of file diff --git a/src/shapes_storage.cc b/src/shapes_storage.cc index ca3fdf7b..1d18f7de 100644 --- a/src/shapes_storage.cc +++ b/src/shapes_storage.cc @@ -2,6 +2,8 @@ #include +#include "cista/strong.h" + #include "fmt/core.h" #include "utl/verify.h" @@ -27,8 +29,7 @@ shapes_storage::shapes_storage(std::filesystem::path path, mm_vec{mm("shape_offsets_idx.bin")}}, trip_offset_indices_{mm("shape_trip_offsets.bin")}, boxes_{mm_vec{mm("shape_boxes_data.bin")}, - mm_vec{mm("shape_boxes_idx.bin")}} - {} + mm_vec{mm("shape_boxes_idx.bin")}} {} cista::mmap shapes_storage::mm(char const* file) { return cista::mmap{(p_ / file).generic_string().c_str(), mode_}; @@ -92,8 +93,8 @@ void shapes_storage::add_trip_shape_offsets( } geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { - utl::verify(route_idx < boxes_.size(), "Route index {} is out of bounds", - route_idx); + utl::verify(cista::to_idx(route_idx) < boxes_.size(), + "Route index {} is out of bounds", route_idx); // 0: bounding box for trip return boxes_[route_idx][0]; } @@ -101,8 +102,8 @@ geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { std::optional shapes_storage::get_bounding_box( nigiri::route_idx_t const route_idx, std::size_t const segment) const { - utl::verify(route_idx < boxes_.size(), "Route index {} is out of bounds", - route_idx); + utl::verify(cista::to_idx(route_idx) < boxes_.size(), + "Route index {} is out of bounds", route_idx); auto const& boxes = boxes_[route_idx]; // 1-N: bounding box for segment return segment + 1 < boxes.size() ? boxes[segment + 1] From 2a3452454dbd49d185d1c789fa0032d28f77f4ea Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:54:47 +0100 Subject: [PATCH 12/60] Store stops per shapes to process --- include/nigiri/loader/gtfs/shape_prepare.h | 9 ++++++++ src/loader/gtfs/shape_prepare.cc | 27 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 302e4a33..321932b4 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -10,6 +10,15 @@ struct timetable; namespace nigiri::loader::gtfs { +// TODO Rename +struct trip_shapes { + std::vector> stop_sequences_; + shape_idx_t index_offset_; +}; + +trip_shapes get_shape_pairs(shape_loader_state const&, + vector_map const&); + void calculate_shape_offsets(timetable const&, shapes_storage&, vector_map const&, diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 69bd9959..d2503fe5 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -1,10 +1,13 @@ #include "nigiri/loader/gtfs/shape_prepare.h" #include +#include #include #include #include +#include "cista/strong.h" + #include "geo/latlng.h" #include "geo/polyline.h" @@ -19,6 +22,30 @@ namespace nigiri::loader::gtfs { +trip_shapes get_shape_pairs(shape_loader_state const& states, + vector_map const& trips) { + auto const min = states.index_offset_; + auto const length = states.id_map_.size(); + // TODO Change name + auto pairs = trip_shapes{ + .stop_sequences_ = std::vector>(length), + .index_offset_ = min, + }; + for (auto const& trip : trips) { + if (trip.shape_idx_ == shape_idx_t::invalid()) { + continue; + } + auto& candidates = + pairs.stop_sequences_[cista::to_idx(trip.shape_idx_ - min)]; + if (std::all_of(std::execution::par_unseq, candidates.begin(), + candidates.end(), + [&](auto const& it) { return *it != trip.stop_seq_; })) { + candidates.emplace_back(&trip.stop_seq_); + } + } + return pairs; +} + std::size_t get_closest(geo::latlng const& pos, std::span shape) { if (shape.size() < 2U) { From 9dfc2ed2e7d3f8a054fed30bfc95b3ff6230f9c1 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:06:01 +0100 Subject: [PATCH 13/60] Move code into struct --- include/nigiri/loader/gtfs/shape_prepare.h | 4 +++- src/loader/gtfs/shape_prepare.cc | 17 ++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 321932b4..bdf36211 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -12,8 +12,10 @@ namespace nigiri::loader::gtfs { // TODO Rename struct trip_shapes { - std::vector> stop_sequences_; + trip_shapes(shape_loader_state const&, + vector_map const&); shape_idx_t index_offset_; + std::vector> stop_sequences_; }; trip_shapes get_shape_pairs(shape_loader_state const&, diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index d2503fe5..9613e364 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -22,28 +22,23 @@ namespace nigiri::loader::gtfs { -trip_shapes get_shape_pairs(shape_loader_state const& states, - vector_map const& trips) { - auto const min = states.index_offset_; - auto const length = states.id_map_.size(); - // TODO Change name - auto pairs = trip_shapes{ - .stop_sequences_ = std::vector>(length), - .index_offset_ = min, - }; +trip_shapes::trip_shapes(shape_loader_state const& states, + vector_map const& trips) + : index_offset_{states.index_offset_}, + stop_sequences_{ + std::vector>(states.id_map_.size())} { for (auto const& trip : trips) { if (trip.shape_idx_ == shape_idx_t::invalid()) { continue; } auto& candidates = - pairs.stop_sequences_[cista::to_idx(trip.shape_idx_ - min)]; + stop_sequences_[cista::to_idx(trip.shape_idx_ - index_offset_)]; if (std::all_of(std::execution::par_unseq, candidates.begin(), candidates.end(), [&](auto const& it) { return *it != trip.stop_seq_; })) { candidates.emplace_back(&trip.stop_seq_); } } - return pairs; } std::size_t get_closest(geo::latlng const& pos, From e8cc9a8e79cda66b8f62b72bee1a6612f8c38bd3 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:35:25 +0100 Subject: [PATCH 14/60] WIP: Perform parallel calculations --- include/nigiri/loader/gtfs/shape_prepare.h | 26 ++- src/loader/gtfs/load_timetable.cc | 6 +- src/loader/gtfs/shape_prepare.cc | 211 +++++++++++++++++++-- 3 files changed, 219 insertions(+), 24 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index bdf36211..39572996 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -1,7 +1,10 @@ #pragma once +#include "geo/box.h" + #include "nigiri/loader/gtfs/shape.h" #include "nigiri/loader/gtfs/trip.h" +#include "nigiri/types.h" namespace nigiri { struct shapes_storage; @@ -10,12 +13,31 @@ struct timetable; namespace nigiri::loader::gtfs { +struct shape_segment { + explicit shape_segment(stop_seq_t const* stop_seq, + std::vector const* distances); + stop_seq_t const* stop_seq_; + shape_offset_idx_t offset_idx_; + std::vector const* distances_; + std::vector boxes_; +}; + +struct shape_segments { + shape_idx_t shape_idx_; + std::vector offsets_; +}; + // TODO Rename struct trip_shapes { trip_shapes(shape_loader_state const&, - vector_map const&); + vector_map const&); + void calculate_shape_offsets(timetable const&, + shapes_storage&, + shape_loader_state const&); + void store_offsets(shapes_storage&, vector_map const&); + void create_boxes(timetable const&, shapes_storage&); shape_idx_t index_offset_; - std::vector> stop_sequences_; + std::vector shape_segments_; }; trip_shapes get_shape_pairs(shape_loader_state const&, diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index cac66b2f..d33accf6 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -261,8 +261,10 @@ void load_timetable(loader_config const& config, {source_file_idx, trp.from_line_, trp.to_line_}, train_nr, stop_seq_numbers); } + auto shape_pairs = trip_shapes{shape_states, trip_data.data_}; if (shapes_data != nullptr) { - calculate_shape_offsets(tt, *shapes_data, trip_data.data_, shape_states); + shape_pairs.calculate_shape_offsets(tt, *shapes_data, shape_states); + shape_pairs.store_offsets(*shapes_data, trip_data.data_); } auto const timer = scoped_timer{"loader.gtfs.routes.build"}; @@ -369,7 +371,7 @@ void load_timetable(loader_config const& config, // Build bounding boxes if (shapes_data != nullptr) { - calculate_shape_boxes(tt, *shapes_data); + shape_pairs.create_boxes(tt, *shapes_data); } // Build location_routes map diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 9613e364..95239e97 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -18,29 +19,9 @@ #include "nigiri/rt/frun.h" #include "nigiri/shapes_storage.h" #include "nigiri/stop.h" -#include "nigiri/types.h" namespace nigiri::loader::gtfs { -trip_shapes::trip_shapes(shape_loader_state const& states, - vector_map const& trips) - : index_offset_{states.index_offset_}, - stop_sequences_{ - std::vector>(states.id_map_.size())} { - for (auto const& trip : trips) { - if (trip.shape_idx_ == shape_idx_t::invalid()) { - continue; - } - auto& candidates = - stop_sequences_[cista::to_idx(trip.shape_idx_ - index_offset_)]; - if (std::all_of(std::execution::par_unseq, candidates.begin(), - candidates.end(), - [&](auto const& it) { return *it != trip.stop_seq_; })) { - candidates.emplace_back(&trip.stop_seq_); - } - } -} - std::size_t get_closest(geo::latlng const& pos, std::span shape) { if (shape.size() < 2U) { @@ -235,4 +216,194 @@ void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { } } +shape_segment::shape_segment(stop_seq_t const* stop_seq, + std::vector const* distances) + : stop_seq_{stop_seq}, + offset_idx_{shape_offset_idx_t::invalid()}, + distances_{distances}, + boxes_{} {} + +trip_shapes::trip_shapes(shape_loader_state const& states, + vector_map const& trips) + : index_offset_{states.index_offset_}, + shape_segments_(states.id_map_.size()) { + for (auto i = 0U; i < shape_segments_.size(); ++i) { + shape_segments_[i].shape_idx_ = static_cast(index_offset_ + i); + } + for (auto const& trip : trips) { + if (trip.shape_idx_ == shape_idx_t::invalid()) { + continue; + } + auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); + auto& candidates = shape_segments_[idx].offsets_; + if (std::all_of(std::execution::par_unseq, candidates.begin(), + candidates.end(), [&](shape_segment const& it) { + return *it.stop_seq_ != trip.stop_seq_; + })) { + auto distances = + trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; + candidates.emplace_back(&trip.stop_seq_, distances); + } + } +} + +void trip_shapes::calculate_shape_offsets(timetable const& tt, + shapes_storage& shapes_data, + shape_loader_state const& states) { + auto m = std::mutex{}; + std::for_each( + std::execution::par_unseq, shape_segments_.begin(), shape_segments_.end(), + [&](shape_segments& segments) { + if (segments.offsets_.empty()) { + return; + } + auto const shape = shapes_data.get_shape(segments.shape_idx_); + auto const& shape_distances = + states + .distances_[cista::to_idx(segments.shape_idx_ - index_offset_)]; + for (auto& segment : segments.offsets_) { + auto const offsets = [&]() { + if (!shape_distances.empty() && segment.distances_ != nullptr) { + return get_offsets_by_dist_traveled(*segment.distances_, + shape_distances); + } + if (shape.size() < segment.stop_seq_->size()) { + return std::vector{}; + } + + return get_offsets_by_stops(tt, shape, *segment.stop_seq_); + }(); + if (!offsets.empty()) { + auto const guard = std::lock_guard{m}; + segment.offset_idx_ = shapes_data.add_offsets(offsets); + } else { + segment.offset_idx_ = shape_offset_idx_t::invalid(); + } + segment.boxes_ = [&]() { + // Store box of full shape at index 0 + auto boxes = + offsets.empty() + ? std::vector(1) + : std::vector(segment.stop_seq_->size() - 1 + 1); + auto& shape_box = boxes.front(); + auto last_extend = 0UL; + if (!offsets.empty()) { + for (auto const [i, pair] : + utl::enumerate(utl::pairwise(offsets))) { + auto& segment_box = boxes[i + 1U]; + auto const& [from, to] = pair; + for (auto const point : shape.subspan( + cista::to_idx(from), + cista::to_idx(to) - cista::to_idx(from) + 1)) { + shape_box.extend(point); + segment_box.extend(point); + } + auto const from_l = + tt.locations_.coordinates_[stop{(*segment.stop_seq_)[i]} + .location_idx()]; + auto const to_l = + tt.locations_.coordinates_[stop{(*segment.stop_seq_)[i + 1]} + .location_idx()]; + auto const stop_box = geo::make_box({from_l, to_l}); + if (!stop_box.contains(segment_box)) { + last_extend = i + 1U; + } + } + } else { + for (auto const s : *segment.stop_seq_) { + shape_box.extend( + tt.locations_.coordinates_[stop{s}.location_idx()]); + } + } + boxes.reserve(last_extend + 1); + return boxes; + }(); + } + }); +} + +void trip_shapes::store_offsets( + shapes_storage& shapes_data, + vector_map const& trips) { + for (auto const& trip : trips) { + auto const trip_idx = trip.trip_idx_; + auto const shape_idx = trip.shape_idx_; + auto const offset_idx = [&]() { + if (shape_idx == shape_idx_t::invalid()) { + return shape_offset_idx_t::invalid(); + } + auto const& segments = + shape_segments_[cista::to_idx(shape_idx - index_offset_)].offsets_; + auto const segment = + std::find_if(std::execution::par_unseq, begin(segments), + end(segments), [&](shape_segment const& s) { + return *s.stop_seq_ == trip.stop_seq_; + }); + return segment->offset_idx_; + }(); + shapes_data.add_trip_shape_offsets(trip_idx, + cista::pair{shape_idx, offset_idx}); + } +} + +void trip_shapes::create_boxes(timetable const& tt, + shapes_storage& shapes_data) { + auto const new_routes = + interval{static_cast(shapes_data.boxes_.size()), + static_cast(tt.route_transport_ranges_.size())}; + auto route_boxes = + std::vector>(cista::to_idx(new_routes.size())); + std::transform( + std::execution::par, begin(new_routes), end(new_routes), + begin(route_boxes), [&](route_idx_t const r) { + auto const seq = tt.route_location_seq_[r]; + assert(seq.size() > 0U); + auto boxes = std::vector(seq.size() - 1 + 1); + auto last_extend = 1UL; + auto& bounding_box = boxes[0U]; + // 0: bounding box for trip, 1-N: bounding box for each segment + auto const stop_indices = + interval{stop_idx_t{0U}, static_cast(seq.size())}; + for (auto const transport_idx : tt.route_transport_ranges_[r]) { + auto const frun = + rt::frun{tt, nullptr, + rt::run{.t_ = transport{transport_idx}, + .stop_range_ = stop_indices, + .rt_ = rt_transport_idx_t::invalid()}}; + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_data.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + for (auto const idx : absolute_range) { + bounding_box.extend( + tt.locations_.coordinates_[stop{seq[idx]}.location_idx()]); + } + } else { + auto const& segments = + shape_segments_[cista::to_idx(shape_idx - index_offset_)] + .offsets_; + auto const& segment = + std::find_if(std::execution::par_unseq, begin(segments), + end(segments), [&](shape_segment const& s) { + return s.offset_idx_ == offset_idx; + }); + bounding_box.extend(segment->boxes_[0]); + for (auto i = 1U; i < segment->boxes_.size(); ++i) { + boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = + segment->boxes_[i]; + } + last_extend = std::max( + last_extend, segment->boxes_.size() + absolute_range.from_); + } + }); + } + boxes.resize(last_extend); + return boxes; + }); + for (auto const& box : route_boxes) { + shapes_data.boxes_.emplace_back(box); + } +} } // namespace nigiri::loader::gtfs \ No newline at end of file From d38704c25c5bce88f5485974a59b58ace4c0414a Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:18:10 +0100 Subject: [PATCH 15/60] Make methods const --- include/nigiri/loader/gtfs/shape_prepare.h | 4 ++-- src/loader/gtfs/shape_prepare.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 39572996..1b024dd2 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -34,8 +34,8 @@ struct trip_shapes { void calculate_shape_offsets(timetable const&, shapes_storage&, shape_loader_state const&); - void store_offsets(shapes_storage&, vector_map const&); - void create_boxes(timetable const&, shapes_storage&); + void store_offsets(shapes_storage&, vector_map const&) const; + void create_boxes(timetable const&, shapes_storage&) const; shape_idx_t index_offset_; std::vector shape_segments_; }; diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 95239e97..344764fe 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -324,7 +324,7 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, void trip_shapes::store_offsets( shapes_storage& shapes_data, - vector_map const& trips) { + vector_map const& trips) const { for (auto const& trip : trips) { auto const trip_idx = trip.trip_idx_; auto const shape_idx = trip.shape_idx_; @@ -347,7 +347,7 @@ void trip_shapes::store_offsets( } void trip_shapes::create_boxes(timetable const& tt, - shapes_storage& shapes_data) { + shapes_storage& shapes_data) const { auto const new_routes = interval{static_cast(shapes_data.boxes_.size()), static_cast(tt.route_transport_ranges_.size())}; From 8642d1be5cedcf10a552feb38bde080790414564 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:18:40 +0100 Subject: [PATCH 16/60] Update progress bar --- src/loader/gtfs/shape_prepare.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 344764fe..71be62a3 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -251,9 +251,14 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, shapes_storage& shapes_data, shape_loader_state const& states) { auto m = std::mutex{}; + auto const progress_tracker = utl::get_active_progress_tracker(); + progress_tracker->status("Calculating shape offsets") + .out_bounds(98.F, 99.F) + .in_high(shape_segments_.size()); std::for_each( std::execution::par_unseq, shape_segments_.begin(), shape_segments_.end(), [&](shape_segments& segments) { + progress_tracker->increment(); if (segments.offsets_.empty()) { return; } @@ -325,7 +330,12 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, void trip_shapes::store_offsets( shapes_storage& shapes_data, vector_map const& trips) const { + auto const progress_tracker = utl::get_active_progress_tracker(); + progress_tracker->status("Storing trip offsets") + .out_bounds(100.F, 100.F) + .in_high(trips.size()); for (auto const& trip : trips) { + progress_tracker->increment(); auto const trip_idx = trip.trip_idx_; auto const shape_idx = trip.shape_idx_; auto const offset_idx = [&]() { From 3d82cd78694090b4e9dacb232f6261253d1e47ba Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:43:24 +0100 Subject: [PATCH 17/60] Move nullptr check into struct --- include/nigiri/loader/gtfs/shape_prepare.h | 7 +++-- src/loader/gtfs/load_timetable.cc | 10 ++----- src/loader/gtfs/shape_prepare.cc | 33 ++++++++++++++-------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 1b024dd2..f358e026 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -32,12 +32,13 @@ struct trip_shapes { trip_shapes(shape_loader_state const&, vector_map const&); void calculate_shape_offsets(timetable const&, - shapes_storage&, + shapes_storage*, shape_loader_state const&); - void store_offsets(shapes_storage&, vector_map const&) const; - void create_boxes(timetable const&, shapes_storage&) const; + void store_offsets(vector_map const&) const; + void create_boxes(timetable const&) const; shape_idx_t index_offset_; std::vector shape_segments_; + shapes_storage* shapes_; }; trip_shapes get_shape_pairs(shape_loader_state const&, diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index d33accf6..d001bb6e 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -262,10 +262,8 @@ void load_timetable(loader_config const& config, train_nr, stop_seq_numbers); } auto shape_pairs = trip_shapes{shape_states, trip_data.data_}; - if (shapes_data != nullptr) { - shape_pairs.calculate_shape_offsets(tt, *shapes_data, shape_states); - shape_pairs.store_offsets(*shapes_data, trip_data.data_); - } + shape_pairs.calculate_shape_offsets(tt, shapes_data, shape_states); + shape_pairs.store_offsets(trip_data.data_); auto const timer = scoped_timer{"loader.gtfs.routes.build"}; auto const attributes = std::basic_string{}; @@ -370,9 +368,7 @@ void load_timetable(loader_config const& config, } // Build bounding boxes - if (shapes_data != nullptr) { - shape_pairs.create_boxes(tt, *shapes_data); - } + shape_pairs.create_boxes(tt); // Build location_routes map for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) { diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 71be62a3..302348e0 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -226,7 +226,8 @@ shape_segment::shape_segment(stop_seq_t const* stop_seq, trip_shapes::trip_shapes(shape_loader_state const& states, vector_map const& trips) : index_offset_{states.index_offset_}, - shape_segments_(states.id_map_.size()) { + shape_segments_(states.id_map_.size()), + shapes_{nullptr} { for (auto i = 0U; i < shape_segments_.size(); ++i) { shape_segments_[i].shape_idx_ = static_cast(index_offset_ + i); } @@ -248,8 +249,12 @@ trip_shapes::trip_shapes(shape_loader_state const& states, } void trip_shapes::calculate_shape_offsets(timetable const& tt, - shapes_storage& shapes_data, + shapes_storage* shapes_data, shape_loader_state const& states) { + if (shapes_data == nullptr) { + return; + } + shapes_ = shapes_data; auto m = std::mutex{}; auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Calculating shape offsets") @@ -262,7 +267,7 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, if (segments.offsets_.empty()) { return; } - auto const shape = shapes_data.get_shape(segments.shape_idx_); + auto const shape = shapes_->get_shape(segments.shape_idx_); auto const& shape_distances = states .distances_[cista::to_idx(segments.shape_idx_ - index_offset_)]; @@ -280,7 +285,7 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, }(); if (!offsets.empty()) { auto const guard = std::lock_guard{m}; - segment.offset_idx_ = shapes_data.add_offsets(offsets); + segment.offset_idx_ = shapes_->add_offsets(offsets); } else { segment.offset_idx_ = shape_offset_idx_t::invalid(); } @@ -328,8 +333,10 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, } void trip_shapes::store_offsets( - shapes_storage& shapes_data, vector_map const& trips) const { + if (shapes_ == nullptr) { + return; + } auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Storing trip offsets") .out_bounds(100.F, 100.F) @@ -351,15 +358,17 @@ void trip_shapes::store_offsets( }); return segment->offset_idx_; }(); - shapes_data.add_trip_shape_offsets(trip_idx, - cista::pair{shape_idx, offset_idx}); + shapes_->add_trip_shape_offsets(trip_idx, + cista::pair{shape_idx, offset_idx}); } } -void trip_shapes::create_boxes(timetable const& tt, - shapes_storage& shapes_data) const { +void trip_shapes::create_boxes(timetable const& tt) const { + if (shapes_ == nullptr) { + return; + } auto const new_routes = - interval{static_cast(shapes_data.boxes_.size()), + interval{static_cast(shapes_->boxes_.size()), static_cast(tt.route_transport_ranges_.size())}; auto route_boxes = std::vector>(cista::to_idx(new_routes.size())); @@ -383,7 +392,7 @@ void trip_shapes::create_boxes(timetable const& tt, frun.for_each_trip([&](trip_idx_t const trip_idx, interval const absolute_range) { auto const [shape_idx, offset_idx] = - shapes_data.trip_offset_indices_[trip_idx]; + shapes_->trip_offset_indices_[trip_idx]; if (shape_idx == shape_idx_t::invalid() || offset_idx == shape_offset_idx_t::invalid()) { for (auto const idx : absolute_range) { @@ -413,7 +422,7 @@ void trip_shapes::create_boxes(timetable const& tt, return boxes; }); for (auto const& box : route_boxes) { - shapes_data.boxes_.emplace_back(box); + shapes_->boxes_.emplace_back(box); } } } // namespace nigiri::loader::gtfs \ No newline at end of file From e70487478f4e2a1b0a0ebc9b795ebade434761ba Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:38:50 +0100 Subject: [PATCH 18/60] Check for execution policy support --- src/loader/gtfs/shape_prepare.cc | 51 ++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 302348e0..9ca9e0cf 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -237,10 +237,13 @@ trip_shapes::trip_shapes(shape_loader_state const& states, } auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); auto& candidates = shape_segments_[idx].offsets_; - if (std::all_of(std::execution::par_unseq, candidates.begin(), - candidates.end(), [&](shape_segment const& it) { - return *it.stop_seq_ != trip.stop_seq_; - })) { + if (std::all_of( +#if __cpp_lib_execution + std::execution::par_unseq, +#endif + candidates.begin(), candidates.end(), [&](shape_segment const& it) { + return *it.stop_seq_ != trip.stop_seq_; + })) { auto distances = trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; candidates.emplace_back(&trip.stop_seq_, distances); @@ -261,7 +264,10 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, .out_bounds(98.F, 99.F) .in_high(shape_segments_.size()); std::for_each( - std::execution::par_unseq, shape_segments_.begin(), shape_segments_.end(), +#if __cpp_lib_execution + std::execution::par_unseq, +#endif + shape_segments_.begin(), shape_segments_.end(), [&](shape_segments& segments) { progress_tracker->increment(); if (segments.offsets_.empty()) { @@ -351,11 +357,13 @@ void trip_shapes::store_offsets( } auto const& segments = shape_segments_[cista::to_idx(shape_idx - index_offset_)].offsets_; - auto const segment = - std::find_if(std::execution::par_unseq, begin(segments), - end(segments), [&](shape_segment const& s) { - return *s.stop_seq_ == trip.stop_seq_; - }); + auto const segment = std::find_if( +#if __cpp_lib_execution + std::execution::par_unseq, +#endif + begin(segments), end(segments), [&](shape_segment const& s) { + return *s.stop_seq_ == trip.stop_seq_; + }); return segment->offset_idx_; }(); shapes_->add_trip_shape_offsets(trip_idx, @@ -373,8 +381,11 @@ void trip_shapes::create_boxes(timetable const& tt) const { auto route_boxes = std::vector>(cista::to_idx(new_routes.size())); std::transform( - std::execution::par, begin(new_routes), end(new_routes), - begin(route_boxes), [&](route_idx_t const r) { +#if __cpp_lib_execution + std::execution::par, +#endif + begin(new_routes), end(new_routes), begin(route_boxes), + [&](route_idx_t const r) { auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto boxes = std::vector(seq.size() - 1 + 1); @@ -403,11 +414,13 @@ void trip_shapes::create_boxes(timetable const& tt) const { auto const& segments = shape_segments_[cista::to_idx(shape_idx - index_offset_)] .offsets_; - auto const& segment = - std::find_if(std::execution::par_unseq, begin(segments), - end(segments), [&](shape_segment const& s) { - return s.offset_idx_ == offset_idx; - }); + auto const& segment = std::find_if( +#if __cpp_lib_execution + std::execution::par_unseq, +#endif + begin(segments), end(segments), [&](shape_segment const& s) { + return s.offset_idx_ == offset_idx; + }); bounding_box.extend(segment->boxes_[0]); for (auto i = 1U; i < segment->boxes_.size(); ++i) { boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = @@ -421,8 +434,8 @@ void trip_shapes::create_boxes(timetable const& tt) const { boxes.resize(last_extend); return boxes; }); - for (auto const& box : route_boxes) { - shapes_->boxes_.emplace_back(box); + for (auto const& boxes : route_boxes) { + shapes_->boxes_.emplace_back(boxes); } } } // namespace nigiri::loader::gtfs \ No newline at end of file From 7d81cb53d86d11ab32e077b669e1fb07a1f19973 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:54:13 +0100 Subject: [PATCH 19/60] Remove unused code --- include/nigiri/loader/gtfs/shape_prepare.h | 10 -- src/loader/gtfs/shape_prepare.cc | 132 --------------------- 2 files changed, 142 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index f358e026..c1daf50a 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -41,14 +41,4 @@ struct trip_shapes { shapes_storage* shapes_; }; -trip_shapes get_shape_pairs(shape_loader_state const&, - vector_map const&); - -void calculate_shape_offsets(timetable const&, - shapes_storage&, - vector_map const&, - shape_loader_state const&); - -void calculate_shape_boxes(timetable const&, shapes_storage&); - } // namespace nigiri::loader::gtfs diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 9ca9e0cf..c59bb696 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -84,138 +84,6 @@ std::vector get_offsets_by_dist_traveled( return offsets; } -void calculate_shape_offsets(timetable const& tt, - shapes_storage& shapes_data, - vector_map const& trips, - shape_loader_state const& shape_states) { - auto const progress_tracker = utl::get_active_progress_tracker(); - progress_tracker->status("Calculating shape offsets") - .out_bounds(98.F, 100.F) - .in_high(trips.size()); - - auto const key_hash = - [](std::pair const& pair) noexcept { - auto h = cista::BASE_HASH; - h = cista::hash_combine(h, cista::hashing{}(pair.first)); - h = cista::hash_combine(h, cista::hashing{}(*pair.second)); - return h; - }; - auto const key_compare = - [](std::pair const& a, - std::pair const& b) noexcept { - return (a.first == b.first) && (*a.second == *b.second); - }; - auto shape_offsets_cache = - hash_map, shape_offset_idx_t, - decltype(key_hash), decltype(key_compare)>{}; - for (auto const& trip : trips) { - progress_tracker->increment(); - auto const trip_idx = trip.trip_idx_; - auto const shape_idx = trip.shape_idx_; - - auto const shape_offset_idx = utl::get_or_create( - shape_offsets_cache, std::pair{shape_idx, &trip.stop_seq_}, [&]() { - if (shape_idx == shape_idx_t::invalid() || - trip.stop_seq_.size() < 2U) { - return shape_offset_idx_t::invalid(); - } - auto const& shape_distances = - shape_states - .distances_[to_idx(shape_idx - shape_states.index_offset_)]; - if (!shape_distances.empty() && !trip.distance_traveled_.empty()) { - auto const offsets = get_offsets_by_dist_traveled( - trip.distance_traveled_, shape_distances); - return shapes_data.add_offsets(offsets); - } - auto const shape = shapes_data.get_shape(shape_idx); - if (shape.size() < trip.stop_seq_.size()) { - return shape_offset_idx_t::invalid(); // >= 1 shape/point required - } - auto const offsets = get_offsets_by_stops(tt, shape, trip.stop_seq_); - return shapes_data.add_offsets(offsets); - }); - shapes_data.add_trip_shape_offsets( - trip_idx, cista::pair{shape_offset_idx == shape_offset_idx_t::invalid() - ? shape_idx_t::invalid() - : shape_idx, - shape_offset_idx}); - } -} - -void calculate_shape_boxes(timetable const& tt, shapes_storage& shapes_data) { - auto cached_shape_boxes = - hash_map, - std::vector>{}; - auto const new_routes = - interval{static_cast(shapes_data.boxes_.size()), - static_cast(tt.route_transport_ranges_.size())}; - for (auto const r : new_routes) { - auto const seq = tt.route_location_seq_[r]; - assert(seq.size() > 0U); - auto segment_boxes = std::vector(seq.size()); - auto last_extend = 0U; - // 0: bounding box for trip, 1-N: bounding box for segment - auto& bounding_box = segment_boxes[0U]; - auto const stop_indices = - interval{stop_idx_t{0U}, static_cast(seq.size())}; - for (auto const transport_idx : tt.route_transport_ranges_[r]) { - auto const frun = rt::frun{tt, nullptr, - rt::run{.t_ = transport{transport_idx}, - .stop_range_ = stop_indices, - .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip([&](trip_idx_t const trip_idx, - interval const absolute_range) { - auto const key = shapes_data.trip_offset_indices_[trip_idx]; - auto const& shape_boxes = - utl::get_or_create(cached_shape_boxes, key, [&]() { - if (key.first == shape_idx_t::invalid() || - key.second == shape_offset_idx_t::invalid()) { - return std::vector{}; - } - auto const shape = shapes_data.get_shape(key.first); - auto const& offsets = shapes_data.offsets_[key.second]; - auto boxes = std::vector(offsets.size() - 1); - for (auto const [i, pair] : - utl::enumerate(utl::pairwise(offsets))) { - auto& box = boxes[i]; - auto const& [from, to] = pair; - for (auto const point : shape.subspan( - cista::to_idx(from), - cista::to_idx(to) - cista::to_idx(from) + 1)) { - box.extend(point); - } - } - return boxes; - }); - auto prev_pos = tt.locations_.coordinates_.at( - stop{seq[absolute_range.from_]}.location_idx()); - bounding_box.extend(prev_pos); - for (auto const [from, to] : utl::pairwise(absolute_range)) { - auto const next_pos = - tt.locations_.coordinates_.at(stop{seq[to]}.location_idx()); - auto& box = segment_boxes[cista::to_idx(to)]; - bounding_box.extend(next_pos); - box.extend(prev_pos); - box.extend(next_pos); - if (!shape_boxes.empty()) { - auto const& shape_box = shape_boxes[static_cast( - cista::to_idx(from) - cista::to_idx(absolute_range.from_))]; - if (!box.contains(shape_box)) { - bounding_box.extend(shape_box); - box.extend(shape_box); - last_extend = std::max(last_extend, from + 1U); - } - } - prev_pos = next_pos; - } - }); - } - // 0: bounding box for trip, 1-N: bounding box for segment - segment_boxes.resize(last_extend + 1); - shapes_data.boxes_.emplace_back(segment_boxes); - } -} - shape_segment::shape_segment(stop_seq_t const* stop_seq, std::vector const* distances) : stop_seq_{stop_seq}, From 22ba3292791480b37058d03248954b4c06f8abc0 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:50:54 +0100 Subject: [PATCH 20/60] Make nullopt more explicit --- src/shapes_storage.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shapes_storage.cc b/src/shapes_storage.cc index 1d18f7de..719fb0b0 100644 --- a/src/shapes_storage.cc +++ b/src/shapes_storage.cc @@ -107,7 +107,7 @@ std::optional shapes_storage::get_bounding_box( auto const& boxes = boxes_[route_idx]; // 1-N: bounding box for segment return segment + 1 < boxes.size() ? boxes[segment + 1] - : std::optional{}; + : std::optional{std::nullopt}; } } // namespace nigiri \ No newline at end of file From 60dab2864a1c0e4a22ca8da161e589358c757916 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 08:28:18 +0100 Subject: [PATCH 21/60] Fix MSVC build --- include/nigiri/common/interval.h | 4 ++-- src/loader/gtfs/shape_prepare.cc | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/nigiri/common/interval.h b/include/nigiri/common/interval.h index 8df73e8a..14414629 100644 --- a/include/nigiri/common/interval.h +++ b/include/nigiri/common/interval.h @@ -47,8 +47,8 @@ struct interval { t_ -= x; return *this; } - iterator operator+(difference_type const x) const { return *this += x; } - iterator operator-(difference_type const x) const { return *this -= x; } + iterator operator+(difference_type const x) const { return { t_ + x }; } + iterator operator-(difference_type const x) const { return { t_ - x }; } friend difference_type operator-(iterator const& a, iterator const& b) { return static_cast(cista::to_idx(a.t_) - cista::to_idx(b.t_)); diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index c59bb696..4aaa4944 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -295,7 +295,8 @@ void trip_shapes::create_boxes(timetable const& tt) const { segment->boxes_[i]; } last_extend = std::max( - last_extend, segment->boxes_.size() + absolute_range.from_); + last_extend, + static_cast(segment->boxes_.size() + absolute_range.from_)); } }); } From b09d4c0d21ad039471bb5a1a2c5118ae34801b77 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:42:23 +0100 Subject: [PATCH 22/60] Fix formatting --- src/loader/gtfs/shape_prepare.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 4aaa4944..15ddede7 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -294,9 +294,10 @@ void trip_shapes::create_boxes(timetable const& tt) const { boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = segment->boxes_[i]; } - last_extend = std::max( - last_extend, - static_cast(segment->boxes_.size() + absolute_range.from_)); + last_extend = + std::max(last_extend, + static_cast(segment->boxes_.size() + + absolute_range.from_)); } }); } From ec3bbdde64180f6399bbb5359d99194fe1da6b2a Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:45:00 +0100 Subject: [PATCH 23/60] Fix formatting --- include/nigiri/common/interval.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/nigiri/common/interval.h b/include/nigiri/common/interval.h index 14414629..3dc455b9 100644 --- a/include/nigiri/common/interval.h +++ b/include/nigiri/common/interval.h @@ -47,8 +47,8 @@ struct interval { t_ -= x; return *this; } - iterator operator+(difference_type const x) const { return { t_ + x }; } - iterator operator-(difference_type const x) const { return { t_ - x }; } + iterator operator+(difference_type const x) const { return {t_ + x}; } + iterator operator-(difference_type const x) const { return {t_ - x}; } friend difference_type operator-(iterator const& a, iterator const& b) { return static_cast(cista::to_idx(a.t_) - cista::to_idx(b.t_)); From eda90d5fc613929d0af605ebeb17a78c152e1300 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:03:09 +0100 Subject: [PATCH 24/60] Add test for routes containing sequential trips --- test/rt/frun_shape_test.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc index 1a8830ff..e06b9aeb 100644 --- a/test/rt/frun_shape_test.cc +++ b/test/rt/frun_shape_test.cc @@ -442,6 +442,20 @@ TEST( leg_shape); } } + // Testing bounding boxes + { + // Full route A -> F -> G -> H -> I -> J -> K (Index 1) + EXPECT_EQ((geo::make_box({{1.0, 0.5}, {5.0, 3.0}})), + shapes_data.get_bounding_box(route_idx_t{1U})); + // Shape segment not contained in simple box + // H -> I + { + auto const segment_box = + shapes_data.get_bounding_box(route_idx_t{1}, 4); + ASSERT_TRUE(segment_box.has_value()); + EXPECT_EQ((geo::make_box({{3.0, 2.5}, {4.0, 3.0}})), *segment_box); + } + } } // Multiple trips, some with and some without shape { From 339ba6fd6b44c7570756d89fdb83fa62bab5683d Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:57:27 +0100 Subject: [PATCH 25/60] Rename shape result objects --- include/nigiri/loader/gtfs/shape_prepare.h | 49 ++++++++++---------- src/loader/gtfs/load_timetable.cc | 8 ++-- src/loader/gtfs/shape_prepare.cc | 54 +++++++++++----------- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index c1daf50a..36fbdd6d 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -2,42 +2,41 @@ #include "geo/box.h" -#include "nigiri/loader/gtfs/shape.h" #include "nigiri/loader/gtfs/trip.h" #include "nigiri/types.h" namespace nigiri { +struct shape_loader_state; struct shapes_storage; struct timetable; } // namespace nigiri namespace nigiri::loader::gtfs { -struct shape_segment { - explicit shape_segment(stop_seq_t const* stop_seq, - std::vector const* distances); - stop_seq_t const* stop_seq_; - shape_offset_idx_t offset_idx_; - std::vector const* distances_; - std::vector boxes_; -}; - -struct shape_segments { - shape_idx_t shape_idx_; - std::vector offsets_; -}; - -// TODO Rename -struct trip_shapes { - trip_shapes(shape_loader_state const&, - vector_map const&); - void calculate_shape_offsets(timetable const&, - shapes_storage*, - shape_loader_state const&); - void store_offsets(vector_map const&) const; - void create_boxes(timetable const&) const; +struct shape_prepare { + struct shape_results { + struct result { + explicit result(stop_seq_t const* stop_seq, + std::vector const* distances); + stop_seq_t const* stop_seq_; + std::vector const* distances_; + shape_offset_idx_t offset_idx_; + std::vector boxes_; + }; + + shape_idx_t shape_idx_; + std::vector results_; + }; + + shape_prepare(shape_loader_state const&, + vector_map const&); + void calculate_results(timetable const&, + shapes_storage*, + shape_loader_state const&); + void write_trip_shape_offsets(vector_map const&) const; + void write_route_boxes(timetable const&) const; shape_idx_t index_offset_; - std::vector shape_segments_; + std::vector shape_results_; shapes_storage* shapes_; }; diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index d001bb6e..1e26079e 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -261,9 +261,9 @@ void load_timetable(loader_config const& config, {source_file_idx, trp.from_line_, trp.to_line_}, train_nr, stop_seq_numbers); } - auto shape_pairs = trip_shapes{shape_states, trip_data.data_}; - shape_pairs.calculate_shape_offsets(tt, shapes_data, shape_states); - shape_pairs.store_offsets(trip_data.data_); + auto shape_pairs = shape_prepare{shape_states, trip_data.data_}; + shape_pairs.calculate_results(tt, shapes_data, shape_states); + shape_pairs.write_trip_shape_offsets(trip_data.data_); auto const timer = scoped_timer{"loader.gtfs.routes.build"}; auto const attributes = std::basic_string{}; @@ -368,7 +368,7 @@ void load_timetable(loader_config const& config, } // Build bounding boxes - shape_pairs.create_boxes(tt); + shape_pairs.write_route_boxes(tt); // Build location_routes map for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) { diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 15ddede7..4b680f6d 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -13,9 +13,9 @@ #include "geo/polyline.h" #include "utl/enumerate.h" -#include "utl/get_or_create.h" #include "utl/progress_tracker.h" +#include "nigiri/loader/gtfs/shape.h" #include "nigiri/rt/frun.h" #include "nigiri/shapes_storage.h" #include "nigiri/stop.h" @@ -84,32 +84,33 @@ std::vector get_offsets_by_dist_traveled( return offsets; } -shape_segment::shape_segment(stop_seq_t const* stop_seq, - std::vector const* distances) +shape_prepare::shape_results::result::result( + stop_seq_t const* stop_seq, std::vector const* distances) : stop_seq_{stop_seq}, - offset_idx_{shape_offset_idx_t::invalid()}, distances_{distances}, + offset_idx_{shape_offset_idx_t::invalid()}, boxes_{} {} -trip_shapes::trip_shapes(shape_loader_state const& states, - vector_map const& trips) +shape_prepare::shape_prepare(shape_loader_state const& states, + vector_map const& trips) : index_offset_{states.index_offset_}, - shape_segments_(states.id_map_.size()), + shape_results_(states.id_map_.size()), shapes_{nullptr} { - for (auto i = 0U; i < shape_segments_.size(); ++i) { - shape_segments_[i].shape_idx_ = static_cast(index_offset_ + i); + for (auto i = 0U; i < shape_results_.size(); ++i) { + shape_results_[i].shape_idx_ = static_cast(index_offset_ + i); } for (auto const& trip : trips) { if (trip.shape_idx_ == shape_idx_t::invalid()) { continue; } auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); - auto& candidates = shape_segments_[idx].offsets_; + auto& candidates = shape_results_[idx].results_; if (std::all_of( #if __cpp_lib_execution std::execution::par_unseq, #endif - candidates.begin(), candidates.end(), [&](shape_segment const& it) { + candidates.begin(), candidates.end(), + [&](shape_results::result const& it) { return *it.stop_seq_ != trip.stop_seq_; })) { auto distances = @@ -119,9 +120,9 @@ trip_shapes::trip_shapes(shape_loader_state const& states, } } -void trip_shapes::calculate_shape_offsets(timetable const& tt, - shapes_storage* shapes_data, - shape_loader_state const& states) { +void shape_prepare::calculate_results(timetable const& tt, + shapes_storage* shapes_data, + shape_loader_state const& states) { if (shapes_data == nullptr) { return; } @@ -130,22 +131,22 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Calculating shape offsets") .out_bounds(98.F, 99.F) - .in_high(shape_segments_.size()); + .in_high(shape_results_.size()); std::for_each( #if __cpp_lib_execution std::execution::par_unseq, #endif - shape_segments_.begin(), shape_segments_.end(), - [&](shape_segments& segments) { + shape_results_.begin(), shape_results_.end(), + [&](shape_results& segments) { progress_tracker->increment(); - if (segments.offsets_.empty()) { + if (segments.results_.empty()) { return; } auto const shape = shapes_->get_shape(segments.shape_idx_); auto const& shape_distances = states .distances_[cista::to_idx(segments.shape_idx_ - index_offset_)]; - for (auto& segment : segments.offsets_) { + for (auto& segment : segments.results_) { auto const offsets = [&]() { if (!shape_distances.empty() && segment.distances_ != nullptr) { return get_offsets_by_dist_traveled(*segment.distances_, @@ -206,7 +207,7 @@ void trip_shapes::calculate_shape_offsets(timetable const& tt, }); } -void trip_shapes::store_offsets( +void shape_prepare::write_trip_shape_offsets( vector_map const& trips) const { if (shapes_ == nullptr) { return; @@ -224,12 +225,12 @@ void trip_shapes::store_offsets( return shape_offset_idx_t::invalid(); } auto const& segments = - shape_segments_[cista::to_idx(shape_idx - index_offset_)].offsets_; + shape_results_[cista::to_idx(shape_idx - index_offset_)].results_; auto const segment = std::find_if( #if __cpp_lib_execution std::execution::par_unseq, #endif - begin(segments), end(segments), [&](shape_segment const& s) { + begin(segments), end(segments), [&](shape_results::result const& s) { return *s.stop_seq_ == trip.stop_seq_; }); return segment->offset_idx_; @@ -239,7 +240,7 @@ void trip_shapes::store_offsets( } } -void trip_shapes::create_boxes(timetable const& tt) const { +void shape_prepare::write_route_boxes(timetable const& tt) const { if (shapes_ == nullptr) { return; } @@ -280,13 +281,14 @@ void trip_shapes::create_boxes(timetable const& tt) const { } } else { auto const& segments = - shape_segments_[cista::to_idx(shape_idx - index_offset_)] - .offsets_; + shape_results_[cista::to_idx(shape_idx - index_offset_)] + .results_; auto const& segment = std::find_if( #if __cpp_lib_execution std::execution::par_unseq, #endif - begin(segments), end(segments), [&](shape_segment const& s) { + begin(segments), end(segments), + [&](shape_results::result const& s) { return s.offset_idx_ == offset_idx; }); bounding_box.extend(segment->boxes_[0]); From b1973f742d86bbbd131582596a695fc00bc88027 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:38:12 +0100 Subject: [PATCH 26/60] Make intended usage more obvious --- include/nigiri/loader/gtfs/shape_prepare.h | 10 +++--- src/loader/gtfs/load_timetable.cc | 18 ++++++++--- src/loader/gtfs/shape_prepare.cc | 37 ++++++++++------------ 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 36fbdd6d..b71e1ad0 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -29,15 +29,15 @@ struct shape_prepare { }; shape_prepare(shape_loader_state const&, - vector_map const&); - void calculate_results(timetable const&, - shapes_storage*, - shape_loader_state const&); + vector_map const&, + shapes_storage&); + void calculate_results(timetable const&, shape_loader_state const&); void write_trip_shape_offsets(vector_map const&) const; void write_route_boxes(timetable const&) const; shape_idx_t index_offset_; std::vector shape_results_; - shapes_storage* shapes_; + shapes_storage& shapes_; + bool results_ready_; }; } // namespace nigiri::loader::gtfs diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 1e26079e..5249ff33 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "utl/get_or_create.h" @@ -261,9 +262,16 @@ void load_timetable(loader_config const& config, {source_file_idx, trp.from_line_, trp.to_line_}, train_nr, stop_seq_numbers); } - auto shape_pairs = shape_prepare{shape_states, trip_data.data_}; - shape_pairs.calculate_results(tt, shapes_data, shape_states); - shape_pairs.write_trip_shape_offsets(trip_data.data_); + + auto shape_results = + (shapes_data == nullptr) + ? std::nullopt + : std::make_optional( + shape_prepare{shape_states, trip_data.data_, *shapes_data}); + if (shape_results.has_value()) { + shape_results->calculate_results(tt, shape_states); + shape_results->write_trip_shape_offsets(trip_data.data_); + } auto const timer = scoped_timer{"loader.gtfs.routes.build"}; auto const attributes = std::basic_string{}; @@ -368,7 +376,9 @@ void load_timetable(loader_config const& config, } // Build bounding boxes - shape_pairs.write_route_boxes(tt); + if (shape_results.has_value()) { + shape_results->write_route_boxes(tt); + } // Build location_routes map for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) { diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 4b680f6d..63b7a734 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -14,6 +14,7 @@ #include "utl/enumerate.h" #include "utl/progress_tracker.h" +#include "utl/verify.h" #include "nigiri/loader/gtfs/shape.h" #include "nigiri/rt/frun.h" @@ -92,10 +93,12 @@ shape_prepare::shape_results::result::result( boxes_{} {} shape_prepare::shape_prepare(shape_loader_state const& states, - vector_map const& trips) + vector_map const& trips, + shapes_storage& shapes_data) : index_offset_{states.index_offset_}, shape_results_(states.id_map_.size()), - shapes_{nullptr} { + shapes_{shapes_data}, + results_ready_{false} { for (auto i = 0U; i < shape_results_.size(); ++i) { shape_results_[i].shape_idx_ = static_cast(index_offset_ + i); } @@ -121,12 +124,7 @@ shape_prepare::shape_prepare(shape_loader_state const& states, } void shape_prepare::calculate_results(timetable const& tt, - shapes_storage* shapes_data, shape_loader_state const& states) { - if (shapes_data == nullptr) { - return; - } - shapes_ = shapes_data; auto m = std::mutex{}; auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Calculating shape offsets") @@ -142,7 +140,7 @@ void shape_prepare::calculate_results(timetable const& tt, if (segments.results_.empty()) { return; } - auto const shape = shapes_->get_shape(segments.shape_idx_); + auto const shape = shapes_.get_shape(segments.shape_idx_); auto const& shape_distances = states .distances_[cista::to_idx(segments.shape_idx_ - index_offset_)]; @@ -160,7 +158,7 @@ void shape_prepare::calculate_results(timetable const& tt, }(); if (!offsets.empty()) { auto const guard = std::lock_guard{m}; - segment.offset_idx_ = shapes_->add_offsets(offsets); + segment.offset_idx_ = shapes_.add_offsets(offsets); } else { segment.offset_idx_ = shape_offset_idx_t::invalid(); } @@ -205,16 +203,15 @@ void shape_prepare::calculate_results(timetable const& tt, }(); } }); + results_ready_ = true; } void shape_prepare::write_trip_shape_offsets( vector_map const& trips) const { - if (shapes_ == nullptr) { - return; - } + utl::verify(results_ready_, "Operation requires calculated results"); auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Storing trip offsets") - .out_bounds(100.F, 100.F) + .out_bounds(99.F, 100.F) .in_high(trips.size()); for (auto const& trip : trips) { progress_tracker->increment(); @@ -235,17 +232,15 @@ void shape_prepare::write_trip_shape_offsets( }); return segment->offset_idx_; }(); - shapes_->add_trip_shape_offsets(trip_idx, - cista::pair{shape_idx, offset_idx}); + shapes_.add_trip_shape_offsets(trip_idx, + cista::pair{shape_idx, offset_idx}); } } void shape_prepare::write_route_boxes(timetable const& tt) const { - if (shapes_ == nullptr) { - return; - } + utl::verify(results_ready_, "Operation requires calculated results"); auto const new_routes = - interval{static_cast(shapes_->boxes_.size()), + interval{static_cast(shapes_.boxes_.size()), static_cast(tt.route_transport_ranges_.size())}; auto route_boxes = std::vector>(cista::to_idx(new_routes.size())); @@ -272,7 +267,7 @@ void shape_prepare::write_route_boxes(timetable const& tt) const { frun.for_each_trip([&](trip_idx_t const trip_idx, interval const absolute_range) { auto const [shape_idx, offset_idx] = - shapes_->trip_offset_indices_[trip_idx]; + shapes_.trip_offset_indices_[trip_idx]; if (shape_idx == shape_idx_t::invalid() || offset_idx == shape_offset_idx_t::invalid()) { for (auto const idx : absolute_range) { @@ -307,7 +302,7 @@ void shape_prepare::write_route_boxes(timetable const& tt) const { return boxes; }); for (auto const& boxes : route_boxes) { - shapes_->boxes_.emplace_back(boxes); + shapes_.boxes_.emplace_back(boxes); } } } // namespace nigiri::loader::gtfs \ No newline at end of file From a2c7d758bf53d2ab4446579c9b95838147ccd834 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:50:58 +0100 Subject: [PATCH 27/60] Move data on last usage --- src/loader/gtfs/load_timetable.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 5249ff33..2b123212 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -269,7 +269,7 @@ void load_timetable(loader_config const& config, : std::make_optional( shape_prepare{shape_states, trip_data.data_, *shapes_data}); if (shape_results.has_value()) { - shape_results->calculate_results(tt, shape_states); + shape_results->calculate_results(tt, std::move(shape_states)); shape_results->write_trip_shape_offsets(trip_data.data_); } From 85d4638d49d440d3f6e92d6fa6affbc4b456013d Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:22:56 +0100 Subject: [PATCH 28/60] Update variable names --- src/loader/gtfs/shape_prepare.cc | 62 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 63b7a734..5e1ebc24 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -135,39 +135,39 @@ void shape_prepare::calculate_results(timetable const& tt, std::execution::par_unseq, #endif shape_results_.begin(), shape_results_.end(), - [&](shape_results& segments) { + [&](shape_results& results) { progress_tracker->increment(); - if (segments.results_.empty()) { + if (results.results_.empty()) { return; } - auto const shape = shapes_.get_shape(segments.shape_idx_); + auto const shape = shapes_.get_shape(results.shape_idx_); auto const& shape_distances = states - .distances_[cista::to_idx(segments.shape_idx_ - index_offset_)]; - for (auto& segment : segments.results_) { + .distances_[cista::to_idx(results.shape_idx_ - index_offset_)]; + for (auto& result : results.results_) { auto const offsets = [&]() { - if (!shape_distances.empty() && segment.distances_ != nullptr) { - return get_offsets_by_dist_traveled(*segment.distances_, + if (!shape_distances.empty() && result.distances_ != nullptr) { + return get_offsets_by_dist_traveled(*result.distances_, shape_distances); } - if (shape.size() < segment.stop_seq_->size()) { + if (shape.size() < result.stop_seq_->size()) { return std::vector{}; } - return get_offsets_by_stops(tt, shape, *segment.stop_seq_); + return get_offsets_by_stops(tt, shape, *result.stop_seq_); }(); if (!offsets.empty()) { auto const guard = std::lock_guard{m}; - segment.offset_idx_ = shapes_.add_offsets(offsets); + result.offset_idx_ = shapes_.add_offsets(offsets); } else { - segment.offset_idx_ = shape_offset_idx_t::invalid(); + result.offset_idx_ = shape_offset_idx_t::invalid(); } - segment.boxes_ = [&]() { + result.boxes_ = [&]() { // Store box of full shape at index 0 auto boxes = offsets.empty() ? std::vector(1) - : std::vector(segment.stop_seq_->size() - 1 + 1); + : std::vector(result.stop_seq_->size() - 1 + 1); auto& shape_box = boxes.front(); auto last_extend = 0UL; if (!offsets.empty()) { @@ -182,10 +182,10 @@ void shape_prepare::calculate_results(timetable const& tt, segment_box.extend(point); } auto const from_l = - tt.locations_.coordinates_[stop{(*segment.stop_seq_)[i]} + tt.locations_.coordinates_[stop{(*result.stop_seq_)[i]} .location_idx()]; auto const to_l = - tt.locations_.coordinates_[stop{(*segment.stop_seq_)[i + 1]} + tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} .location_idx()]; auto const stop_box = geo::make_box({from_l, to_l}); if (!stop_box.contains(segment_box)) { @@ -193,7 +193,7 @@ void shape_prepare::calculate_results(timetable const& tt, } } } else { - for (auto const s : *segment.stop_seq_) { + for (auto const s : *result.stop_seq_) { shape_box.extend( tt.locations_.coordinates_[stop{s}.location_idx()]); } @@ -221,16 +221,16 @@ void shape_prepare::write_trip_shape_offsets( if (shape_idx == shape_idx_t::invalid()) { return shape_offset_idx_t::invalid(); } - auto const& segments = + auto const& results = shape_results_[cista::to_idx(shape_idx - index_offset_)].results_; - auto const segment = std::find_if( + auto const result = std::find_if( #if __cpp_lib_execution std::execution::par_unseq, #endif - begin(segments), end(segments), [&](shape_results::result const& s) { - return *s.stop_seq_ == trip.stop_seq_; + begin(results), end(results), [&](shape_results::result const& res) { + return *res.stop_seq_ == trip.stop_seq_; }); - return segment->offset_idx_; + return result->offset_idx_; }(); shapes_.add_trip_shape_offsets(trip_idx, cista::pair{shape_idx, offset_idx}); @@ -255,7 +255,7 @@ void shape_prepare::write_route_boxes(timetable const& tt) const { auto boxes = std::vector(seq.size() - 1 + 1); auto last_extend = 1UL; auto& bounding_box = boxes[0U]; - // 0: bounding box for trip, 1-N: bounding box for each segment + // 0: bounding box for route, 1-N: bounding box for each segment auto const stop_indices = interval{stop_idx_t{0U}, static_cast(seq.size())}; for (auto const transport_idx : tt.route_transport_ranges_[r]) { @@ -275,25 +275,25 @@ void shape_prepare::write_route_boxes(timetable const& tt) const { tt.locations_.coordinates_[stop{seq[idx]}.location_idx()]); } } else { - auto const& segments = + auto const& results = shape_results_[cista::to_idx(shape_idx - index_offset_)] .results_; - auto const& segment = std::find_if( + auto const& result = std::find_if( #if __cpp_lib_execution std::execution::par_unseq, #endif - begin(segments), end(segments), - [&](shape_results::result const& s) { - return s.offset_idx_ == offset_idx; + begin(results), end(results), + [&](shape_results::result const& res) { + return res.offset_idx_ == offset_idx; }); - bounding_box.extend(segment->boxes_[0]); - for (auto i = 1U; i < segment->boxes_.size(); ++i) { + bounding_box.extend(result->boxes_[0]); + for (auto i = 1U; i < result->boxes_.size(); ++i) { boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = - segment->boxes_[i]; + result->boxes_[i]; } last_extend = std::max(last_extend, - static_cast(segment->boxes_.size() + + static_cast(result->boxes_.size() + absolute_range.from_)); } }); From c37e7927be6a6771ee72e10f533bbcc22db8ae5e Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:28:35 +0100 Subject: [PATCH 29/60] Renane variable --- src/loader/gtfs/shape_prepare.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 5e1ebc24..d014ddd0 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -107,12 +107,12 @@ shape_prepare::shape_prepare(shape_loader_state const& states, continue; } auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); - auto& candidates = shape_results_[idx].results_; + auto& results = shape_results_[idx].results_; if (std::all_of( #if __cpp_lib_execution std::execution::par_unseq, #endif - candidates.begin(), candidates.end(), + results.begin(), results.end(), [&](shape_results::result const& it) { return *it.stop_seq_ != trip.stop_seq_; })) { From b6a1b97f568efe6dd38933d23c80a51e5b56ed90 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:28:47 +0100 Subject: [PATCH 30/60] Add missing const --- src/loader/gtfs/shape_prepare.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index d014ddd0..268a7e4f 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -116,9 +116,9 @@ shape_prepare::shape_prepare(shape_loader_state const& states, [&](shape_results::result const& it) { return *it.stop_seq_ != trip.stop_seq_; })) { - auto distances = + auto const distances = trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; - candidates.emplace_back(&trip.stop_seq_, distances); + results.emplace_back(&trip.stop_seq_, std::move(distances)); } } } From 289a0001276c1b68835d4969ed2564b7cea2e995 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 06:30:29 +0100 Subject: [PATCH 31/60] Apply suggested newline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Felix Gündling --- include/nigiri/loader/gtfs/shape_prepare.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index b71e1ad0..11f0fcd6 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -35,6 +35,7 @@ struct shape_prepare { void write_trip_shape_offsets(vector_map const&) const; void write_route_boxes(timetable const&) const; shape_idx_t index_offset_; + std::vector shape_results_; shapes_storage& shapes_; bool results_ready_; From ecf11c6f72df3ee17881dbf8ea000f9445e9ce01 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 06:44:44 +0100 Subject: [PATCH 32/60] Improve readability --- include/nigiri/shapes_storage.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/nigiri/shapes_storage.h b/include/nigiri/shapes_storage.h index 641bbb6f..2bc7ac01 100644 --- a/include/nigiri/shapes_storage.h +++ b/include/nigiri/shapes_storage.h @@ -26,7 +26,8 @@ struct shapes_storage { void add_trip_shape_offsets( trip_idx_t, cista::pair const&); geo::box get_bounding_box(route_idx_t) const; - std::optional get_bounding_box(route_idx_t, std::size_t) const; + std::optional get_bounding_box(route_idx_t, + std::size_t segment) const; cista::mmap::protection mode_; std::filesystem::path p_; From 38b55a66035aab768fa366a458e8ec5adc0d17c8 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:06:09 +0100 Subject: [PATCH 33/60] Update parallel computations --- src/loader/gtfs/shape_prepare.cc | 257 ++++++++++++++----------------- 1 file changed, 116 insertions(+), 141 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 268a7e4f..0db64c91 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -1,7 +1,6 @@ #include "nigiri/loader/gtfs/shape_prepare.h" #include -#include #include #include #include @@ -13,6 +12,8 @@ #include "geo/polyline.h" #include "utl/enumerate.h" +#include "utl/helpers/algorithm.h" +#include "utl/parallel_for.h" #include "utl/progress_tracker.h" #include "utl/verify.h" @@ -20,6 +21,7 @@ #include "nigiri/rt/frun.h" #include "nigiri/shapes_storage.h" #include "nigiri/stop.h" +#include "nigiri/timetable.h" namespace nigiri::loader::gtfs { @@ -108,14 +110,9 @@ shape_prepare::shape_prepare(shape_loader_state const& states, } auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); auto& results = shape_results_[idx].results_; - if (std::all_of( -#if __cpp_lib_execution - std::execution::par_unseq, -#endif - results.begin(), results.end(), - [&](shape_results::result const& it) { - return *it.stop_seq_ != trip.stop_seq_; - })) { + if (utl::all_of(results, [&](shape_results::result const& it) { + return *it.stop_seq_ != trip.stop_seq_; + })) { auto const distances = trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; results.emplace_back(&trip.stop_seq_, std::move(distances)); @@ -130,79 +127,72 @@ void shape_prepare::calculate_results(timetable const& tt, progress_tracker->status("Calculating shape offsets") .out_bounds(98.F, 99.F) .in_high(shape_results_.size()); - std::for_each( -#if __cpp_lib_execution - std::execution::par_unseq, -#endif - shape_results_.begin(), shape_results_.end(), - [&](shape_results& results) { - progress_tracker->increment(); - if (results.results_.empty()) { - return; + utl::parallel_for(shape_results_, [&](shape_results& results) { + progress_tracker->increment(); + if (results.results_.empty()) { + return; + } + auto const shape = shapes_.get_shape(results.shape_idx_); + auto const& shape_distances = + states.distances_[cista::to_idx(results.shape_idx_ - index_offset_)]; + for (auto& result : results.results_) { + auto const offsets = [&]() { + if (!shape_distances.empty() && result.distances_ != nullptr) { + return get_offsets_by_dist_traveled(*result.distances_, + shape_distances); + } + if (shape.size() < result.stop_seq_->size()) { + return std::vector{}; } - auto const shape = shapes_.get_shape(results.shape_idx_); - auto const& shape_distances = - states - .distances_[cista::to_idx(results.shape_idx_ - index_offset_)]; - for (auto& result : results.results_) { - auto const offsets = [&]() { - if (!shape_distances.empty() && result.distances_ != nullptr) { - return get_offsets_by_dist_traveled(*result.distances_, - shape_distances); + + return get_offsets_by_stops(tt, shape, *result.stop_seq_); + }(); + if (!offsets.empty()) { + auto const guard = std::lock_guard{m}; + result.offset_idx_ = shapes_.add_offsets(offsets); + } else { + result.offset_idx_ = shape_offset_idx_t::invalid(); + } + result.boxes_ = [&]() { + // Store box of full shape at index 0 + auto boxes = + offsets.empty() + ? std::vector(1) + : std::vector(result.stop_seq_->size() - 1 + 1); + auto& shape_box = boxes.front(); + auto last_extend = 0UL; + if (!offsets.empty()) { + for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { + auto& segment_box = boxes[i + 1U]; + auto const& [from, to] = pair; + for (auto const point : + shape.subspan(cista::to_idx(from), + cista::to_idx(to) - cista::to_idx(from) + 1)) { + shape_box.extend(point); + segment_box.extend(point); } - if (shape.size() < result.stop_seq_->size()) { - return std::vector{}; + auto const from_l = + tt.locations_ + .coordinates_[stop{(*result.stop_seq_)[i]}.location_idx()]; + auto const to_l = + tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} + .location_idx()]; + auto const stop_box = geo::make_box({from_l, to_l}); + if (!stop_box.contains(segment_box)) { + last_extend = i + 1U; } - - return get_offsets_by_stops(tt, shape, *result.stop_seq_); - }(); - if (!offsets.empty()) { - auto const guard = std::lock_guard{m}; - result.offset_idx_ = shapes_.add_offsets(offsets); - } else { - result.offset_idx_ = shape_offset_idx_t::invalid(); } - result.boxes_ = [&]() { - // Store box of full shape at index 0 - auto boxes = - offsets.empty() - ? std::vector(1) - : std::vector(result.stop_seq_->size() - 1 + 1); - auto& shape_box = boxes.front(); - auto last_extend = 0UL; - if (!offsets.empty()) { - for (auto const [i, pair] : - utl::enumerate(utl::pairwise(offsets))) { - auto& segment_box = boxes[i + 1U]; - auto const& [from, to] = pair; - for (auto const point : shape.subspan( - cista::to_idx(from), - cista::to_idx(to) - cista::to_idx(from) + 1)) { - shape_box.extend(point); - segment_box.extend(point); - } - auto const from_l = - tt.locations_.coordinates_[stop{(*result.stop_seq_)[i]} - .location_idx()]; - auto const to_l = - tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} - .location_idx()]; - auto const stop_box = geo::make_box({from_l, to_l}); - if (!stop_box.contains(segment_box)) { - last_extend = i + 1U; - } - } - } else { - for (auto const s : *result.stop_seq_) { - shape_box.extend( - tt.locations_.coordinates_[stop{s}.location_idx()]); - } - } - boxes.reserve(last_extend + 1); - return boxes; - }(); + } else { + for (auto const s : *result.stop_seq_) { + shape_box.extend( + tt.locations_.coordinates_[stop{s}.location_idx()]); + } } - }); + boxes.reserve(last_extend + 1); + return boxes; + }(); + } + }); results_ready_ = true; } @@ -223,13 +213,11 @@ void shape_prepare::write_trip_shape_offsets( } auto const& results = shape_results_[cista::to_idx(shape_idx - index_offset_)].results_; - auto const result = std::find_if( -#if __cpp_lib_execution - std::execution::par_unseq, -#endif - begin(results), end(results), [&](shape_results::result const& res) { + auto const result = + utl::find_if(results, [&](shape_results::result const& res) { return *res.stop_seq_ == trip.stop_seq_; }); + assert(result != end(results)); return result->offset_idx_; }(); shapes_.add_trip_shape_offsets(trip_idx, @@ -239,68 +227,55 @@ void shape_prepare::write_trip_shape_offsets( void shape_prepare::write_route_boxes(timetable const& tt) const { utl::verify(results_ready_, "Operation requires calculated results"); - auto const new_routes = - interval{static_cast(shapes_.boxes_.size()), - static_cast(tt.route_transport_ranges_.size())}; - auto route_boxes = - std::vector>(cista::to_idx(new_routes.size())); - std::transform( -#if __cpp_lib_execution - std::execution::par, -#endif - begin(new_routes), end(new_routes), begin(route_boxes), - [&](route_idx_t const r) { - auto const seq = tt.route_location_seq_[r]; - assert(seq.size() > 0U); - auto boxes = std::vector(seq.size() - 1 + 1); - auto last_extend = 1UL; - auto& bounding_box = boxes[0U]; - // 0: bounding box for route, 1-N: bounding box for each segment - auto const stop_indices = - interval{stop_idx_t{0U}, static_cast(seq.size())}; - for (auto const transport_idx : tt.route_transport_ranges_[r]) { - auto const frun = - rt::frun{tt, nullptr, - rt::run{.t_ = transport{transport_idx}, - .stop_range_ = stop_indices, - .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip([&](trip_idx_t const trip_idx, - interval const absolute_range) { - auto const [shape_idx, offset_idx] = - shapes_.trip_offset_indices_[trip_idx]; - if (shape_idx == shape_idx_t::invalid() || - offset_idx == shape_offset_idx_t::invalid()) { - for (auto const idx : absolute_range) { - bounding_box.extend( - tt.locations_.coordinates_[stop{seq[idx]}.location_idx()]); - } - } else { - auto const& results = - shape_results_[cista::to_idx(shape_idx - index_offset_)] - .results_; - auto const& result = std::find_if( -#if __cpp_lib_execution - std::execution::par_unseq, -#endif - begin(results), end(results), - [&](shape_results::result const& res) { - return res.offset_idx_ == offset_idx; - }); - bounding_box.extend(result->boxes_[0]); - for (auto i = 1U; i < result->boxes_.size(); ++i) { - boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = - result->boxes_[i]; - } - last_extend = - std::max(last_extend, - static_cast(result->boxes_.size() + + auto const route_offset = shapes_.boxes_.size(); + auto const routes_count = tt.route_transport_ranges_.size() - route_offset; + auto route_boxes = std::vector>(routes_count); + utl::parallel_for(std::views::iota(0U, routes_count), [&](std::size_t const + box_idx) { + auto const r = static_cast(box_idx + route_offset); + auto const seq = tt.route_location_seq_[r]; + assert(seq.size() > 0U); + auto boxes = std::vector(seq.size() - 1 + 1); + auto last_extend = 1UL; + auto& bounding_box = boxes[0U]; + // 0: bounding box for route, 1-N: bounding box for each segment + auto const stop_indices = + interval{stop_idx_t{0U}, static_cast(seq.size())}; + for (auto const transport_idx : tt.route_transport_ranges_[r]) { + auto const frun = rt::frun{tt, nullptr, + rt::run{.t_ = transport{transport_idx}, + .stop_range_ = stop_indices, + .rt_ = rt_transport_idx_t::invalid()}}; + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + for (auto const idx : absolute_range) { + bounding_box.extend( + tt.locations_.coordinates_[stop{seq[idx]}.location_idx()]); + } + } else { + auto const& result = utl::find_if( + shape_results_[cista::to_idx(shape_idx - index_offset_)].results_, + [&](shape_results::result const& res) { + return res.offset_idx_ == offset_idx; + }); + bounding_box.extend(result->boxes_[0]); + for (auto i = 1U; i < result->boxes_.size(); ++i) { + boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = + result->boxes_[i]; + } + last_extend = std::max( + last_extend, static_cast(result->boxes_.size() + absolute_range.from_)); - } - }); } - boxes.resize(last_extend); - return boxes; }); + } + boxes.resize(last_extend); + route_boxes[box_idx] = std::move(boxes); + }); for (auto const& boxes : route_boxes) { shapes_.boxes_.emplace_back(boxes); } From 55925f9abc1f1a84d33065820dfe856d750f54d2 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:06:50 +0100 Subject: [PATCH 34/60] Improve method names --- include/nigiri/loader/gtfs/shape_prepare.h | 5 +++-- src/loader/gtfs/load_timetable.cc | 4 ++-- src/loader/gtfs/shape_prepare.cc | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 11f0fcd6..fcf707ff 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -32,8 +32,9 @@ struct shape_prepare { vector_map const&, shapes_storage&); void calculate_results(timetable const&, shape_loader_state const&); - void write_trip_shape_offsets(vector_map const&) const; - void write_route_boxes(timetable const&) const; + void create_trip_shape_offsets( + vector_map const&) const; + void create_route_boxes(timetable const&) const; shape_idx_t index_offset_; std::vector shape_results_; diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 2b123212..9f583f5a 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -270,7 +270,7 @@ void load_timetable(loader_config const& config, shape_prepare{shape_states, trip_data.data_, *shapes_data}); if (shape_results.has_value()) { shape_results->calculate_results(tt, std::move(shape_states)); - shape_results->write_trip_shape_offsets(trip_data.data_); + shape_results->create_trip_shape_offsets(trip_data.data_); } auto const timer = scoped_timer{"loader.gtfs.routes.build"}; @@ -377,7 +377,7 @@ void load_timetable(loader_config const& config, // Build bounding boxes if (shape_results.has_value()) { - shape_results->write_route_boxes(tt); + shape_results->create_route_boxes(tt); } // Build location_routes map diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 0db64c91..719a019c 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -196,7 +196,7 @@ void shape_prepare::calculate_results(timetable const& tt, results_ready_ = true; } -void shape_prepare::write_trip_shape_offsets( +void shape_prepare::create_trip_shape_offsets( vector_map const& trips) const { utl::verify(results_ready_, "Operation requires calculated results"); auto const progress_tracker = utl::get_active_progress_tracker(); @@ -225,7 +225,7 @@ void shape_prepare::write_trip_shape_offsets( } } -void shape_prepare::write_route_boxes(timetable const& tt) const { +void shape_prepare::create_route_boxes(timetable const& tt) const { utl::verify(results_ready_, "Operation requires calculated results"); auto const route_offset = shapes_.boxes_.size(); auto const routes_count = tt.route_transport_ranges_.size() - route_offset; From b3272d808ae393405aafa82c7aea7f2b359666f4 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:31:10 +0100 Subject: [PATCH 35/60] Split shape boxes into two files --- include/nigiri/loader/gtfs/shape_prepare.h | 3 +- include/nigiri/shapes_storage.h | 3 +- src/loader/gtfs/shape_prepare.cc | 105 ++++++++++----------- src/shapes_storage.cc | 20 ++-- 4 files changed, 66 insertions(+), 65 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index fcf707ff..f247ac33 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -21,7 +21,8 @@ struct shape_prepare { stop_seq_t const* stop_seq_; std::vector const* distances_; shape_offset_idx_t offset_idx_; - std::vector boxes_; + geo::box trip_box_; + std::vector segment_boxes_; }; shape_idx_t shape_idx_; diff --git a/include/nigiri/shapes_storage.h b/include/nigiri/shapes_storage.h index 2bc7ac01..50eaf9c3 100644 --- a/include/nigiri/shapes_storage.h +++ b/include/nigiri/shapes_storage.h @@ -36,7 +36,8 @@ struct shapes_storage { mm_vecvec offsets_; mm_vec_map> trip_offset_indices_; - mm_vecvec boxes_; + mm_vec_map route_boxes_; + mm_vecvec segment_boxes_; }; } // namespace nigiri \ No newline at end of file diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 719a019c..59d7722b 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -92,7 +92,8 @@ shape_prepare::shape_results::result::result( : stop_seq_{stop_seq}, distances_{distances}, offset_idx_{shape_offset_idx_t::invalid()}, - boxes_{} {} + trip_box_{}, + segment_boxes_{} {} shape_prepare::shape_prepare(shape_loader_state const& states, vector_map const& trips, @@ -136,6 +137,7 @@ void shape_prepare::calculate_results(timetable const& tt, auto const& shape_distances = states.distances_[cista::to_idx(results.shape_idx_ - index_offset_)]; for (auto& result : results.results_) { + auto const offsets = [&]() { if (!shape_distances.empty() && result.distances_ != nullptr) { return get_offsets_by_dist_traveled(*result.distances_, @@ -147,50 +149,44 @@ void shape_prepare::calculate_results(timetable const& tt, return get_offsets_by_stops(tt, shape, *result.stop_seq_); }(); + if (!offsets.empty()) { auto const guard = std::lock_guard{m}; result.offset_idx_ = shapes_.add_offsets(offsets); } else { result.offset_idx_ = shape_offset_idx_t::invalid(); } - result.boxes_ = [&]() { - // Store box of full shape at index 0 - auto boxes = - offsets.empty() - ? std::vector(1) - : std::vector(result.stop_seq_->size() - 1 + 1); - auto& shape_box = boxes.front(); - auto last_extend = 0UL; - if (!offsets.empty()) { - for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { - auto& segment_box = boxes[i + 1U]; - auto const& [from, to] = pair; - for (auto const point : - shape.subspan(cista::to_idx(from), - cista::to_idx(to) - cista::to_idx(from) + 1)) { - shape_box.extend(point); - segment_box.extend(point); - } - auto const from_l = - tt.locations_ - .coordinates_[stop{(*result.stop_seq_)[i]}.location_idx()]; - auto const to_l = - tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} - .location_idx()]; - auto const stop_box = geo::make_box({from_l, to_l}); - if (!stop_box.contains(segment_box)) { - last_extend = i + 1U; - } + + if (!offsets.empty()) { + result.segment_boxes_ = std::vector(offsets.size() - 1); + auto box_count = 0UL; + for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { + auto& segment_box = result.segment_boxes_[i]; + auto const& [from, to] = pair; + for (auto const point : + shape.subspan(cista::to_idx(from), + cista::to_idx(to) - cista::to_idx(from) + 1)) { + result.trip_box_.extend(point); + segment_box.extend(point); } - } else { - for (auto const s : *result.stop_seq_) { - shape_box.extend( - tt.locations_.coordinates_[stop{s}.location_idx()]); + auto const from_l = + tt.locations_ + .coordinates_[stop{(*result.stop_seq_)[i]}.location_idx()]; + auto const to_l = + tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} + .location_idx()]; + auto const stop_box = geo::make_box({from_l, to_l}); + if (!stop_box.contains(segment_box)) { + box_count = i + 1U; } } - boxes.reserve(last_extend + 1); - return boxes; - }(); + result.segment_boxes_.resize(box_count); + } else { + for (auto const s : *result.stop_seq_) { + result.trip_box_.extend( + tt.locations_.coordinates_[stop{s}.location_idx()]); + } + } } }); results_ready_ = true; @@ -227,18 +223,18 @@ void shape_prepare::create_trip_shape_offsets( void shape_prepare::create_route_boxes(timetable const& tt) const { utl::verify(results_ready_, "Operation requires calculated results"); - auto const route_offset = shapes_.boxes_.size(); + auto const route_offset = shapes_.route_boxes_.size(); auto const routes_count = tt.route_transport_ranges_.size() - route_offset; - auto route_boxes = std::vector>(routes_count); + auto route_boxes = std::vector(routes_count); + auto route_segment_boxes = std::vector>(routes_count); utl::parallel_for(std::views::iota(0U, routes_count), [&](std::size_t const box_idx) { auto const r = static_cast(box_idx + route_offset); auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); - auto boxes = std::vector(seq.size() - 1 + 1); - auto last_extend = 1UL; - auto& bounding_box = boxes[0U]; - // 0: bounding box for route, 1-N: bounding box for each segment + auto bounding_box = geo::box{}; + auto segment_boxes = std::vector(seq.size() - 1); + auto box_count = 0UL; auto const stop_indices = interval{stop_idx_t{0U}, static_cast(seq.size())}; for (auto const transport_idx : tt.route_transport_ranges_[r]) { @@ -262,22 +258,25 @@ void shape_prepare::create_route_boxes(timetable const& tt) const { [&](shape_results::result const& res) { return res.offset_idx_ == offset_idx; }); - bounding_box.extend(result->boxes_[0]); - for (auto i = 1U; i < result->boxes_.size(); ++i) { - boxes[(i - 1) + cista::to_idx(absolute_range.from_) + 1] = - result->boxes_[i]; + bounding_box.extend(result->trip_box_); + for (auto i = 0U; i < result->segment_boxes_.size(); ++i) { + segment_boxes[i + cista::to_idx(absolute_range.from_)] = + result->segment_boxes_[i]; } - last_extend = std::max( - last_extend, static_cast(result->boxes_.size() + - absolute_range.from_)); + box_count = std::max(box_count, static_cast( + result->segment_boxes_.size() + + absolute_range.from_)); } }); } - boxes.resize(last_extend); - route_boxes[box_idx] = std::move(boxes); + segment_boxes.resize(box_count); + route_boxes[box_idx] = std::move(bounding_box); + route_segment_boxes[box_idx] = std::move(segment_boxes); }); - for (auto const& boxes : route_boxes) { - shapes_.boxes_.emplace_back(boxes); + for (auto const [route_box, segment_boxes] : + utl::zip(route_boxes, route_segment_boxes)) { + shapes_.route_boxes_.emplace_back(route_box); + shapes_.segment_boxes_.emplace_back(segment_boxes); } } } // namespace nigiri::loader::gtfs \ No newline at end of file diff --git a/src/shapes_storage.cc b/src/shapes_storage.cc index 719fb0b0..95eff4e5 100644 --- a/src/shapes_storage.cc +++ b/src/shapes_storage.cc @@ -28,8 +28,10 @@ shapes_storage::shapes_storage(std::filesystem::path path, offsets_{mm_vec{mm("shape_offsets_data.bin")}, mm_vec{mm("shape_offsets_idx.bin")}}, trip_offset_indices_{mm("shape_trip_offsets.bin")}, - boxes_{mm_vec{mm("shape_boxes_data.bin")}, - mm_vec{mm("shape_boxes_idx.bin")}} {} + route_boxes_{mm("shape_route_boxes.bin")}, + segment_boxes_{mm_vec{mm("shape_segment_boxes_data.bin")}, + mm_vec{mm("shape_segment_boxes_idx.bin")}} { +} cista::mmap shapes_storage::mm(char const* file) { return cista::mmap{(p_ / file).generic_string().c_str(), mode_}; @@ -93,21 +95,19 @@ void shapes_storage::add_trip_shape_offsets( } geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { - utl::verify(cista::to_idx(route_idx) < boxes_.size(), + utl::verify(cista::to_idx(route_idx) < route_boxes_.size(), "Route index {} is out of bounds", route_idx); - // 0: bounding box for trip - return boxes_[route_idx][0]; + return route_boxes_[route_idx]; } std::optional shapes_storage::get_bounding_box( nigiri::route_idx_t const route_idx, std::size_t const segment) const { - utl::verify(cista::to_idx(route_idx) < boxes_.size(), + utl::verify(cista::to_idx(route_idx) < segment_boxes_.size(), "Route index {} is out of bounds", route_idx); - auto const& boxes = boxes_[route_idx]; - // 1-N: bounding box for segment - return segment + 1 < boxes.size() ? boxes[segment + 1] - : std::optional{std::nullopt}; + auto const& boxes = segment_boxes_[route_idx]; + return segment < boxes.size() ? boxes[segment] + : std::optional{std::nullopt}; } } // namespace nigiri \ No newline at end of file From e1e857861ddb057d582058b0c98d4d8496e16e8d Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:32:41 +0100 Subject: [PATCH 36/60] Simplify parallel usage --- src/loader/gtfs/shape_prepare.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 59d7722b..66760d14 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -227,8 +227,7 @@ void shape_prepare::create_route_boxes(timetable const& tt) const { auto const routes_count = tt.route_transport_ranges_.size() - route_offset; auto route_boxes = std::vector(routes_count); auto route_segment_boxes = std::vector>(routes_count); - utl::parallel_for(std::views::iota(0U, routes_count), [&](std::size_t const - box_idx) { + utl::parallel_for_run(route_boxes.size(), [&](std::size_t const box_idx) { auto const r = static_cast(box_idx + route_offset); auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); From 53e9a9e29a16e3ceb6c126e060d920d12bb21dc7 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Wed, 6 Nov 2024 13:37:56 +0100 Subject: [PATCH 37/60] Rename variables --- include/nigiri/loader/gtfs/shape_prepare.h | 6 +- include/nigiri/shapes_storage.h | 4 +- src/loader/gtfs/load_timetable.cc | 2 +- src/loader/gtfs/shape_prepare.cc | 70 +++++++++++----------- src/shapes_storage.cc | 20 +++---- 5 files changed, 52 insertions(+), 50 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index f247ac33..56829134 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -21,8 +21,8 @@ struct shape_prepare { stop_seq_t const* stop_seq_; std::vector const* distances_; shape_offset_idx_t offset_idx_; - geo::box trip_box_; - std::vector segment_boxes_; + geo::box trip_bbox_; + std::vector segment_bboxes_; }; shape_idx_t shape_idx_; @@ -35,7 +35,7 @@ struct shape_prepare { void calculate_results(timetable const&, shape_loader_state const&); void create_trip_shape_offsets( vector_map const&) const; - void create_route_boxes(timetable const&) const; + void create_route_bounding_boxes(timetable const&) const; shape_idx_t index_offset_; std::vector shape_results_; diff --git a/include/nigiri/shapes_storage.h b/include/nigiri/shapes_storage.h index 50eaf9c3..c9050175 100644 --- a/include/nigiri/shapes_storage.h +++ b/include/nigiri/shapes_storage.h @@ -36,8 +36,8 @@ struct shapes_storage { mm_vecvec offsets_; mm_vec_map> trip_offset_indices_; - mm_vec_map route_boxes_; - mm_vecvec segment_boxes_; + mm_vec_map route_bboxes_; + mm_vecvec route_segment_bboxes_; }; } // namespace nigiri \ No newline at end of file diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 9f583f5a..fbcaa277 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -377,7 +377,7 @@ void load_timetable(loader_config const& config, // Build bounding boxes if (shape_results.has_value()) { - shape_results->create_route_boxes(tt); + shape_results->create_route_bounding_boxes(tt); } // Build location_routes map diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 66760d14..7b34c458 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -92,8 +92,8 @@ shape_prepare::shape_results::result::result( : stop_seq_{stop_seq}, distances_{distances}, offset_idx_{shape_offset_idx_t::invalid()}, - trip_box_{}, - segment_boxes_{} {} + trip_bbox_{}, + segment_bboxes_{} {} shape_prepare::shape_prepare(shape_loader_state const& states, vector_map const& trips, @@ -158,16 +158,16 @@ void shape_prepare::calculate_results(timetable const& tt, } if (!offsets.empty()) { - result.segment_boxes_ = std::vector(offsets.size() - 1); - auto box_count = 0UL; + result.segment_bboxes_ = std::vector(offsets.size() - 1); + auto bbox_count = 0UL; for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { - auto& segment_box = result.segment_boxes_[i]; + auto& segment_bbox = result.segment_bboxes_[i]; auto const& [from, to] = pair; for (auto const point : shape.subspan(cista::to_idx(from), cista::to_idx(to) - cista::to_idx(from) + 1)) { - result.trip_box_.extend(point); - segment_box.extend(point); + result.trip_bbox_.extend(point); + segment_bbox.extend(point); } auto const from_l = tt.locations_ @@ -175,15 +175,15 @@ void shape_prepare::calculate_results(timetable const& tt, auto const to_l = tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} .location_idx()]; - auto const stop_box = geo::make_box({from_l, to_l}); - if (!stop_box.contains(segment_box)) { - box_count = i + 1U; + auto const stop_bbox = geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(segment_bbox)) { + bbox_count = i + 1U; } } - result.segment_boxes_.resize(box_count); + result.segment_bboxes_.resize(bbox_count); } else { for (auto const s : *result.stop_seq_) { - result.trip_box_.extend( + result.trip_bbox_.extend( tt.locations_.coordinates_[stop{s}.location_idx()]); } } @@ -221,19 +221,21 @@ void shape_prepare::create_trip_shape_offsets( } } -void shape_prepare::create_route_boxes(timetable const& tt) const { +void shape_prepare::create_route_bounding_boxes(timetable const& tt) const { utl::verify(results_ready_, "Operation requires calculated results"); - auto const route_offset = shapes_.route_boxes_.size(); + auto const route_offset = shapes_.route_bboxes_.size(); auto const routes_count = tt.route_transport_ranges_.size() - route_offset; - auto route_boxes = std::vector(routes_count); - auto route_segment_boxes = std::vector>(routes_count); - utl::parallel_for_run(route_boxes.size(), [&](std::size_t const box_idx) { - auto const r = static_cast(box_idx + route_offset); + auto route_bboxes = std::vector(routes_count); + auto route_segment_bboxes = std::vector>(routes_count); + utl::parallel_for_run(route_bboxes.size(), [&](std::size_t const bbox_idx) { + auto const r = static_cast(bbox_idx + route_offset); auto const seq = tt.route_location_seq_[r]; assert(seq.size() > 0U); auto bounding_box = geo::box{}; - auto segment_boxes = std::vector(seq.size() - 1); - auto box_count = 0UL; + auto segment_bboxes = std::vector(seq.size() - 1); + auto bbox_count = 0UL; + auto processed = + std::vector>{}; auto const stop_indices = interval{stop_idx_t{0U}, static_cast(seq.size())}; for (auto const transport_idx : tt.route_transport_ranges_[r]) { @@ -257,25 +259,25 @@ void shape_prepare::create_route_boxes(timetable const& tt) const { [&](shape_results::result const& res) { return res.offset_idx_ == offset_idx; }); - bounding_box.extend(result->trip_box_); - for (auto i = 0U; i < result->segment_boxes_.size(); ++i) { - segment_boxes[i + cista::to_idx(absolute_range.from_)] = - result->segment_boxes_[i]; + bounding_box.extend(result->trip_bbox_); + for (auto i = 0U; i < result->segment_bboxes_.size(); ++i) { + segment_bboxes[i + cista::to_idx(absolute_range.from_)] = + result->segment_bboxes_[i]; } - box_count = std::max(box_count, static_cast( - result->segment_boxes_.size() + - absolute_range.from_)); + bbox_count = std::max(bbox_count, static_cast( + result->segment_bboxes_.size() + + absolute_range.from_)); } }); } - segment_boxes.resize(box_count); - route_boxes[box_idx] = std::move(bounding_box); - route_segment_boxes[box_idx] = std::move(segment_boxes); + segment_bboxes.resize(bbox_count); + route_bboxes[bbox_idx] = std::move(bounding_box); + route_segment_bboxes[bbox_idx] = std::move(segment_bboxes); }); - for (auto const [route_box, segment_boxes] : - utl::zip(route_boxes, route_segment_boxes)) { - shapes_.route_boxes_.emplace_back(route_box); - shapes_.segment_boxes_.emplace_back(segment_boxes); + for (auto const [route_bbox, segment_bboxes] : + utl::zip(route_bboxes, route_segment_bboxes)) { + shapes_.route_bboxes_.emplace_back(route_bbox); + shapes_.route_segment_bboxes_.emplace_back(segment_bboxes); } } } // namespace nigiri::loader::gtfs \ No newline at end of file diff --git a/src/shapes_storage.cc b/src/shapes_storage.cc index 95eff4e5..969262b2 100644 --- a/src/shapes_storage.cc +++ b/src/shapes_storage.cc @@ -28,10 +28,10 @@ shapes_storage::shapes_storage(std::filesystem::path path, offsets_{mm_vec{mm("shape_offsets_data.bin")}, mm_vec{mm("shape_offsets_idx.bin")}}, trip_offset_indices_{mm("shape_trip_offsets.bin")}, - route_boxes_{mm("shape_route_boxes.bin")}, - segment_boxes_{mm_vec{mm("shape_segment_boxes_data.bin")}, - mm_vec{mm("shape_segment_boxes_idx.bin")}} { -} + route_bboxes_{mm("shape_route_bboxes.bin")}, + route_segment_bboxes_{ + mm_vec{mm("shape_route_segment_bboxes_data.bin")}, + mm_vec{mm("shape_route_segment_bboxes_idx.bin")}} {} cista::mmap shapes_storage::mm(char const* file) { return cista::mmap{(p_ / file).generic_string().c_str(), mode_}; @@ -95,19 +95,19 @@ void shapes_storage::add_trip_shape_offsets( } geo::box shapes_storage::get_bounding_box(route_idx_t const route_idx) const { - utl::verify(cista::to_idx(route_idx) < route_boxes_.size(), + utl::verify(cista::to_idx(route_idx) < route_bboxes_.size(), "Route index {} is out of bounds", route_idx); - return route_boxes_[route_idx]; + return route_bboxes_[route_idx]; } std::optional shapes_storage::get_bounding_box( nigiri::route_idx_t const route_idx, std::size_t const segment) const { - utl::verify(cista::to_idx(route_idx) < segment_boxes_.size(), + utl::verify(cista::to_idx(route_idx) < route_segment_bboxes_.size(), "Route index {} is out of bounds", route_idx); - auto const& boxes = segment_boxes_[route_idx]; - return segment < boxes.size() ? boxes[segment] - : std::optional{std::nullopt}; + auto const& bboxes = route_segment_bboxes_[route_idx]; + return segment < bboxes.size() ? bboxes[segment] + : std::optional{std::nullopt}; } } // namespace nigiri \ No newline at end of file From ec8f3edac9164f3fab962d7de8b06d7c6c9cf8fd Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:20:08 +0100 Subject: [PATCH 38/60] WIP: Change code to tasks Warning: Dangling references will cause use after free --- include/nigiri/loader/gtfs/shape_prepare.h | 6 + src/loader/gtfs/load_timetable.cc | 3 + src/loader/gtfs/shape_prepare.cc | 155 +++++++++++++++++++++ 3 files changed, 164 insertions(+) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index 56829134..aa8035e0 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -43,4 +43,10 @@ struct shape_prepare { bool results_ready_; }; +void calculate_shape_offsets_and_bboxes( + timetable const&, + shapes_storage&, + vector_map const&, + shape_loader_state const&); + } // namespace nigiri::loader::gtfs diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index fbcaa277..ed3d7787 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -375,6 +375,9 @@ void load_timetable(loader_config const& config, progress_tracker->increment(); } + if (shapes_data != nullptr) { + calculate_shape_offsets_and_bboxes(tt, *shapes_data, trip_data.data_, shape_states); + } // Build bounding boxes if (shape_results.has_value()) { shape_results->create_route_bounding_boxes(tt); diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 7b34c458..554eb87c 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -14,10 +14,13 @@ #include "utl/enumerate.h" #include "utl/helpers/algorithm.h" #include "utl/parallel_for.h" +#include "utl/pipes/transform.h" +#include "utl/pipes/vec.h" #include "utl/progress_tracker.h" #include "utl/verify.h" #include "nigiri/loader/gtfs/shape.h" +#include "nigiri/loader/gtfs/trip.h" #include "nigiri/rt/frun.h" #include "nigiri/shapes_storage.h" #include "nigiri/stop.h" @@ -280,4 +283,156 @@ void shape_prepare::create_route_bounding_boxes(timetable const& tt) const { shapes_.route_segment_bboxes_.emplace_back(segment_bboxes); } } + +struct offset_task { + struct results { + struct result { + std::vector offsets_; + geo::box trip_bbox; + std::vector segment_bboxes_; + }; + // shape_idx_t shape_idx_; + std::vector results_; + }; + std::function()> task_; + std::optional results_; +}; + +std::vector create_offset_tasks( + timetable const& tt, + shapes_storage& shapes_data, + vector_map const& trips, + shape_loader_state const& states) { + using input_pair = cista::pair const*>; + struct input_data { + shape_idx_t shape_idx_; + std::vector inputs_; + }; + + auto const index_offset = cista::to_idx(states.index_offset_); + auto inputs = std::vector(states.id_map_.size()); + for (auto [idx, input] : utl::enumerate(inputs)) { + input.shape_idx_ = static_cast(idx + index_offset); + } + for (auto const& trip : trips) { + if (trip.shape_idx_ == shape_idx_t::invalid()) { + continue; + } + auto const idx = cista::to_idx(trip.shape_idx_ - index_offset); + auto& shape_inputs = inputs[idx].inputs_; + auto const it = std::ranges::lower_bound( + shape_inputs, trip.stop_seq_, + [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, + [](input_pair const& p) { return *p.first; }); + if (it != end(shape_inputs) && *it->first == trip.stop_seq_) { + continue; + } + auto const distances = + trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; + shape_inputs.emplace(it, &trip.stop_seq_, std::move(distances)); + } + + return utl::all(std::move(inputs)) // + | + utl::transform([&tt, &shapes_data, &states, + index_offset](input_data const& input) { + return offset_task{ + .task_ = + [&]() { + auto const shape = shapes_data.get_shape(input.shape_idx_); + auto const& shape_distances = + states.distances_[cista::to_idx(input.shape_idx_ - + index_offset)]; + return std::make_optional({ + // .results_ = utl::all(std::move(input.inputs_)) // + .results_ = + utl::all(input.inputs_) // + | + utl::transform( + [&](input_pair const& pair) + -> offset_task::results::result { + auto const& [stop_seq, distances] = pair; + // Calculate offsets + auto const offsets = [&]() { + if (!shape_distances.empty() && + distances != nullptr) { + return get_offsets_by_dist_traveled( + *distances, shape_distances); + } + if (shape.size() < stop_seq->size()) { + return std::vector{}; + } + + return get_offsets_by_stops(tt, shape, + *stop_seq); + }(); + // Calculate bounding boxes + auto trip_bbox = geo::box{}; + auto segment_bboxes = + std::vector{}; + if (!offsets.empty()) { + segment_bboxes.resize( + (offsets.size() - 1)); + auto bbox_count = 0UL; + for (auto const [i, segment] : + utl::enumerate( + utl::pairwise(offsets))) { + auto& segment_bbox = segment_bboxes[i]; + auto const& [from, to] = segment; + for (auto const point : shape.subspan( + cista::to_idx(from), + cista::to_idx(to) - + cista::to_idx(from) + 1)) { + trip_bbox.extend(point); + segment_bbox.extend(point); + } + auto const from_l = + tt.locations_.coordinates_ + [stop{(*stop_seq)[i]} + .location_idx()]; + auto const to_l = + tt.locations_.coordinates_ + [stop{(*stop_seq)[i + 1]} + .location_idx()]; + auto const stop_bbox = + geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(segment_bbox)) { + bbox_count = i + 1U; + } + } + segment_bboxes.resize(bbox_count); + } else { + for (auto const s : *stop_seq) { + trip_bbox.extend( + tt.locations_.coordinates_ + [stop{s}.location_idx()]); + } + } + // + return offset_task::results::result{ + .offsets_ = std::move(offsets), + .trip_bbox = std::move(trip_bbox), + .segment_bboxes_ = + std::move(segment_bboxes), + }; + }) // + | utl::vec(), + }); + }, + .results_ = std::nullopt, + }; + }) // + | utl::vec(); +} + + +void calculate_shape_offsets_and_bboxes( + timetable const& tt, + shapes_storage& shapes_data, + vector_map const& trips, + shape_loader_state const& shape_states) { + auto offset_tasks = create_offset_tasks(tt, shapes_data, trips, shape_states); + utl::parallel_for(offset_tasks, + [](offset_task& task) { task.results_ = task.task_(); }); +} } // namespace nigiri::loader::gtfs \ No newline at end of file From 7234e20b2b1e88709b749839809ef992801711ec Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:31:00 +0100 Subject: [PATCH 39/60] Fix use after free --- src/loader/gtfs/shape_prepare.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 554eb87c..0474a232 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -334,17 +334,15 @@ std::vector create_offset_tasks( return utl::all(std::move(inputs)) // | - utl::transform([&tt, &shapes_data, &states, - index_offset](input_data const& input) { + utl::transform([&](input_data const& in) { return offset_task{ .task_ = - [&]() { + [&, input = in, index_offset]() { auto const shape = shapes_data.get_shape(input.shape_idx_); auto const& shape_distances = states.distances_[cista::to_idx(input.shape_idx_ - index_offset)]; return std::make_optional({ - // .results_ = utl::all(std::move(input.inputs_)) // .results_ = utl::all(input.inputs_) // | From e1e003ab03c0faeba279e91b8fd8bc712b3561f7 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:58:32 +0100 Subject: [PATCH 40/60] Create offset entries --- src/loader/gtfs/shape_prepare.cc | 48 +++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 0474a232..18f2563d 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -14,6 +15,7 @@ #include "utl/enumerate.h" #include "utl/helpers/algorithm.h" #include "utl/parallel_for.h" +#include "utl/pipes/all.h" #include "utl/pipes/transform.h" #include "utl/pipes/vec.h" #include "utl/progress_tracker.h" @@ -287,6 +289,7 @@ void shape_prepare::create_route_bounding_boxes(timetable const& tt) const { struct offset_task { struct results { struct result { + stop_seq_t const* stop_seq_; std::vector offsets_; geo::box trip_bbox; std::vector segment_bboxes_; @@ -408,6 +411,7 @@ std::vector create_offset_tasks( } // return offset_task::results::result{ + .stop_seq_ = pair.first, .offsets_ = std::move(offsets), .trip_bbox = std::move(trip_bbox), .segment_bboxes_ = @@ -423,6 +427,39 @@ std::vector create_offset_tasks( | utl::vec(); } +void assign_shape_offsets(shapes_storage& shapes_data, + vector_map const& trips, + std::vector tasks, + shape_idx_t const index_offset) { + assert(utl::all_of(tasks, [](offset_task const& task) { + return task.results_ != std::nullopt; + })); + for (auto const& trip : trips) { + auto const trip_idx = trip.trip_idx_; + auto const shape_idx = trip.shape_idx_; + if (shape_idx == shape_idx_t::invalid()) { + shapes_data.add_trip_shape_offsets( + trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); + } else { + auto const& results = + tasks[cista::to_idx(shape_idx - index_offset)].results_->results_; + auto const it = std::ranges::lower_bound( + results, trip.stop_seq_, + [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, + [](offset_task::results::result const& res) { + return *res.stop_seq_; + }); + if (it->offsets_.empty()) { + shapes_data.add_trip_shape_offsets( + trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); + } else { + shapes_data.add_trip_shape_offsets( + trip_idx, + cista::pair{shape_idx, shapes_data.add_offsets(it->offsets_)}); + } + } + } +} void calculate_shape_offsets_and_bboxes( timetable const& tt, @@ -430,7 +467,16 @@ void calculate_shape_offsets_and_bboxes( vector_map const& trips, shape_loader_state const& shape_states) { auto offset_tasks = create_offset_tasks(tt, shapes_data, trips, shape_states); + auto const offset_progress_tracker = utl::get_active_progress_tracker(); + offset_progress_tracker->status("Creating trip offsets") + .out_bounds(98.F, 99.F) + .in_high(trips.size()); utl::parallel_for(offset_tasks, - [](offset_task& task) { task.results_ = task.task_(); }); + [&offset_progress_tracker](offset_task& task) { + offset_progress_tracker->increment(); + task.results_ = task.task_(); + }); + assign_shape_offsets(shapes_data, trips, offset_tasks, + shape_states.index_offset_); } } // namespace nigiri::loader::gtfs \ No newline at end of file From ea7cddf93afd128c2a73786200e120eec98699aa Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:01:58 +0100 Subject: [PATCH 41/60] Create bounding boxes --- src/loader/gtfs/shape_prepare.cc | 143 ++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 13 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 18f2563d..95ac8591 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -290,8 +290,8 @@ struct offset_task { struct results { struct result { stop_seq_t const* stop_seq_; - std::vector offsets_; - geo::box trip_bbox; + shape_offset_idx_t shape_offset_idx_; + geo::box trip_bbox_; std::vector segment_bboxes_; }; // shape_idx_t shape_idx_; @@ -335,6 +335,7 @@ std::vector create_offset_tasks( shape_inputs.emplace(it, &trip.stop_seq_, std::move(distances)); } + auto m = std::mutex{}; return utl::all(std::move(inputs)) // | utl::transform([&](input_data const& in) { @@ -367,6 +368,15 @@ std::vector create_offset_tasks( return get_offsets_by_stops(tt, shape, *stop_seq); }(); + auto const shape_offset_idx = [&]() { + if (offsets.empty()) { + return shape_offset_idx_t::invalid(); + } else { + auto const guard = + std::lock_guard{m}; + return shapes_data.add_offsets(offsets); + } + }(); // Calculate bounding boxes auto trip_bbox = geo::box{}; auto segment_bboxes = @@ -412,8 +422,8 @@ std::vector create_offset_tasks( // return offset_task::results::result{ .stop_seq_ = pair.first, - .offsets_ = std::move(offsets), - .trip_bbox = std::move(trip_bbox), + .shape_offset_idx_ = shape_offset_idx, + .trip_bbox_ = std::move(trip_bbox), .segment_bboxes_ = std::move(segment_bboxes), }; @@ -449,18 +459,112 @@ void assign_shape_offsets(shapes_storage& shapes_data, [](offset_task::results::result const& res) { return *res.stop_seq_; }); - if (it->offsets_.empty()) { - shapes_data.add_trip_shape_offsets( - trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); - } else { - shapes_data.add_trip_shape_offsets( - trip_idx, - cista::pair{shape_idx, shapes_data.add_offsets(it->offsets_)}); - } + shapes_data.add_trip_shape_offsets( + trip_idx, cista::pair{shape_idx, it->shape_offset_idx_}); } } } +struct bbox_task { + struct result { + geo::box route_bbox_; + std::vector route_segment_bboxes_; + }; + std::function()> task_; + std::optional result_; +}; + +std::vector create_bbox_tasks(timetable const& tt, + shapes_storage& shapes_data, + std::vector const& tasks, + std::size_t const route_offset, + shape_idx_t const shape_offset) { + assert(utl::all_of(tasks, [](offset_task const& task) { + return task.results_ != std::nullopt; + })); + auto const new_routes = + interval{static_cast(route_offset), + static_cast(tt.route_transport_ranges_.size() - + route_offset)}; + return utl::all(new_routes) // + | + utl::transform([&](route_idx_t const& r) { + return bbox_task{ + .task_ = + [&, r, shape_offset]() { + auto const seq = tt.route_location_seq_[r]; + assert(seq.size() > 0U); + auto bounding_box = geo::box{}; + auto segment_bboxes = std::vector{}; + segment_bboxes.resize(seq.size() - 1); + auto bbox_count = 0UL; + auto const stop_indices = interval{ + stop_idx_t{0U}, static_cast(seq.size())}; + + for (auto const transport_idx : + tt.route_transport_ranges_[r]) { + auto const frun = rt::frun{ + tt, nullptr, + rt::run{.t_ = transport{transport_idx}, + .stop_range_ = stop_indices, + .rt_ = rt_transport_idx_t::invalid()}}; + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const + absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_data.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + for (auto const idx : absolute_range) { + bounding_box.extend( + tt.locations_.coordinates_ + [stop{seq[idx]}.location_idx()]); + } + } else { + auto const& results = + // TODO : INVALID OFFSET? + tasks[cista::to_idx(shape_idx - shape_offset)] + .results_->results_; + auto const result = std::ranges::lower_bound( + results, offset_idx, + [](shape_offset_idx_t const a, + shape_offset_idx_t const b) { return a < b; }, + [](offset_task::results::result const& res) { + return res.shape_offset_idx_; + }); + bounding_box.extend(result->trip_bbox_); + for (auto i = 0U; i < result->segment_bboxes_.size(); + ++i) { + segment_bboxes[i + cista::to_idx( + absolute_range.from_)] = + result->segment_bboxes_[i]; + } + bbox_count = std::max( + bbox_count, static_cast( + result->segment_bboxes_.size() + + absolute_range.from_)); + } + }); + } + + return std::make_optional(bbox_task::result{ + std::move(bounding_box), std::move(segment_bboxes)}); + }, + .result_ = std::nullopt, + }; + }) | + utl::vec(); +} + +void assign_bounding_boxes(shapes_storage& shapes_data, + std::vector const& tasks) { + for (auto const& task : tasks) { + shapes_data.route_bboxes_.emplace_back(task.result_->route_bbox_); + shapes_data.route_segment_bboxes_.emplace_back( + task.result_->route_segment_bboxes_); + } +} + void calculate_shape_offsets_and_bboxes( timetable const& tt, shapes_storage& shapes_data, @@ -470,7 +574,7 @@ void calculate_shape_offsets_and_bboxes( auto const offset_progress_tracker = utl::get_active_progress_tracker(); offset_progress_tracker->status("Creating trip offsets") .out_bounds(98.F, 99.F) - .in_high(trips.size()); + .in_high(offset_tasks.size()); utl::parallel_for(offset_tasks, [&offset_progress_tracker](offset_task& task) { offset_progress_tracker->increment(); @@ -478,5 +582,18 @@ void calculate_shape_offsets_and_bboxes( }); assign_shape_offsets(shapes_data, trips, offset_tasks, shape_states.index_offset_); + + auto bbox_tasks = create_bbox_tasks(tt, shapes_data, offset_tasks, + shapes_data.route_bboxes_.size(), + shape_states.index_offset_); + auto const bbox_progress_tracker = utl::get_active_progress_tracker(); + bbox_progress_tracker->status("Creating bounding boxes") + .out_bounds(99.F, 100.F) + .in_high(bbox_tasks.size()); + utl::parallel_for(bbox_tasks, [&bbox_progress_tracker](bbox_task& task) { + bbox_progress_tracker->increment(); + task.result_ = task.task_(); + }); + assign_bounding_boxes(shapes_data, bbox_tasks); } } // namespace nigiri::loader::gtfs \ No newline at end of file From 20dc322f05d18c4dfd5f288d18d95ce6dbe33a70 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:45:31 +0100 Subject: [PATCH 42/60] Remove mutex --- src/loader/gtfs/shape_prepare.cc | 93 +++++++++++++++++++------------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 95ac8591..838a82f6 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -290,11 +290,10 @@ struct offset_task { struct results { struct result { stop_seq_t const* stop_seq_; - shape_offset_idx_t shape_offset_idx_; + std::vector offsets_; geo::box trip_bbox_; std::vector segment_bboxes_; }; - // shape_idx_t shape_idx_; std::vector results_; }; std::function()> task_; @@ -335,7 +334,6 @@ std::vector create_offset_tasks( shape_inputs.emplace(it, &trip.stop_seq_, std::move(distances)); } - auto m = std::mutex{}; return utl::all(std::move(inputs)) // | utl::transform([&](input_data const& in) { @@ -368,15 +366,6 @@ std::vector create_offset_tasks( return get_offsets_by_stops(tt, shape, *stop_seq); }(); - auto const shape_offset_idx = [&]() { - if (offsets.empty()) { - return shape_offset_idx_t::invalid(); - } else { - auto const guard = - std::lock_guard{m}; - return shapes_data.add_offsets(offsets); - } - }(); // Calculate bounding boxes auto trip_bbox = geo::box{}; auto segment_bboxes = @@ -422,7 +411,7 @@ std::vector create_offset_tasks( // return offset_task::results::result{ .stop_seq_ = pair.first, - .shape_offset_idx_ = shape_offset_idx, + .offsets_ = std::move(offsets), .trip_bbox_ = std::move(trip_bbox), .segment_bboxes_ = std::move(segment_bboxes), @@ -437,13 +426,47 @@ std::vector create_offset_tasks( | utl::vec(); } -void assign_shape_offsets(shapes_storage& shapes_data, - vector_map const& trips, - std::vector tasks, - shape_idx_t const index_offset) { +struct shape_results { + struct trip { + stop_seq_t const* stop_seq_; + shape_offset_idx_t shape_offset_idx_; + geo::box trip_bbox_; + std::vector segment_bboxes_; + }; + std::vector trips_; +}; + +std::vector assign_shape_offsets( + shapes_storage& shapes_data, + vector_map const& trips, + std::vector&& tasks, + shape_idx_t const index_offset) { assert(utl::all_of(tasks, [](offset_task const& task) { return task.results_ != std::nullopt; })); + auto const entries = + utl::all(std::move(tasks)) // + | + utl::transform([&](offset_task const& task) { + return shape_results{ + .trips_ = + utl::all(task.results_->results_) // + | + utl::transform([&](offset_task::results::result const& result) { + return shape_results::trip{ + .stop_seq_ = result.stop_seq_, + .shape_offset_idx_ = + result.offsets_.empty() + ? shape_offset_idx_t::invalid() + : shapes_data.add_offsets(result.offsets_), + .trip_bbox_ = result.trip_bbox_, + .segment_bboxes_ = result.segment_bboxes_, + }; + }) // + | utl::vec(), + }; + }) | + utl::vec(); for (auto const& trip : trips) { auto const trip_idx = trip.trip_idx_; auto const shape_idx = trip.shape_idx_; @@ -452,17 +475,16 @@ void assign_shape_offsets(shapes_storage& shapes_data, trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); } else { auto const& results = - tasks[cista::to_idx(shape_idx - index_offset)].results_->results_; + entries[cista::to_idx(shape_idx - index_offset)].trips_; auto const it = std::ranges::lower_bound( results, trip.stop_seq_, [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, - [](offset_task::results::result const& res) { - return *res.stop_seq_; - }); + [](shape_results::trip const& entry) { return *entry.stop_seq_; }); shapes_data.add_trip_shape_offsets( trip_idx, cista::pair{shape_idx, it->shape_offset_idx_}); } } + return entries; } struct bbox_task { @@ -474,11 +496,12 @@ struct bbox_task { std::optional result_; }; -std::vector create_bbox_tasks(timetable const& tt, - shapes_storage& shapes_data, - std::vector const& tasks, - std::size_t const route_offset, - shape_idx_t const shape_offset) { +std::vector create_bbox_tasks( + timetable const& tt, + shapes_storage& shapes_data, + std::vector const& results, + std::size_t const route_offset, + shape_idx_t const shape_offset) { assert(utl::all_of(tasks, [](offset_task const& task) { return task.results_ != std::nullopt; })); @@ -521,16 +544,14 @@ std::vector create_bbox_tasks(timetable const& tt, [stop{seq[idx]}.location_idx()]); } } else { - auto const& results = - // TODO : INVALID OFFSET? - tasks[cista::to_idx(shape_idx - shape_offset)] - .results_->results_; auto const result = std::ranges::lower_bound( - results, offset_idx, + results[cista::to_idx(shape_idx - shape_offset)] + .trips_, + offset_idx, [](shape_offset_idx_t const a, shape_offset_idx_t const b) { return a < b; }, - [](offset_task::results::result const& res) { - return res.shape_offset_idx_; + [](shape_results::trip const& trip) { + return trip.shape_offset_idx_; }); bounding_box.extend(result->trip_bbox_); for (auto i = 0U; i < result->segment_bboxes_.size(); @@ -580,10 +601,10 @@ void calculate_shape_offsets_and_bboxes( offset_progress_tracker->increment(); task.results_ = task.task_(); }); - assign_shape_offsets(shapes_data, trips, offset_tasks, - shape_states.index_offset_); + auto results = assign_shape_offsets( + shapes_data, trips, std::move(offset_tasks), shape_states.index_offset_); - auto bbox_tasks = create_bbox_tasks(tt, shapes_data, offset_tasks, + auto bbox_tasks = create_bbox_tasks(tt, shapes_data, results, shapes_data.route_bboxes_.size(), shape_states.index_offset_); auto const bbox_progress_tracker = utl::get_active_progress_tracker(); From ee37475365b89a4b500cdc38ee354bbd76bc69c8 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:51:37 +0100 Subject: [PATCH 43/60] Delete old code --- include/nigiri/loader/gtfs/shape_prepare.h | 32 ---- src/loader/gtfs/load_timetable.cc | 14 -- src/loader/gtfs/shape_prepare.cc | 195 +-------------------- 3 files changed, 1 insertion(+), 240 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index aa8035e0..c0bd02c9 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -1,7 +1,5 @@ #pragma once -#include "geo/box.h" - #include "nigiri/loader/gtfs/trip.h" #include "nigiri/types.h" @@ -13,36 +11,6 @@ struct timetable; namespace nigiri::loader::gtfs { -struct shape_prepare { - struct shape_results { - struct result { - explicit result(stop_seq_t const* stop_seq, - std::vector const* distances); - stop_seq_t const* stop_seq_; - std::vector const* distances_; - shape_offset_idx_t offset_idx_; - geo::box trip_bbox_; - std::vector segment_bboxes_; - }; - - shape_idx_t shape_idx_; - std::vector results_; - }; - - shape_prepare(shape_loader_state const&, - vector_map const&, - shapes_storage&); - void calculate_results(timetable const&, shape_loader_state const&); - void create_trip_shape_offsets( - vector_map const&) const; - void create_route_bounding_boxes(timetable const&) const; - shape_idx_t index_offset_; - - std::vector shape_results_; - shapes_storage& shapes_; - bool results_ready_; -}; - void calculate_shape_offsets_and_bboxes( timetable const&, shapes_storage&, diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index ed3d7787..c05dddc3 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -263,16 +263,6 @@ void load_timetable(loader_config const& config, train_nr, stop_seq_numbers); } - auto shape_results = - (shapes_data == nullptr) - ? std::nullopt - : std::make_optional( - shape_prepare{shape_states, trip_data.data_, *shapes_data}); - if (shape_results.has_value()) { - shape_results->calculate_results(tt, std::move(shape_states)); - shape_results->create_trip_shape_offsets(trip_data.data_); - } - auto const timer = scoped_timer{"loader.gtfs.routes.build"}; auto const attributes = std::basic_string{}; auto lines = hash_map{}; @@ -378,10 +368,6 @@ void load_timetable(loader_config const& config, if (shapes_data != nullptr) { calculate_shape_offsets_and_bboxes(tt, *shapes_data, trip_data.data_, shape_states); } - // Build bounding boxes - if (shape_results.has_value()) { - shape_results->create_route_bounding_boxes(tt); - } // Build location_routes map for (auto l = tt.location_routes_.size(); l != tt.n_locations(); ++l) { diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 838a82f6..097839d7 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -9,6 +9,7 @@ #include "cista/strong.h" +#include "geo/box.h" #include "geo/latlng.h" #include "geo/polyline.h" @@ -92,200 +93,6 @@ std::vector get_offsets_by_dist_traveled( return offsets; } -shape_prepare::shape_results::result::result( - stop_seq_t const* stop_seq, std::vector const* distances) - : stop_seq_{stop_seq}, - distances_{distances}, - offset_idx_{shape_offset_idx_t::invalid()}, - trip_bbox_{}, - segment_bboxes_{} {} - -shape_prepare::shape_prepare(shape_loader_state const& states, - vector_map const& trips, - shapes_storage& shapes_data) - : index_offset_{states.index_offset_}, - shape_results_(states.id_map_.size()), - shapes_{shapes_data}, - results_ready_{false} { - for (auto i = 0U; i < shape_results_.size(); ++i) { - shape_results_[i].shape_idx_ = static_cast(index_offset_ + i); - } - for (auto const& trip : trips) { - if (trip.shape_idx_ == shape_idx_t::invalid()) { - continue; - } - auto const idx = cista::to_idx(trip.shape_idx_ - index_offset_); - auto& results = shape_results_[idx].results_; - if (utl::all_of(results, [&](shape_results::result const& it) { - return *it.stop_seq_ != trip.stop_seq_; - })) { - auto const distances = - trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; - results.emplace_back(&trip.stop_seq_, std::move(distances)); - } - } -} - -void shape_prepare::calculate_results(timetable const& tt, - shape_loader_state const& states) { - auto m = std::mutex{}; - auto const progress_tracker = utl::get_active_progress_tracker(); - progress_tracker->status("Calculating shape offsets") - .out_bounds(98.F, 99.F) - .in_high(shape_results_.size()); - utl::parallel_for(shape_results_, [&](shape_results& results) { - progress_tracker->increment(); - if (results.results_.empty()) { - return; - } - auto const shape = shapes_.get_shape(results.shape_idx_); - auto const& shape_distances = - states.distances_[cista::to_idx(results.shape_idx_ - index_offset_)]; - for (auto& result : results.results_) { - - auto const offsets = [&]() { - if (!shape_distances.empty() && result.distances_ != nullptr) { - return get_offsets_by_dist_traveled(*result.distances_, - shape_distances); - } - if (shape.size() < result.stop_seq_->size()) { - return std::vector{}; - } - - return get_offsets_by_stops(tt, shape, *result.stop_seq_); - }(); - - if (!offsets.empty()) { - auto const guard = std::lock_guard{m}; - result.offset_idx_ = shapes_.add_offsets(offsets); - } else { - result.offset_idx_ = shape_offset_idx_t::invalid(); - } - - if (!offsets.empty()) { - result.segment_bboxes_ = std::vector(offsets.size() - 1); - auto bbox_count = 0UL; - for (auto const [i, pair] : utl::enumerate(utl::pairwise(offsets))) { - auto& segment_bbox = result.segment_bboxes_[i]; - auto const& [from, to] = pair; - for (auto const point : - shape.subspan(cista::to_idx(from), - cista::to_idx(to) - cista::to_idx(from) + 1)) { - result.trip_bbox_.extend(point); - segment_bbox.extend(point); - } - auto const from_l = - tt.locations_ - .coordinates_[stop{(*result.stop_seq_)[i]}.location_idx()]; - auto const to_l = - tt.locations_.coordinates_[stop{(*result.stop_seq_)[i + 1]} - .location_idx()]; - auto const stop_bbox = geo::make_box({from_l, to_l}); - if (!stop_bbox.contains(segment_bbox)) { - bbox_count = i + 1U; - } - } - result.segment_bboxes_.resize(bbox_count); - } else { - for (auto const s : *result.stop_seq_) { - result.trip_bbox_.extend( - tt.locations_.coordinates_[stop{s}.location_idx()]); - } - } - } - }); - results_ready_ = true; -} - -void shape_prepare::create_trip_shape_offsets( - vector_map const& trips) const { - utl::verify(results_ready_, "Operation requires calculated results"); - auto const progress_tracker = utl::get_active_progress_tracker(); - progress_tracker->status("Storing trip offsets") - .out_bounds(99.F, 100.F) - .in_high(trips.size()); - for (auto const& trip : trips) { - progress_tracker->increment(); - auto const trip_idx = trip.trip_idx_; - auto const shape_idx = trip.shape_idx_; - auto const offset_idx = [&]() { - if (shape_idx == shape_idx_t::invalid()) { - return shape_offset_idx_t::invalid(); - } - auto const& results = - shape_results_[cista::to_idx(shape_idx - index_offset_)].results_; - auto const result = - utl::find_if(results, [&](shape_results::result const& res) { - return *res.stop_seq_ == trip.stop_seq_; - }); - assert(result != end(results)); - return result->offset_idx_; - }(); - shapes_.add_trip_shape_offsets(trip_idx, - cista::pair{shape_idx, offset_idx}); - } -} - -void shape_prepare::create_route_bounding_boxes(timetable const& tt) const { - utl::verify(results_ready_, "Operation requires calculated results"); - auto const route_offset = shapes_.route_bboxes_.size(); - auto const routes_count = tt.route_transport_ranges_.size() - route_offset; - auto route_bboxes = std::vector(routes_count); - auto route_segment_bboxes = std::vector>(routes_count); - utl::parallel_for_run(route_bboxes.size(), [&](std::size_t const bbox_idx) { - auto const r = static_cast(bbox_idx + route_offset); - auto const seq = tt.route_location_seq_[r]; - assert(seq.size() > 0U); - auto bounding_box = geo::box{}; - auto segment_bboxes = std::vector(seq.size() - 1); - auto bbox_count = 0UL; - auto processed = - std::vector>{}; - auto const stop_indices = - interval{stop_idx_t{0U}, static_cast(seq.size())}; - for (auto const transport_idx : tt.route_transport_ranges_[r]) { - auto const frun = rt::frun{tt, nullptr, - rt::run{.t_ = transport{transport_idx}, - .stop_range_ = stop_indices, - .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip([&](trip_idx_t const trip_idx, - interval const absolute_range) { - auto const [shape_idx, offset_idx] = - shapes_.trip_offset_indices_[trip_idx]; - if (shape_idx == shape_idx_t::invalid() || - offset_idx == shape_offset_idx_t::invalid()) { - for (auto const idx : absolute_range) { - bounding_box.extend( - tt.locations_.coordinates_[stop{seq[idx]}.location_idx()]); - } - } else { - auto const& result = utl::find_if( - shape_results_[cista::to_idx(shape_idx - index_offset_)].results_, - [&](shape_results::result const& res) { - return res.offset_idx_ == offset_idx; - }); - bounding_box.extend(result->trip_bbox_); - for (auto i = 0U; i < result->segment_bboxes_.size(); ++i) { - segment_bboxes[i + cista::to_idx(absolute_range.from_)] = - result->segment_bboxes_[i]; - } - bbox_count = std::max(bbox_count, static_cast( - result->segment_bboxes_.size() + - absolute_range.from_)); - } - }); - } - segment_bboxes.resize(bbox_count); - route_bboxes[bbox_idx] = std::move(bounding_box); - route_segment_bboxes[bbox_idx] = std::move(segment_bboxes); - }); - for (auto const [route_bbox, segment_bboxes] : - utl::zip(route_bboxes, route_segment_bboxes)) { - shapes_.route_bboxes_.emplace_back(route_bbox); - shapes_.route_segment_bboxes_.emplace_back(segment_bboxes); - } -} - struct offset_task { struct results { struct result { From 05ce7c7960ea1e92d55feee799bbed4bf438ab7c Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:02:45 +0100 Subject: [PATCH 44/60] Rename variables --- src/loader/gtfs/shape_prepare.cc | 96 +++++++++++++++++--------------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 097839d7..beab09a2 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -233,7 +233,7 @@ std::vector create_offset_tasks( | utl::vec(); } -struct shape_results { +struct bbox_and_offset_data { struct trip { stop_seq_t const* stop_seq_; shape_offset_idx_t shape_offset_idx_; @@ -243,7 +243,7 @@ struct shape_results { std::vector trips_; }; -std::vector assign_shape_offsets( +std::vector assign_shape_offsets( shapes_storage& shapes_data, vector_map const& trips, std::vector&& tasks, @@ -255,12 +255,12 @@ std::vector assign_shape_offsets( utl::all(std::move(tasks)) // | utl::transform([&](offset_task const& task) { - return shape_results{ + return bbox_and_offset_data{ .trips_ = utl::all(task.results_->results_) // | utl::transform([&](offset_task::results::result const& result) { - return shape_results::trip{ + return bbox_and_offset_data::trip{ .stop_seq_ = result.stop_seq_, .shape_offset_idx_ = result.offsets_.empty() @@ -286,7 +286,9 @@ std::vector assign_shape_offsets( auto const it = std::ranges::lower_bound( results, trip.stop_seq_, [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, - [](shape_results::trip const& entry) { return *entry.stop_seq_; }); + [](bbox_and_offset_data::trip const& entry) { + return *entry.stop_seq_; + }); shapes_data.add_trip_shape_offsets( trip_idx, cista::pair{shape_idx, it->shape_offset_idx_}); } @@ -306,7 +308,7 @@ struct bbox_task { std::vector create_bbox_tasks( timetable const& tt, shapes_storage& shapes_data, - std::vector const& results, + std::vector const& bbox_data, std::size_t const route_offset, shape_idx_t const shape_offset) { assert(utl::all_of(tasks, [](offset_task const& task) { @@ -338,41 +340,45 @@ std::vector create_bbox_tasks( rt::run{.t_ = transport{transport_idx}, .stop_range_ = stop_indices, .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip([&](trip_idx_t const trip_idx, - interval const - absolute_range) { - auto const [shape_idx, offset_idx] = - shapes_data.trip_offset_indices_[trip_idx]; - if (shape_idx == shape_idx_t::invalid() || - offset_idx == shape_offset_idx_t::invalid()) { - for (auto const idx : absolute_range) { - bounding_box.extend( - tt.locations_.coordinates_ - [stop{seq[idx]}.location_idx()]); - } - } else { - auto const result = std::ranges::lower_bound( - results[cista::to_idx(shape_idx - shape_offset)] - .trips_, - offset_idx, - [](shape_offset_idx_t const a, - shape_offset_idx_t const b) { return a < b; }, - [](shape_results::trip const& trip) { - return trip.shape_offset_idx_; - }); - bounding_box.extend(result->trip_bbox_); - for (auto i = 0U; i < result->segment_bboxes_.size(); - ++i) { - segment_bboxes[i + cista::to_idx( - absolute_range.from_)] = - result->segment_bboxes_[i]; - } - bbox_count = std::max( - bbox_count, static_cast( - result->segment_bboxes_.size() + - absolute_range.from_)); - } - }); + frun.for_each_trip( + [&](trip_idx_t const trip_idx, + interval const absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_data.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + for (auto const idx : absolute_range) { + bounding_box.extend( + tt.locations_.coordinates_ + [stop{seq[idx]}.location_idx()]); + } + } else { + auto const result = std::ranges::lower_bound( + bbox_data[cista::to_idx(shape_idx - + shape_offset)] + .trips_, + offset_idx, + [](shape_offset_idx_t const a, + shape_offset_idx_t const b) { + return a < b; + }, + [](bbox_and_offset_data::trip const& trip) { + return trip.shape_offset_idx_; + }); + bounding_box.extend(result->trip_bbox_); + for (auto i = 0U; + i < result->segment_bboxes_.size(); ++i) { + segment_bboxes[i + cista::to_idx( + absolute_range.from_)] = + result->segment_bboxes_[i]; + } + bbox_count = + std::max(bbox_count, + static_cast( + result->segment_bboxes_.size() + + absolute_range.from_)); + } + }); } return std::make_optional(bbox_task::result{ @@ -385,7 +391,7 @@ std::vector create_bbox_tasks( } void assign_bounding_boxes(shapes_storage& shapes_data, - std::vector const& tasks) { + std::vector const&& tasks) { for (auto const& task : tasks) { shapes_data.route_bboxes_.emplace_back(task.result_->route_bbox_); shapes_data.route_segment_bboxes_.emplace_back( @@ -408,10 +414,10 @@ void calculate_shape_offsets_and_bboxes( offset_progress_tracker->increment(); task.results_ = task.task_(); }); - auto results = assign_shape_offsets( + auto bbox_data = assign_shape_offsets( shapes_data, trips, std::move(offset_tasks), shape_states.index_offset_); - auto bbox_tasks = create_bbox_tasks(tt, shapes_data, results, + auto bbox_tasks = create_bbox_tasks(tt, shapes_data, bbox_data, shapes_data.route_bboxes_.size(), shape_states.index_offset_); auto const bbox_progress_tracker = utl::get_active_progress_tracker(); @@ -422,6 +428,6 @@ void calculate_shape_offsets_and_bboxes( bbox_progress_tracker->increment(); task.result_ = task.task_(); }); - assign_bounding_boxes(shapes_data, bbox_tasks); + assign_bounding_boxes(shapes_data, std::move(bbox_tasks)); } } // namespace nigiri::loader::gtfs \ No newline at end of file From d75fb44df4a956fe4ab62eba054f87779545f974 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:40:51 +0100 Subject: [PATCH 45/60] Remove intermediate struct --- src/loader/gtfs/shape_prepare.cc | 195 +++++++++++++++---------------- 1 file changed, 92 insertions(+), 103 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index beab09a2..3830db12 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -1,7 +1,6 @@ #include "nigiri/loader/gtfs/shape_prepare.h" #include -#include #include #include #include @@ -14,13 +13,11 @@ #include "geo/polyline.h" #include "utl/enumerate.h" -#include "utl/helpers/algorithm.h" #include "utl/parallel_for.h" #include "utl/pipes/all.h" #include "utl/pipes/transform.h" #include "utl/pipes/vec.h" #include "utl/progress_tracker.h" -#include "utl/verify.h" #include "nigiri/loader/gtfs/shape.h" #include "nigiri/loader/gtfs/trip.h" @@ -94,17 +91,14 @@ std::vector get_offsets_by_dist_traveled( } struct offset_task { - struct results { - struct result { - stop_seq_t const* stop_seq_; - std::vector offsets_; - geo::box trip_bbox_; - std::vector segment_bboxes_; - }; - std::vector results_; + struct result { + stop_seq_t const* stop_seq_; + std::vector offsets_; + geo::box trip_bbox_; + std::vector segment_bboxes_; }; - std::function()> task_; - std::optional results_; + std::function>()> task_; + std::optional> results_; }; std::vector create_offset_tasks( @@ -151,79 +145,75 @@ std::vector create_offset_tasks( auto const& shape_distances = states.distances_[cista::to_idx(input.shape_idx_ - index_offset)]; - return std::make_optional({ - .results_ = - utl::all(input.inputs_) // + return std::make_optional< + std::vector>({ + utl::all(input.inputs_) // | - utl::transform( - [&](input_pair const& pair) - -> offset_task::results::result { - auto const& [stop_seq, distances] = pair; - // Calculate offsets - auto const offsets = [&]() { - if (!shape_distances.empty() && - distances != nullptr) { - return get_offsets_by_dist_traveled( - *distances, shape_distances); - } - if (shape.size() < stop_seq->size()) { - return std::vector{}; - } + utl::transform([&](input_pair const& pair) + -> offset_task::result { + auto const& [stop_seq, distances] = pair; + // Calculate offsets + auto const offsets = [&]() { + if (!shape_distances.empty() && + distances != nullptr) { + return get_offsets_by_dist_traveled( + *distances, shape_distances); + } + if (shape.size() < stop_seq->size()) { + return std::vector{}; + } - return get_offsets_by_stops(tt, shape, - *stop_seq); - }(); - // Calculate bounding boxes - auto trip_bbox = geo::box{}; - auto segment_bboxes = - std::vector{}; - if (!offsets.empty()) { - segment_bboxes.resize( - (offsets.size() - 1)); - auto bbox_count = 0UL; - for (auto const [i, segment] : - utl::enumerate( - utl::pairwise(offsets))) { - auto& segment_bbox = segment_bboxes[i]; - auto const& [from, to] = segment; - for (auto const point : shape.subspan( - cista::to_idx(from), - cista::to_idx(to) - - cista::to_idx(from) + 1)) { - trip_bbox.extend(point); - segment_bbox.extend(point); - } - auto const from_l = - tt.locations_.coordinates_ - [stop{(*stop_seq)[i]} - .location_idx()]; - auto const to_l = - tt.locations_.coordinates_ - [stop{(*stop_seq)[i + 1]} - .location_idx()]; - auto const stop_bbox = - geo::make_box({from_l, to_l}); - if (!stop_bbox.contains(segment_bbox)) { - bbox_count = i + 1U; - } - } - segment_bboxes.resize(bbox_count); - } else { - for (auto const s : *stop_seq) { - trip_bbox.extend( - tt.locations_.coordinates_ - [stop{s}.location_idx()]); - } + return get_offsets_by_stops(tt, shape, + *stop_seq); + }(); + // Calculate bounding boxes + auto trip_bbox = geo::box{}; + auto segment_bboxes = std::vector{}; + if (!offsets.empty()) { + segment_bboxes.resize((offsets.size() - 1)); + auto bbox_count = 0UL; + for (auto const [i, segment] : + utl::enumerate(utl::pairwise(offsets))) { + auto& segment_bbox = segment_bboxes[i]; + auto const& [from, to] = segment; + for (auto const point : shape.subspan( + cista::to_idx(from), + cista::to_idx(to) - + cista::to_idx(from) + 1)) { + trip_bbox.extend(point); + segment_bbox.extend(point); } - // - return offset_task::results::result{ - .stop_seq_ = pair.first, - .offsets_ = std::move(offsets), - .trip_bbox_ = std::move(trip_bbox), - .segment_bboxes_ = - std::move(segment_bboxes), - }; - }) // + auto const from_l = + tt.locations_ + .coordinates_[stop{(*stop_seq)[i]} + .location_idx()]; + auto const to_l = + tt.locations_ + .coordinates_[stop{ + (*stop_seq)[i + 1]} + .location_idx()]; + auto const stop_bbox = + geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(segment_bbox)) { + bbox_count = i + 1U; + } + } + segment_bboxes.resize(bbox_count); + } else { + for (auto const s : *stop_seq) { + trip_bbox.extend( + tt.locations_.coordinates_ + [stop{s}.location_idx()]); + } + } + // Create result + return offset_task::result{ + .stop_seq_ = pair.first, + .offsets_ = std::move(offsets), + .trip_bbox_ = std::move(trip_bbox), + .segment_bboxes_ = std::move(segment_bboxes), + }; + }) // | utl::vec(), }); }, @@ -253,26 +243,24 @@ std::vector assign_shape_offsets( })); auto const entries = utl::all(std::move(tasks)) // - | - utl::transform([&](offset_task const& task) { - return bbox_and_offset_data{ - .trips_ = - utl::all(task.results_->results_) // - | - utl::transform([&](offset_task::results::result const& result) { - return bbox_and_offset_data::trip{ - .stop_seq_ = result.stop_seq_, - .shape_offset_idx_ = - result.offsets_.empty() - ? shape_offset_idx_t::invalid() - : shapes_data.add_offsets(result.offsets_), - .trip_bbox_ = result.trip_bbox_, - .segment_bboxes_ = result.segment_bboxes_, - }; - }) // - | utl::vec(), - }; - }) | + | utl::transform([&](offset_task const& task) { + return bbox_and_offset_data{ + .trips_ = + utl::all(*task.results_) // + | utl::transform([&](offset_task::result const& result) { + return bbox_and_offset_data::trip{ + .stop_seq_ = result.stop_seq_, + .shape_offset_idx_ = + result.offsets_.empty() + ? shape_offset_idx_t::invalid() + : shapes_data.add_offsets(result.offsets_), + .trip_bbox_ = result.trip_bbox_, + .segment_bboxes_ = result.segment_bboxes_, + }; + }) // + | utl::vec(), + }; + }) | utl::vec(); for (auto const& trip : trips) { auto const trip_idx = trip.trip_idx_; @@ -430,4 +418,5 @@ void calculate_shape_offsets_and_bboxes( }); assign_bounding_boxes(shapes_data, std::move(bbox_tasks)); } + } // namespace nigiri::loader::gtfs \ No newline at end of file From ced2e8dba4f7a29c38b421550911311f530b11bd Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:55:50 +0100 Subject: [PATCH 46/60] Fix compile issues --- src/loader/gtfs/shape_prepare.cc | 138 +++++++++++++++---------------- 1 file changed, 65 insertions(+), 73 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 3830db12..9568f3e5 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -103,7 +103,7 @@ struct offset_task { std::vector create_offset_tasks( timetable const& tt, - shapes_storage& shapes_data, + shapes_storage const& shapes_data, vector_map const& trips, shape_loader_state const& states) { using input_pair = cista::pair const*>; @@ -145,77 +145,72 @@ std::vector create_offset_tasks( auto const& shape_distances = states.distances_[cista::to_idx(input.shape_idx_ - index_offset)]; - return std::make_optional< - std::vector>({ + return std::make_optional( utl::all(input.inputs_) // - | - utl::transform([&](input_pair const& pair) - -> offset_task::result { - auto const& [stop_seq, distances] = pair; - // Calculate offsets - auto const offsets = [&]() { - if (!shape_distances.empty() && - distances != nullptr) { - return get_offsets_by_dist_traveled( - *distances, shape_distances); - } - if (shape.size() < stop_seq->size()) { - return std::vector{}; - } + | + utl::transform([&](input_pair const& pair) + -> offset_task::result { + auto const& [stop_seq, distances] = pair; + // Calculate offsets + auto const offsets = [&]() { + if (!shape_distances.empty() && + distances != nullptr) { + return get_offsets_by_dist_traveled( + *distances, shape_distances); + } + if (shape.size() < stop_seq->size()) { + return std::vector{}; + } - return get_offsets_by_stops(tt, shape, - *stop_seq); - }(); - // Calculate bounding boxes - auto trip_bbox = geo::box{}; - auto segment_bboxes = std::vector{}; - if (!offsets.empty()) { - segment_bboxes.resize((offsets.size() - 1)); - auto bbox_count = 0UL; - for (auto const [i, segment] : - utl::enumerate(utl::pairwise(offsets))) { - auto& segment_bbox = segment_bboxes[i]; - auto const& [from, to] = segment; - for (auto const point : shape.subspan( - cista::to_idx(from), - cista::to_idx(to) - - cista::to_idx(from) + 1)) { - trip_bbox.extend(point); - segment_bbox.extend(point); - } - auto const from_l = - tt.locations_ - .coordinates_[stop{(*stop_seq)[i]} - .location_idx()]; - auto const to_l = - tt.locations_ - .coordinates_[stop{ - (*stop_seq)[i + 1]} - .location_idx()]; - auto const stop_bbox = - geo::make_box({from_l, to_l}); - if (!stop_bbox.contains(segment_bbox)) { - bbox_count = i + 1U; - } - } - segment_bboxes.resize(bbox_count); - } else { - for (auto const s : *stop_seq) { - trip_bbox.extend( - tt.locations_.coordinates_ - [stop{s}.location_idx()]); - } + return get_offsets_by_stops(tt, shape, *stop_seq); + }(); + // Calculate bounding boxes + auto trip_bbox = geo::box{}; + auto segment_bboxes = std::vector{}; + if (!offsets.empty()) { + segment_bboxes.resize((offsets.size() - 1)); + auto bbox_count = 0UL; + for (auto const [i, segment] : + utl::enumerate(utl::pairwise(offsets))) { + auto& segment_bbox = segment_bboxes[i]; + auto const& [from, to] = segment; + for (auto const point : shape.subspan( + cista::to_idx(from), + cista::to_idx(to) - + cista::to_idx(from) + 1)) { + trip_bbox.extend(point); + segment_bbox.extend(point); } - // Create result - return offset_task::result{ - .stop_seq_ = pair.first, - .offsets_ = std::move(offsets), - .trip_bbox_ = std::move(trip_bbox), - .segment_bboxes_ = std::move(segment_bboxes), - }; - }) // - | utl::vec(), - }); + auto const from_l = + tt.locations_.coordinates_ + [stop{(*stop_seq)[i]}.location_idx()]; + auto const to_l = + tt.locations_ + .coordinates_[stop{(*stop_seq)[i + 1]} + .location_idx()]; + auto const stop_bbox = + geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(segment_bbox)) { + bbox_count = i + 1U; + } + } + segment_bboxes.resize(bbox_count); + } else { + for (auto const s : *stop_seq) { + trip_bbox.extend( + tt.locations_ + .coordinates_[stop{s}.location_idx()]); + } + } + // Create result + return offset_task::result{ + .stop_seq_ = pair.first, + .offsets_ = std::move(offsets), + .trip_bbox_ = std::move(trip_bbox), + .segment_bboxes_ = std::move(segment_bboxes), + }; + }) // + | utl::vec()); }, .results_ = std::nullopt, }; @@ -295,13 +290,10 @@ struct bbox_task { std::vector create_bbox_tasks( timetable const& tt, - shapes_storage& shapes_data, + shapes_storage const& shapes_data, std::vector const& bbox_data, std::size_t const route_offset, shape_idx_t const shape_offset) { - assert(utl::all_of(tasks, [](offset_task const& task) { - return task.results_ != std::nullopt; - })); auto const new_routes = interval{static_cast(route_offset), static_cast(tt.route_transport_ranges_.size() - From 4f09b87f053885bb285f79dad7cf25f4698bc5b3 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:56:13 +0100 Subject: [PATCH 47/60] Fix invalid range --- src/loader/gtfs/shape_prepare.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 9568f3e5..4e55e7ee 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -296,8 +296,7 @@ std::vector create_bbox_tasks( shape_idx_t const shape_offset) { auto const new_routes = interval{static_cast(route_offset), - static_cast(tt.route_transport_ranges_.size() - - route_offset)}; + static_cast(tt.route_transport_ranges_.size())}; return utl::all(new_routes) // | utl::transform([&](route_idx_t const& r) { From edbfedaa8ec897b7c7fac553c945d5c160f148dc Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 07:38:45 +0100 Subject: [PATCH 48/60] Fix empty bound boxes --- src/loader/gtfs/shape_prepare.cc | 1 + test/shape_test.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 4e55e7ee..fc76cd31 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -359,6 +359,7 @@ std::vector create_bbox_tasks( } }); } + segment_bboxes.resize(bbox_count); return std::make_optional(bbox_task::result{ std::move(bounding_box), std::move(segment_bboxes)}); diff --git a/test/shape_test.cc b/test/shape_test.cc index f4ff0a1a..24076f72 100644 --- a/test/shape_test.cc +++ b/test/shape_test.cc @@ -181,7 +181,7 @@ TEST(shape, single_trip_with_shape) { } // Shape contained in bounding box { - EXPECT_FALSE(shapes_data.get_bounding_box(route_idx_t{2}, 4).has_value()); + EXPECT_FALSE(shapes_data.get_bounding_box(route_idx_t{4}, 0).has_value()); } } } From 16ed9084a6630f64213e8531decd41c1c652550b Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:23:18 +0100 Subject: [PATCH 49/60] Fix empty bound boxes for sequential trips --- src/loader/gtfs/shape_prepare.cc | 32 ++++++++---- test/rt/frun_shape_test.cc | 83 +++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index fc76cd31..3a4a9dff 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -13,6 +13,7 @@ #include "geo/polyline.h" #include "utl/enumerate.h" +#include "utl/pairwise.h" #include "utl/parallel_for.h" #include "utl/pipes/all.h" #include "utl/pipes/transform.h" @@ -324,14 +325,8 @@ std::vector create_bbox_tasks( interval const absolute_range) { auto const [shape_idx, offset_idx] = shapes_data.trip_offset_indices_[trip_idx]; - if (shape_idx == shape_idx_t::invalid() || - offset_idx == shape_offset_idx_t::invalid()) { - for (auto const idx : absolute_range) { - bounding_box.extend( - tt.locations_.coordinates_ - [stop{seq[idx]}.location_idx()]); - } - } else { + if (shape_idx != shape_idx_t::invalid() && + offset_idx != shape_offset_idx_t::invalid()) { auto const result = std::ranges::lower_bound( bbox_data[cista::to_idx(shape_idx - shape_offset)] @@ -359,7 +354,26 @@ std::vector create_bbox_tasks( } }); } - segment_bboxes.resize(bbox_count); + segment_bboxes.resize(bbox_count); + // Extend bounding boxes to contain all stops + for (auto const [i, segment] : + utl::enumerate(utl::pairwise(stop_indices))) { + auto const [from, to] = segment; + auto const from_l = + tt.locations_ + .coordinates_[stop{seq[from]}.location_idx()]; + auto const to_l = + tt.locations_ + .coordinates_[stop{seq[to]}.location_idx()]; + if (i == 0U) { + bounding_box.extend(from_l); + } + bounding_box.extend(to_l); + if (i < bbox_count) { + segment_bboxes[i].extend(from_l); + segment_bboxes[i].extend(to_l); + } + } return std::make_optional(bbox_task::result{ std::move(bounding_box), std::move(segment_bboxes)}); diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc index e06b9aeb..ee918c7e 100644 --- a/test/rt/frun_shape_test.cc +++ b/test/rt/frun_shape_test.cc @@ -64,23 +64,31 @@ SERVICE_1,20240101,1 # routes.txt route_id,agency_id,route_short_name,route_long_name,route_type ROUTE_1,AGENCY_1,Route 1,,3 +ROUTE_2,AGENCY_1,Route 2,,3 +ROUTE_3,AGENCY_1,Route 3,,3 +ROUTE_4,AGENCY_1,Route 4,,3 +ROUTE_5,AGENCY_1,Route 5,,3 +ROUTE_6,AGENCY_1,Route 6,,3 +ROUTE_7,AGENCY_1,Route 7,,3 +ROUTE_8,AGENCY_1,Route 8,,3 +ROUTE_9,AGENCY_1,Route 9,,3 # trips.txt route_id,service_id,trip_id,trip_headsign,block_id,shape_id, ROUTE_1,SERVICE_1,TRIP_1,E,BLOCK_1,SHAPE_1, -ROUTE_1,SERVICE_1,TRIP_2,E,BLOCK_2,SHAPE_2, -ROUTE_1,SERVICE_1,TRIP_3,E,BLOCK_2,SHAPE_3, -ROUTE_1,SERVICE_1,TRIP_4,E,BLOCK_2,SHAPE_4, -ROUTE_1,SERVICE_1,TRIP_5,E,BLOCK_3,SHAPE_5, -ROUTE_1,SERVICE_1,TRIP_5+,E,BLOCK_5+,SHAPE_5, -ROUTE_1,SERVICE_1,TRIP_6,E,BLOCK_4,, -ROUTE_1,SERVICE_1,TRIP_7,E,BLOCK_5,SHAPE_2, -ROUTE_1,SERVICE_1,TRIP_8,E,BLOCK_5,, -ROUTE_1,SERVICE_1,TRIP_9,E,BLOCK_5,, -ROUTE_1,SERVICE_1,TRIP_10,E,BLOCK_5,SHAPE_6, -ROUTE_1,SERVICE_1,TRIP_11,E,BLOCK_6,SHAPE_7, -ROUTE_1,SERVICE_1,TRIP_12,E,BLOCK_7,SHAPE_8, -ROUTE_1,SERVICE_1,TRIP_13,E,BLOCK_8,SHAPE_9, +ROUTE_2,SERVICE_1,TRIP_2,E,BLOCK_2,SHAPE_2, +ROUTE_2,SERVICE_1,TRIP_3,E,BLOCK_2,SHAPE_3, +ROUTE_2,SERVICE_1,TRIP_4,E,BLOCK_2,SHAPE_4, +ROUTE_3,SERVICE_1,TRIP_5,E,BLOCK_3,SHAPE_5, +ROUTE_4,SERVICE_1,TRIP_5+,E,BLOCK_5+,SHAPE_5, +ROUTE_5,SERVICE_1,TRIP_6,E,BLOCK_4,, +ROUTE_6,SERVICE_1,TRIP_7,E,BLOCK_5,SHAPE_2, +ROUTE_6,SERVICE_1,TRIP_8,E,BLOCK_5,, +ROUTE_6,SERVICE_1,TRIP_9,E,BLOCK_5,, +ROUTE_6,SERVICE_1,TRIP_10,E,BLOCK_5,SHAPE_6, +ROUTE_7,SERVICE_1,TRIP_11,E,BLOCK_11,SHAPE_11, +ROUTE_8,SERVICE_1,TRIP_12,E,BLOCK_12,SHAPE_12, +ROUTE_9,SERVICE_1,TRIP_13,E,BLOCK_13,SHAPE_13, # shapes.txt "shape_id","shape_pt_lat","shape_pt_lon","shape_pt_sequence","shape_dist_traveled" @@ -124,22 +132,22 @@ SHAPE_6,6.5,2.5,2, SHAPE_6,7.0,2.0,3, SHAPE_6,6.5,1.5,4, SHAPE_6,7.0,1.0,5, -SHAPE_7,1.0,1.0,0,0.0 -SHAPE_7,1.5,1.5,1,0.7 -SHAPE_7,2.0,2.0,2,1.4 -SHAPE_7,2.5,2.5,3,2.1 -SHAPE_7,3.0,3.0,4,2.9 -SHAPE_7,3.5,3.5,5,3.5 -SHAPE_7,4.0,4.0,6,4.2 -SHAPE_8,1.0,1.0,0,0.0 -SHAPE_8,1.5,1.5,1,0.0 -SHAPE_8,2.0,2.0,2,2.0 -SHAPE_8,3.0,3.0,3,4.0 -SHAPE_9,1.0,1.0,0, -SHAPE_9,2.5,2.5,1, -SHAPE_9,3.625,3.625,2, -SHAPE_9,4.0,4.0,3, -SHAPE_9,4.0,4.0,4, +SHAPE_11,1.0,1.0,0,0.0 +SHAPE_11,1.5,1.5,1,0.7 +SHAPE_11,2.0,2.0,2,1.4 +SHAPE_11,2.5,2.5,3,2.1 +SHAPE_11,3.0,3.0,4,2.9 +SHAPE_11,3.5,3.5,5,3.5 +SHAPE_11,4.0,4.0,6,4.2 +SHAPE_12,1.0,1.0,0,0.0 +SHAPE_12,1.5,1.5,1,0.0 +SHAPE_12,2.0,2.0,2,2.0 +SHAPE_12,3.0,3.0,3,4.0 +SHAPE_13,1.0,1.0,0, +SHAPE_13,2.5,2.5,1, +SHAPE_13,3.625,3.625,2, +SHAPE_13,4.0,4.0,3, +SHAPE_13,4.0,4.0,4, # stop_times.txt trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type,shape_dist_traveled @@ -763,6 +771,23 @@ TEST( leg_shape); } } + // Bounding boxes for route containing sequential trips + { + // Trip with bounding boxes defined by stops if followed by trip where + // bounding boxes are extended by shape + // Route 6 + { + auto const r = route_idx_t{5U}; + EXPECT_EQ((geo::make_box({{1.0, 0.5}, {7.0, 3.0}})), + shapes_data.get_bounding_box(r)); + auto const extended_bbox = shapes_data.get_bounding_box(r, 6); + ASSERT_TRUE(extended_bbox.has_value()); + EXPECT_EQ((geo::make_box({{6.5, 2.0}, {7.0, 3.0}})), *extended_bbox); + auto const test_bbox = shapes_data.get_bounding_box(r, 5); + ASSERT_TRUE(extended_bbox.has_value()); + EXPECT_EQ((geo::make_box({{6.0, 2.0}, {7.0, 3.0}})), *test_bbox); + } + } } } // namespace \ No newline at end of file From dd18872f6716ed8954d402becad72de5b37164ae Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:53:12 +0100 Subject: [PATCH 50/60] Reduce stored data for simple sequential trips --- src/loader/gtfs/shape_prepare.cc | 65 ++++++++++++++++---------------- test/rt/frun_shape_test.cc | 20 ++++++++++ 2 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 3a4a9dff..5bcad039 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -320,39 +320,38 @@ std::vector create_bbox_tasks( rt::run{.t_ = transport{transport_idx}, .stop_range_ = stop_indices, .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip( - [&](trip_idx_t const trip_idx, - interval const absolute_range) { - auto const [shape_idx, offset_idx] = - shapes_data.trip_offset_indices_[trip_idx]; - if (shape_idx != shape_idx_t::invalid() && - offset_idx != shape_offset_idx_t::invalid()) { - auto const result = std::ranges::lower_bound( - bbox_data[cista::to_idx(shape_idx - - shape_offset)] - .trips_, - offset_idx, - [](shape_offset_idx_t const a, - shape_offset_idx_t const b) { - return a < b; - }, - [](bbox_and_offset_data::trip const& trip) { - return trip.shape_offset_idx_; - }); - bounding_box.extend(result->trip_bbox_); - for (auto i = 0U; - i < result->segment_bboxes_.size(); ++i) { - segment_bboxes[i + cista::to_idx( - absolute_range.from_)] = - result->segment_bboxes_[i]; - } - bbox_count = - std::max(bbox_count, - static_cast( - result->segment_bboxes_.size() + - absolute_range.from_)); - } - }); + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const + absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_data.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + return; + } + auto const result = std::ranges::lower_bound( + bbox_data[cista::to_idx(shape_idx - shape_offset)] + .trips_, + offset_idx, + [](shape_offset_idx_t const a, + shape_offset_idx_t const b) { return a < b; }, + [](bbox_and_offset_data::trip const& trip) { + return trip.shape_offset_idx_; + }); + bounding_box.extend(result->trip_bbox_); + auto const& bboxes = result->segment_bboxes_; + if (bboxes.empty()) { + return; + } + for (auto const [i, bbox] : utl::enumerate(bboxes)) { + segment_bboxes[i + cista::to_idx( + absolute_range.from_)] = bbox; + } + bbox_count = + std::max(bbox_count, + bboxes.size() + + cista::to_idx(absolute_range.from_)); + }); } segment_bboxes.resize(bbox_count); // Extend bounding boxes to contain all stops diff --git a/test/rt/frun_shape_test.cc b/test/rt/frun_shape_test.cc index ee918c7e..fca96b92 100644 --- a/test/rt/frun_shape_test.cc +++ b/test/rt/frun_shape_test.cc @@ -50,6 +50,7 @@ N,N,3.0,3.0 N1,N1,3.5,3.0 O,O,4.0,4.0 Q,Q,0.0,0.0 +Q+,Q+,-1.0,-1.0 S,S,4.0,1.0 T,T,5.0,1.0 U,U,6.0,2.0 @@ -82,6 +83,7 @@ ROUTE_2,SERVICE_1,TRIP_4,E,BLOCK_2,SHAPE_4, ROUTE_3,SERVICE_1,TRIP_5,E,BLOCK_3,SHAPE_5, ROUTE_4,SERVICE_1,TRIP_5+,E,BLOCK_5+,SHAPE_5, ROUTE_5,SERVICE_1,TRIP_6,E,BLOCK_4,, +ROUTE_5,SERVICE_1,TRIP_6+,E,BLOCK_4,SHAPE_QQ+, ROUTE_6,SERVICE_1,TRIP_7,E,BLOCK_5,SHAPE_2, ROUTE_6,SERVICE_1,TRIP_8,E,BLOCK_5,, ROUTE_6,SERVICE_1,TRIP_9,E,BLOCK_5,, @@ -127,6 +129,9 @@ SHAPE_5,2.5,3.0,9,6.53 SHAPE_5,3.0,3.0,10,7.03 SHAPE_5,3.5,3.0,11,7.53 SHAPE_5,4.0,4.0,11,8.71 +SHAPE_QQ+,0.0,0.0,1, +SHAPE_QQ+,-0.4,-0.6,2, +SHAPE_QQ+,-1.0,-1.0,2, SHAPE_6,7.0,3.0,1, SHAPE_6,6.5,2.5,2, SHAPE_6,7.0,2.0,3, @@ -175,6 +180,8 @@ TRIP_5+,12:30:00,12:30:00,N1,4,0,0, TRIP_5+,13:00:00,13:00:00,O,5,0,0, TRIP_6,10:00:00,10:00:00,A,1,0,0, TRIP_6,11:00:00,11:00:00,Q,2,0,0, +TRIP_6+,11:00:00,11:00:00,Q,1,0,0, +TRIP_6+,12:00:00,12:00:00,Q+,2,0,0, TRIP_7,10:00:00,10:00:00,A,1,0,0, TRIP_7,11:00:00,11:00:00,F,2,0,0, TRIP_7,12:00:00,12:00:00,G,3,0,0, @@ -778,15 +785,28 @@ TEST( // Route 6 { auto const r = route_idx_t{5U}; + // Ensure the correct route is used EXPECT_EQ((geo::make_box({{1.0, 0.5}, {7.0, 3.0}})), shapes_data.get_bounding_box(r)); + // On 4th trip auto const extended_bbox = shapes_data.get_bounding_box(r, 6); ASSERT_TRUE(extended_bbox.has_value()); EXPECT_EQ((geo::make_box({{6.5, 2.0}, {7.0, 3.0}})), *extended_bbox); + // On 3rd trip auto const test_bbox = shapes_data.get_bounding_box(r, 5); ASSERT_TRUE(extended_bbox.has_value()); EXPECT_EQ((geo::make_box({{6.0, 2.0}, {7.0, 3.0}})), *test_bbox); } + // Do not insert bounding boxes if sequential trip has no segment bounding + // boxes + { + auto const r = route_idx_t{4U}; + // Ensure the correct route is used + EXPECT_EQ((geo::make_box({{-1.0, -1.0}, {1.0, 1.0}})), + shapes_data.get_bounding_box(r)); + // On 1st trip + ASSERT_FALSE(shapes_data.get_bounding_box(r, 0).has_value()); + } } } From 94b2af420911a4610cd3669c7c9bb49a4caf89da Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:01:46 +0100 Subject: [PATCH 51/60] Fix formatting --- src/loader/gtfs/load_timetable.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index c05dddc3..74afd205 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -366,7 +366,8 @@ void load_timetable(loader_config const& config, } if (shapes_data != nullptr) { - calculate_shape_offsets_and_bboxes(tt, *shapes_data, trip_data.data_, shape_states); + calculate_shape_offsets_and_bboxes(tt, *shapes_data, trip_data.data_, + shape_states); } // Build location_routes map From 1fac9fb21f4069dbe3e37764cf120463b4d2a309 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:59:03 +0100 Subject: [PATCH 52/60] Fix size type --- src/loader/gtfs/shape_prepare.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 5bcad039..5c7f9734 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -309,7 +309,7 @@ std::vector create_bbox_tasks( auto bounding_box = geo::box{}; auto segment_bboxes = std::vector{}; segment_bboxes.resize(seq.size() - 1); - auto bbox_count = 0UL; + auto bbox_count = static_cast(0U); auto const stop_indices = interval{ stop_idx_t{0U}, static_cast(seq.size())}; @@ -349,8 +349,9 @@ std::vector create_bbox_tasks( } bbox_count = std::max(bbox_count, - bboxes.size() + - cista::to_idx(absolute_range.from_)); + static_cast( + bboxes.size() + + cista::to_idx(absolute_range.from_))); }); } segment_bboxes.resize(bbox_count); From 1437f02493247f5de0292b6474296dfc6984854b Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:07:45 +0100 Subject: [PATCH 53/60] Attempt to fix macos build --- src/loader/gtfs/shape_prepare.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 5c7f9734..2d5a9438 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -133,7 +133,7 @@ std::vector create_offset_tasks( } auto const distances = trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; - shape_inputs.emplace(it, &trip.stop_seq_, std::move(distances)); + shape_inputs.insert(it, input_pair{&trip.stop_seq_, std::move(distances)}); } return utl::all(std::move(inputs)) // From 244a1c069df5543141b86e12629975bdaf693d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20G=C3=BCndling?= Date: Mon, 11 Nov 2024 11:59:32 +0100 Subject: [PATCH 54/60] wip --- .pkg.lock | 18 +-- include/nigiri/loader/gtfs/shape.h | 14 +- src/loader/gtfs/shape_prepare.cc | 217 ++++++++++++----------------- 3 files changed, 111 insertions(+), 138 deletions(-) diff --git a/.pkg.lock b/.pkg.lock index 16c65745..f7c27c29 100644 --- a/.pkg.lock +++ b/.pkg.lock @@ -1,24 +1,24 @@ -139770128757021168 -cista 48210e6925658163952e458a4d13629ffdb0cea3 +5786421993668347507 +cista fabfc0cbcf07343a2e0c2def3009555b22f0ae46 PEGTL 1c1aa6e650e4d26f10fa398f148ec0cdc5f0808d res b759b93316afeb529b6cb5b2548b24c41e382fb0 date ce88cc33b5551f66655614eeebb7c5b7189025fb googletest 7b64fca6ea0833628d6f86255a81424365f7cc0c fmt dc10f83be70ac2873d5f8d1ce317596f1fd318a2 -utl 77aac494c45d2b070e65fe712abc34ac74a91d0f +utl 8bfa7fc4edc23f716173bdfbcab8294fcc31e457 oh d21c30f40e52a83d6dc09bcffd0067598b5ec069 -zlib ee0742244d93c4237154ae16c3db42b5f284b442 -boost 60cae66449fa3c9599b2b7d3d5d44c65301ed3a3 +zlib-ng 68ab3e2d80253ec5dc3c83691d9ff70477b32cd3 +boost 73549ebca677fe6214202a1ab580362b4f80e653 doctest 70e8f76437b76dd5e9c0a2eb9b907df190ab71a0 -geo 0a14addf42e91b267906a156c9c2564935c03eaf +geo cee17208dba98f5f3ddaf78d0bfff9c083557ba3 miniz 1edbdece9d71dc65c6ff405572ee37cbdcef7af4 libressl 24acd9e710fbe842e863572da9d738715fbc74b8 -curl 3358dac21192864ef2ba47c88704d3b8d8203804 +curl 39c8a51e8ee0ab7ea712886df79c068405a2e008 json 410c74782230daaa15054d6ee0975c0607091cb3 opentelemetry-proto 1624689398a3226c45994d70cb544a1e781dc032 abseil-cpp ba5240842d352b4b67a32092453a2fe5fe53a62e -protobuf d8136b9c6a62db6ce09900ecdeb82bb793096cbd -opentelemetry-cpp ec4aef6b17b697052edef5417825ad71947b2ed1 +protobuf df2dd518c68b882c9dce5346393f8c388108e733 +opentelemetry-cpp 60770dc9dc63e3543fc87d605b2e88fd53d7a414 pugixml 60175e80e2f5e97e027ac78f7e14c5acc009ce50 unordered_dense 77e91016354e6d8cba24a86c5abb807de2534c02 wyhash 1e012b57fc2227a9e583a57e2eacb3da99816d99 diff --git a/include/nigiri/loader/gtfs/shape.h b/include/nigiri/loader/gtfs/shape.h index 6b91d6e5..0befbcae 100644 --- a/include/nigiri/loader/gtfs/shape.h +++ b/include/nigiri/loader/gtfs/shape.h @@ -10,10 +10,20 @@ struct shapes_storage; namespace nigiri::loader::gtfs { +using relative_shape_idx_t = + cista::strong; + struct shape_loader_state { + shape_idx_t get_shape_idx(relative_shape_idx_t const i) const { + return shape_idx_t{to_idx(i) + index_offset_}; + } + relative_shape_idx_t get_relative_idx(shape_idx_t const i) const { + return relative_shape_idx_t{to_idx(i) - index_offset_}; + } + hash_map id_map_{}; - std::vector> distances_{}; - shape_idx_t index_offset_; + vector_map> distances_{}; + shape_idx_t::value_t index_offset_; }; shape_loader_state parse_shapes(std::string_view const, shapes_storage&); diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 2d5a9438..220b0d17 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -13,6 +13,7 @@ #include "geo/polyline.h" #include "utl/enumerate.h" +#include "utl/insert_sorted.h" #include "utl/pairwise.h" #include "utl/parallel_for.h" #include "utl/pipes/all.h" @@ -91,132 +92,44 @@ std::vector get_offsets_by_dist_traveled( return offsets; } -struct offset_task { +struct stop_seq_dist { + bool operator<(stop_seq_dist const& o) const { + return *stop_seq_ < *o.stop_seq_; + } + bool operator==(stop_seq_dist const& o) const { + return *stop_seq_ == *o.stop_seq_; + } + bool operator!=(stop_seq_dist const& o) const { + return *stop_seq_ != *o.stop_seq_; + } + + stop_seq_t const* stop_seq_; + std::vector const* dist_traveled_; struct result { - stop_seq_t const* stop_seq_; + shape_offset_idx_t shape_offset_idx_{shape_offset_idx_t::invalid()}; std::vector offsets_; geo::box trip_bbox_; std::vector segment_bboxes_; - }; - std::function>()> task_; - std::optional> results_; + } result_{}; }; -std::vector create_offset_tasks( - timetable const& tt, - shapes_storage const& shapes_data, +using task = std::vector; + +vector_map create_offset_tasks( vector_map const& trips, shape_loader_state const& states) { - using input_pair = cista::pair const*>; - struct input_data { - shape_idx_t shape_idx_; - std::vector inputs_; - }; - - auto const index_offset = cista::to_idx(states.index_offset_); - auto inputs = std::vector(states.id_map_.size()); - for (auto [idx, input] : utl::enumerate(inputs)) { - input.shape_idx_ = static_cast(idx + index_offset); - } + auto tasks = vector_map{}; + tasks.resize(static_cast(states.id_map_.size())); for (auto const& trip : trips) { if (trip.shape_idx_ == shape_idx_t::invalid()) { continue; } - auto const idx = cista::to_idx(trip.shape_idx_ - index_offset); - auto& shape_inputs = inputs[idx].inputs_; - auto const it = std::ranges::lower_bound( - shape_inputs, trip.stop_seq_, - [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, - [](input_pair const& p) { return *p.first; }); - if (it != end(shape_inputs) && *it->first == trip.stop_seq_) { - continue; - } - auto const distances = - trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_; - shape_inputs.insert(it, input_pair{&trip.stop_seq_, std::move(distances)}); + utl::insert_sorted( + tasks[states.get_relative_idx(trip.shape_idx_)], + {&trip.stop_seq_, + trip.distance_traveled_.empty() ? nullptr : &trip.distance_traveled_}); } - - return utl::all(std::move(inputs)) // - | - utl::transform([&](input_data const& in) { - return offset_task{ - .task_ = - [&, input = in, index_offset]() { - auto const shape = shapes_data.get_shape(input.shape_idx_); - auto const& shape_distances = - states.distances_[cista::to_idx(input.shape_idx_ - - index_offset)]; - return std::make_optional( - utl::all(input.inputs_) // - | - utl::transform([&](input_pair const& pair) - -> offset_task::result { - auto const& [stop_seq, distances] = pair; - // Calculate offsets - auto const offsets = [&]() { - if (!shape_distances.empty() && - distances != nullptr) { - return get_offsets_by_dist_traveled( - *distances, shape_distances); - } - if (shape.size() < stop_seq->size()) { - return std::vector{}; - } - - return get_offsets_by_stops(tt, shape, *stop_seq); - }(); - // Calculate bounding boxes - auto trip_bbox = geo::box{}; - auto segment_bboxes = std::vector{}; - if (!offsets.empty()) { - segment_bboxes.resize((offsets.size() - 1)); - auto bbox_count = 0UL; - for (auto const [i, segment] : - utl::enumerate(utl::pairwise(offsets))) { - auto& segment_bbox = segment_bboxes[i]; - auto const& [from, to] = segment; - for (auto const point : shape.subspan( - cista::to_idx(from), - cista::to_idx(to) - - cista::to_idx(from) + 1)) { - trip_bbox.extend(point); - segment_bbox.extend(point); - } - auto const from_l = - tt.locations_.coordinates_ - [stop{(*stop_seq)[i]}.location_idx()]; - auto const to_l = - tt.locations_ - .coordinates_[stop{(*stop_seq)[i + 1]} - .location_idx()]; - auto const stop_bbox = - geo::make_box({from_l, to_l}); - if (!stop_bbox.contains(segment_bbox)) { - bbox_count = i + 1U; - } - } - segment_bboxes.resize(bbox_count); - } else { - for (auto const s : *stop_seq) { - trip_bbox.extend( - tt.locations_ - .coordinates_[stop{s}.location_idx()]); - } - } - // Create result - return offset_task::result{ - .stop_seq_ = pair.first, - .offsets_ = std::move(offsets), - .trip_bbox_ = std::move(trip_bbox), - .segment_bboxes_ = std::move(segment_bboxes), - }; - }) // - | utl::vec()); - }, - .results_ = std::nullopt, - }; - }) // - | utl::vec(); + return tasks; } struct bbox_and_offset_data { @@ -393,21 +306,72 @@ void assign_bounding_boxes(shapes_storage& shapes_data, } } +void process_task(timetable const& tt, + shapes_storage const& shapes_data, + shape_loader_state const& shape_states, + relative_shape_idx_t const i, + task& t) { + auto const shape = shapes_data.get_shape(shape_states.get_shape_idx(i)); + auto const& shape_distances = shape_states.distances_[i]; + for (auto& x : t) { + auto& r = x.result_; + auto const& [stop_seq, distances] = std::tie(x.stop_seq_, x.dist_traveled_); + + // Calculate offsets + r.offsets_ = (shape.size() < stop_seq->size()) + ? std::vector{} + : (!shape_distances.empty() && distances != nullptr) + ? get_offsets_by_dist_traveled(*distances, shape_distances) + : get_offsets_by_stops(tt, shape, *stop_seq); + + // Calculate bounding boxes + if (r.offsets_.empty()) { + for (auto const s : *stop_seq) { + r.trip_bbox_.extend(tt.locations_.coordinates_[stop{s}.location_idx()]); + } + } else { + r.segment_bboxes_.resize(r.offsets_.size() - 1); + auto is_trivial = true; + for (auto const [segment_idx, segment] : + utl::enumerate(utl::pairwise(r.offsets_))) { + auto const& [from, to] = segment; + for (auto const point : + shape.subspan(cista::to_idx(from), cista::to_idx(to - from + 1))) { + r.trip_bbox_.extend(point); + r.segment_bboxes_[segment_idx].extend(point); + } + auto const from_l = + tt.locations_ + .coordinates_[stop{(*stop_seq)[segment_idx]}.location_idx()]; + auto const to_l = + tt.locations_.coordinates_[stop{(*stop_seq)[segment_idx + 1]} + .location_idx()]; + auto const stop_bbox = geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(r.segment_bboxes_[segment_idx])) { + is_trivial = false; + } + } + if (is_trivial) { + r.segment_bboxes_.clear(); + } + } + } +} + void calculate_shape_offsets_and_bboxes( timetable const& tt, shapes_storage& shapes_data, - vector_map const& trips, - shape_loader_state const& shape_states) { - auto offset_tasks = create_offset_tasks(tt, shapes_data, trips, shape_states); - auto const offset_progress_tracker = utl::get_active_progress_tracker(); - offset_progress_tracker->status("Creating trip offsets") - .out_bounds(98.F, 99.F) - .in_high(offset_tasks.size()); - utl::parallel_for(offset_tasks, - [&offset_progress_tracker](offset_task& task) { - offset_progress_tracker->increment(); - task.results_ = task.task_(); - }); + shape_loader_state const& shape_states, + vector_map const& trips) { + utl::get_active_progress_tracker() + ->status("Creating trip offsets") + .out_bounds(98.F, 99.F); + auto tasks = create_offset_tasks(trips, shape_states); + utl::parallel_for_run(tasks.size(), [&](std::size_t const i) { + auto const s = relative_shape_idx_t{i}; + process_task(tt, shapes_data, shape_states, s, tasks[s]); + }); + auto bbox_data = assign_shape_offsets( shapes_data, trips, std::move(offset_tasks), shape_states.index_offset_); @@ -416,8 +380,7 @@ void calculate_shape_offsets_and_bboxes( shape_states.index_offset_); auto const bbox_progress_tracker = utl::get_active_progress_tracker(); bbox_progress_tracker->status("Creating bounding boxes") - .out_bounds(99.F, 100.F) - .in_high(bbox_tasks.size()); + .out_bounds(99.F, 100.F); utl::parallel_for(bbox_tasks, [&bbox_progress_tracker](bbox_task& task) { bbox_progress_tracker->increment(); task.result_ = task.task_(); From ce3dcc8377a1b025751467f9a7771e3572b01217 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:45:59 +0100 Subject: [PATCH 55/60] Apply pending changes --- include/nigiri/loader/gtfs/shape_prepare.h | 4 +- src/loader/gtfs/load_timetable.cc | 4 +- src/loader/gtfs/shape.cc | 13 +- src/loader/gtfs/shape_prepare.cc | 146 +++++++++++++-------- 4 files changed, 104 insertions(+), 63 deletions(-) diff --git a/include/nigiri/loader/gtfs/shape_prepare.h b/include/nigiri/loader/gtfs/shape_prepare.h index c0bd02c9..8f363828 100644 --- a/include/nigiri/loader/gtfs/shape_prepare.h +++ b/include/nigiri/loader/gtfs/shape_prepare.h @@ -14,7 +14,7 @@ namespace nigiri::loader::gtfs { void calculate_shape_offsets_and_bboxes( timetable const&, shapes_storage&, - vector_map const&, - shape_loader_state const&); + shape_loader_state const&, + vector_map const&); } // namespace nigiri::loader::gtfs diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index 74afd205..d988cb45 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -366,8 +366,8 @@ void load_timetable(loader_config const& config, } if (shapes_data != nullptr) { - calculate_shape_offsets_and_bboxes(tt, *shapes_data, trip_data.data_, - shape_states); + calculate_shape_offsets_and_bboxes(tt, *shapes_data, shape_states, + trip_data.data_); } // Build location_routes map diff --git a/src/loader/gtfs/shape.cc b/src/loader/gtfs/shape.cc index a2791e53..eb4b1286 100644 --- a/src/loader/gtfs/shape.cc +++ b/src/loader/gtfs/shape.cc @@ -26,12 +26,12 @@ shape_loader_state parse_shapes(std::string_view const data, utl::csv_col distance_; }; - auto const index_offset = static_cast(shapes.size()); + auto const index_offset = static_cast(shapes.size()); auto states = shape_loader_state{ .index_offset_ = index_offset, }; auto lookup = cached_lookup(states.id_map_); - auto seq = std::vector>{}; + auto seq = vector_map>{}; auto const progress_tracker = utl::get_active_progress_tracker(); progress_tracker->status("Parse Shapes") @@ -49,7 +49,7 @@ shape_loader_state parse_shapes(std::string_view const data, }); auto polyline = shapes[shape_idx]; polyline.push_back(geo::latlng{*entry.lat_, *entry.lon_}); - auto const state_idx = to_idx(shape_idx - index_offset); + auto const state_idx = states.get_relative_idx(shape_idx); seq[state_idx].push_back(*entry.seq_); auto& distances = states.distances_[state_idx]; if (distances.empty()) { @@ -65,11 +65,13 @@ shape_loader_state parse_shapes(std::string_view const data, }); auto polyline = std::vector(); - auto shape_idx = states.index_offset_; - for (auto i = 0U; i != states.distances_.size(); ++i) { + for (auto const i : + interval{relative_shape_idx_t{0U}, + relative_shape_idx_t{states.distances_.size()}}) { if (utl::is_sorted(seq[i], std::less<>{})) { continue; } + auto const shape_idx = states.get_shape_idx(i); polyline.resize(shapes[shape_idx].size()); for (auto j = 0U; j != shapes[shape_idx].size(); ++j) { @@ -79,7 +81,6 @@ shape_loader_state parse_shapes(std::string_view const data, std::tie(seq[i], states.distances_[i], polyline) = utl::sort_by(seq[i], states.distances_[i], polyline); std::copy(begin(polyline), end(polyline), begin(shapes[shape_idx])); - ++shape_idx; } return states; diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 220b0d17..919bb5b8 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -13,6 +13,7 @@ #include "geo/polyline.h" #include "utl/enumerate.h" +#include "utl/helpers/algorithm.h" #include "utl/insert_sorted.h" #include "utl/pairwise.h" #include "utl/parallel_for.h" @@ -142,35 +143,18 @@ struct bbox_and_offset_data { std::vector trips_; }; -std::vector assign_shape_offsets( - shapes_storage& shapes_data, - vector_map const& trips, - std::vector&& tasks, - shape_idx_t const index_offset) { - assert(utl::all_of(tasks, [](offset_task const& task) { - return task.results_ != std::nullopt; - })); - auto const entries = - utl::all(std::move(tasks)) // - | utl::transform([&](offset_task const& task) { - return bbox_and_offset_data{ - .trips_ = - utl::all(*task.results_) // - | utl::transform([&](offset_task::result const& result) { - return bbox_and_offset_data::trip{ - .stop_seq_ = result.stop_seq_, - .shape_offset_idx_ = - result.offsets_.empty() - ? shape_offset_idx_t::invalid() - : shapes_data.add_offsets(result.offsets_), - .trip_bbox_ = result.trip_bbox_, - .segment_bboxes_ = result.segment_bboxes_, - }; - }) // - | utl::vec(), - }; - }) | - utl::vec(); +void assign_shape_offsets(shapes_storage& shapes_data, + vector_map const& trips, + vector_map& tasks, + shape_loader_state const& states) { + for (auto& shape_tasks : tasks) { + for (auto& task : shape_tasks) { + auto& r = task.result_; + if (!r.offsets_.empty()) { + r.shape_offset_idx_ = shapes_data.add_offsets(std::move(r.offsets_)); + } + } + } for (auto const& trip : trips) { auto const trip_idx = trip.trip_idx_; auto const shape_idx = trip.shape_idx_; @@ -178,19 +162,15 @@ std::vector assign_shape_offsets( shapes_data.add_trip_shape_offsets( trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); } else { - auto const& results = - entries[cista::to_idx(shape_idx - index_offset)].trips_; - auto const it = std::ranges::lower_bound( - results, trip.stop_seq_, + auto const shape_tasks = tasks[states.get_relative_idx(shape_idx)]; + auto const task = std::ranges::lower_bound( + shape_tasks, trip.stop_seq_, [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, - [](bbox_and_offset_data::trip const& entry) { - return *entry.stop_seq_; - }); + [](stop_seq_dist const& s) { return *s.stop_seq_; }); shapes_data.add_trip_shape_offsets( - trip_idx, cista::pair{shape_idx, it->shape_offset_idx_}); + trip_idx, cista::pair{shape_idx, task->result_.shape_offset_idx_}); } } - return entries; } struct bbox_task { @@ -297,6 +277,78 @@ std::vector create_bbox_tasks( utl::vec(); } +void assign_bounding_boxes(timetable const& tt, + shapes_storage& shapes_data, + vector_map& tasks, + shape_loader_state const& shape_states) { + auto const new_routes = + interval{static_cast(shapes_data.route_bboxes_.size()), + static_cast(tt.route_transport_ranges_.size())}; + for (auto const r : new_routes) { + auto const seq = tt.route_location_seq_[r]; + assert(seq.size() > 0U); + auto bounding_box = geo::box{}; + auto segment_bboxes = std::vector{}; + segment_bboxes.resize(seq.size() - 1); + auto const stop_indices = + interval{stop_idx_t{0U}, static_cast(seq.size())}; + + // Create basic bounding boxes, span by stops + for (auto const [i, s] : utl::enumerate(stop_indices)) { + auto const pos = tt.locations_.coordinates_[stop{seq[s]}.location_idx()]; + bounding_box.extend(pos); + if (i > 0U) { + segment_bboxes[i - 1U].extend(pos); + } + if (i < segment_bboxes.size()) { + segment_bboxes[i].extend(pos); + } + } + + // TODO Sorted vector + auto is_trivial = true; + + for (auto const transport_idx : tt.route_transport_ranges_[r]) { + auto const frun = rt::frun{tt, nullptr, + rt::run{.t_ = transport{transport_idx}, + .stop_range_ = stop_indices, + .rt_ = rt_transport_idx_t::invalid()}}; + frun.for_each_trip([&](trip_idx_t const trip_idx, + interval const absolute_range) { + auto const [shape_idx, offset_idx] = + shapes_data.trip_offset_indices_[trip_idx]; + if (shape_idx == shape_idx_t::invalid() || + offset_idx == shape_offset_idx_t::invalid()) { + return; + } + auto const& shape_tasks = + tasks[shape_states.get_relative_idx(shape_idx)]; + auto const it = utl::find_if(shape_tasks, [&](stop_seq_dist const& s) { + return s.result_.shape_offset_idx_ == offset_idx; + }); + + auto const& res = it->result_; + bounding_box.extend(res.trip_bbox_); + auto const& bboxes = res.segment_bboxes_; + if (!bboxes.empty()) { + for (auto const [i, bbox] : utl::enumerate(bboxes)) { + segment_bboxes[i + cista::to_idx(absolute_range.from_)].extend( + bbox); + } + is_trivial = false; + } + }); + } + + if (is_trivial) { + segment_bboxes.clear(); + } + + shapes_data.route_bboxes_.emplace_back(std::move(bounding_box)); + shapes_data.route_segment_bboxes_.emplace_back(std::move(segment_bboxes)); + } +} + void assign_bounding_boxes(shapes_storage& shapes_data, std::vector const&& tasks) { for (auto const& task : tasks) { @@ -365,27 +417,15 @@ void calculate_shape_offsets_and_bboxes( vector_map const& trips) { utl::get_active_progress_tracker() ->status("Creating trip offsets") - .out_bounds(98.F, 99.F); + .out_bounds(98.F, 100.F); auto tasks = create_offset_tasks(trips, shape_states); utl::parallel_for_run(tasks.size(), [&](std::size_t const i) { auto const s = relative_shape_idx_t{i}; process_task(tt, shapes_data, shape_states, s, tasks[s]); }); - auto bbox_data = assign_shape_offsets( - shapes_data, trips, std::move(offset_tasks), shape_states.index_offset_); - - auto bbox_tasks = create_bbox_tasks(tt, shapes_data, bbox_data, - shapes_data.route_bboxes_.size(), - shape_states.index_offset_); - auto const bbox_progress_tracker = utl::get_active_progress_tracker(); - bbox_progress_tracker->status("Creating bounding boxes") - .out_bounds(99.F, 100.F); - utl::parallel_for(bbox_tasks, [&bbox_progress_tracker](bbox_task& task) { - bbox_progress_tracker->increment(); - task.result_ = task.task_(); - }); - assign_bounding_boxes(shapes_data, std::move(bbox_tasks)); + assign_shape_offsets(shapes_data, trips, tasks, shape_states); + assign_bounding_boxes(tt, shapes_data, tasks, shape_states); } } // namespace nigiri::loader::gtfs \ No newline at end of file From 584318d5c1501ce42bd9a76a0ec365020b0a861e Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:49:14 +0100 Subject: [PATCH 56/60] Delete old code --- src/loader/gtfs/shape_prepare.cc | 226 +++++++------------------------ 1 file changed, 51 insertions(+), 175 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 919bb5b8..94f5101b 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -133,15 +133,57 @@ vector_map create_offset_tasks( return tasks; } -struct bbox_and_offset_data { - struct trip { - stop_seq_t const* stop_seq_; - shape_offset_idx_t shape_offset_idx_; - geo::box trip_bbox_; - std::vector segment_bboxes_; - }; - std::vector trips_; -}; +void process_task(timetable const& tt, + shapes_storage const& shapes_data, + shape_loader_state const& shape_states, + relative_shape_idx_t const i, + task& t) { + auto const shape = shapes_data.get_shape(shape_states.get_shape_idx(i)); + auto const& shape_distances = shape_states.distances_[i]; + for (auto& x : t) { + auto& r = x.result_; + auto const& [stop_seq, distances] = std::tie(x.stop_seq_, x.dist_traveled_); + + // Calculate offsets + r.offsets_ = (shape.size() < stop_seq->size()) + ? std::vector{} + : (!shape_distances.empty() && distances != nullptr) + ? get_offsets_by_dist_traveled(*distances, shape_distances) + : get_offsets_by_stops(tt, shape, *stop_seq); + + // Calculate bounding boxes + if (r.offsets_.empty()) { + for (auto const s : *stop_seq) { + r.trip_bbox_.extend(tt.locations_.coordinates_[stop{s}.location_idx()]); + } + } else { + r.segment_bboxes_.resize(r.offsets_.size() - 1); + auto is_trivial = true; + for (auto const [segment_idx, segment] : + utl::enumerate(utl::pairwise(r.offsets_))) { + auto const& [from, to] = segment; + for (auto const point : + shape.subspan(cista::to_idx(from), cista::to_idx(to - from + 1))) { + r.trip_bbox_.extend(point); + r.segment_bboxes_[segment_idx].extend(point); + } + auto const from_l = + tt.locations_ + .coordinates_[stop{(*stop_seq)[segment_idx]}.location_idx()]; + auto const to_l = + tt.locations_.coordinates_[stop{(*stop_seq)[segment_idx + 1]} + .location_idx()]; + auto const stop_bbox = geo::make_box({from_l, to_l}); + if (!stop_bbox.contains(r.segment_bboxes_[segment_idx])) { + is_trivial = false; + } + } + if (is_trivial) { + r.segment_bboxes_.clear(); + } + } + } +} void assign_shape_offsets(shapes_storage& shapes_data, vector_map const& trips, @@ -173,110 +215,6 @@ void assign_shape_offsets(shapes_storage& shapes_data, } } -struct bbox_task { - struct result { - geo::box route_bbox_; - std::vector route_segment_bboxes_; - }; - std::function()> task_; - std::optional result_; -}; - -std::vector create_bbox_tasks( - timetable const& tt, - shapes_storage const& shapes_data, - std::vector const& bbox_data, - std::size_t const route_offset, - shape_idx_t const shape_offset) { - auto const new_routes = - interval{static_cast(route_offset), - static_cast(tt.route_transport_ranges_.size())}; - return utl::all(new_routes) // - | - utl::transform([&](route_idx_t const& r) { - return bbox_task{ - .task_ = - [&, r, shape_offset]() { - auto const seq = tt.route_location_seq_[r]; - assert(seq.size() > 0U); - auto bounding_box = geo::box{}; - auto segment_bboxes = std::vector{}; - segment_bboxes.resize(seq.size() - 1); - auto bbox_count = static_cast(0U); - auto const stop_indices = interval{ - stop_idx_t{0U}, static_cast(seq.size())}; - - for (auto const transport_idx : - tt.route_transport_ranges_[r]) { - auto const frun = rt::frun{ - tt, nullptr, - rt::run{.t_ = transport{transport_idx}, - .stop_range_ = stop_indices, - .rt_ = rt_transport_idx_t::invalid()}}; - frun.for_each_trip([&](trip_idx_t const trip_idx, - interval const - absolute_range) { - auto const [shape_idx, offset_idx] = - shapes_data.trip_offset_indices_[trip_idx]; - if (shape_idx == shape_idx_t::invalid() || - offset_idx == shape_offset_idx_t::invalid()) { - return; - } - auto const result = std::ranges::lower_bound( - bbox_data[cista::to_idx(shape_idx - shape_offset)] - .trips_, - offset_idx, - [](shape_offset_idx_t const a, - shape_offset_idx_t const b) { return a < b; }, - [](bbox_and_offset_data::trip const& trip) { - return trip.shape_offset_idx_; - }); - bounding_box.extend(result->trip_bbox_); - auto const& bboxes = result->segment_bboxes_; - if (bboxes.empty()) { - return; - } - for (auto const [i, bbox] : utl::enumerate(bboxes)) { - segment_bboxes[i + cista::to_idx( - absolute_range.from_)] = bbox; - } - bbox_count = - std::max(bbox_count, - static_cast( - bboxes.size() + - cista::to_idx(absolute_range.from_))); - }); - } - segment_bboxes.resize(bbox_count); - // Extend bounding boxes to contain all stops - for (auto const [i, segment] : - utl::enumerate(utl::pairwise(stop_indices))) { - auto const [from, to] = segment; - auto const from_l = - tt.locations_ - .coordinates_[stop{seq[from]}.location_idx()]; - auto const to_l = - tt.locations_ - .coordinates_[stop{seq[to]}.location_idx()]; - if (i == 0U) { - bounding_box.extend(from_l); - } - bounding_box.extend(to_l); - if (i < bbox_count) { - segment_bboxes[i].extend(from_l); - segment_bboxes[i].extend(to_l); - } - } - - return std::make_optional(bbox_task::result{ - std::move(bounding_box), std::move(segment_bboxes)}); - }, - .result_ = std::nullopt, - }; - }) | - utl::vec(); -} - void assign_bounding_boxes(timetable const& tt, shapes_storage& shapes_data, vector_map& tasks, @@ -305,7 +243,6 @@ void assign_bounding_boxes(timetable const& tt, } } - // TODO Sorted vector auto is_trivial = true; for (auto const transport_idx : tt.route_transport_ranges_[r]) { @@ -349,67 +286,6 @@ void assign_bounding_boxes(timetable const& tt, } } -void assign_bounding_boxes(shapes_storage& shapes_data, - std::vector const&& tasks) { - for (auto const& task : tasks) { - shapes_data.route_bboxes_.emplace_back(task.result_->route_bbox_); - shapes_data.route_segment_bboxes_.emplace_back( - task.result_->route_segment_bboxes_); - } -} - -void process_task(timetable const& tt, - shapes_storage const& shapes_data, - shape_loader_state const& shape_states, - relative_shape_idx_t const i, - task& t) { - auto const shape = shapes_data.get_shape(shape_states.get_shape_idx(i)); - auto const& shape_distances = shape_states.distances_[i]; - for (auto& x : t) { - auto& r = x.result_; - auto const& [stop_seq, distances] = std::tie(x.stop_seq_, x.dist_traveled_); - - // Calculate offsets - r.offsets_ = (shape.size() < stop_seq->size()) - ? std::vector{} - : (!shape_distances.empty() && distances != nullptr) - ? get_offsets_by_dist_traveled(*distances, shape_distances) - : get_offsets_by_stops(tt, shape, *stop_seq); - - // Calculate bounding boxes - if (r.offsets_.empty()) { - for (auto const s : *stop_seq) { - r.trip_bbox_.extend(tt.locations_.coordinates_[stop{s}.location_idx()]); - } - } else { - r.segment_bboxes_.resize(r.offsets_.size() - 1); - auto is_trivial = true; - for (auto const [segment_idx, segment] : - utl::enumerate(utl::pairwise(r.offsets_))) { - auto const& [from, to] = segment; - for (auto const point : - shape.subspan(cista::to_idx(from), cista::to_idx(to - from + 1))) { - r.trip_bbox_.extend(point); - r.segment_bboxes_[segment_idx].extend(point); - } - auto const from_l = - tt.locations_ - .coordinates_[stop{(*stop_seq)[segment_idx]}.location_idx()]; - auto const to_l = - tt.locations_.coordinates_[stop{(*stop_seq)[segment_idx + 1]} - .location_idx()]; - auto const stop_bbox = geo::make_box({from_l, to_l}); - if (!stop_bbox.contains(r.segment_bboxes_[segment_idx])) { - is_trivial = false; - } - } - if (is_trivial) { - r.segment_bboxes_.clear(); - } - } - } -} - void calculate_shape_offsets_and_bboxes( timetable const& tt, shapes_storage& shapes_data, From 426dd8e05b65e9b392db4f1e7e1ca3cca85e17e4 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 12 Nov 2024 09:17:35 +0100 Subject: [PATCH 57/60] Remove unused includes --- src/loader/gtfs/shape_prepare.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 94f5101b..59cfe4c3 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -1,7 +1,6 @@ #include "nigiri/loader/gtfs/shape_prepare.h" #include -#include #include #include #include @@ -17,9 +16,6 @@ #include "utl/insert_sorted.h" #include "utl/pairwise.h" #include "utl/parallel_for.h" -#include "utl/pipes/all.h" -#include "utl/pipes/transform.h" -#include "utl/pipes/vec.h" #include "utl/progress_tracker.h" #include "nigiri/loader/gtfs/shape.h" From f39b5e48d852a6e1263dae220ab3155cdac5d315 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:03:43 +0100 Subject: [PATCH 58/60] Cleanup code --- src/loader/gtfs/load_timetable.cc | 1 - src/loader/gtfs/shape.cc | 3 +-- src/loader/gtfs/shape_prepare.cc | 21 ++++++++++----------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/loader/gtfs/load_timetable.cc b/src/loader/gtfs/load_timetable.cc index d988cb45..2bb0c81c 100644 --- a/src/loader/gtfs/load_timetable.cc +++ b/src/loader/gtfs/load_timetable.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include "utl/get_or_create.h" diff --git a/src/loader/gtfs/shape.cc b/src/loader/gtfs/shape.cc index eb4b1286..b1d4d016 100644 --- a/src/loader/gtfs/shape.cc +++ b/src/loader/gtfs/shape.cc @@ -26,9 +26,8 @@ shape_loader_state parse_shapes(std::string_view const data, utl::csv_col distance_; }; - auto const index_offset = static_cast(shapes.size()); auto states = shape_loader_state{ - .index_offset_ = index_offset, + .index_offset_ = static_cast(shapes.size()), }; auto lookup = cached_lookup(states.id_map_); auto seq = vector_map>{}; diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 59cfe4c3..0214d4bf 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -185,9 +185,9 @@ void assign_shape_offsets(shapes_storage& shapes_data, vector_map const& trips, vector_map& tasks, shape_loader_state const& states) { - for (auto& shape_tasks : tasks) { - for (auto& task : shape_tasks) { - auto& r = task.result_; + for (auto& task : tasks) { + for (auto& x : task) { + auto& r = x.result_; if (!r.offsets_.empty()) { r.shape_offset_idx_ = shapes_data.add_offsets(std::move(r.offsets_)); } @@ -200,20 +200,20 @@ void assign_shape_offsets(shapes_storage& shapes_data, shapes_data.add_trip_shape_offsets( trip_idx, cista::pair{shape_idx, shape_offset_idx_t::invalid()}); } else { - auto const shape_tasks = tasks[states.get_relative_idx(shape_idx)]; - auto const task = std::ranges::lower_bound( - shape_tasks, trip.stop_seq_, + auto const task = tasks[states.get_relative_idx(shape_idx)]; + auto const x = std::ranges::lower_bound( + task, trip.stop_seq_, [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, [](stop_seq_dist const& s) { return *s.stop_seq_; }); shapes_data.add_trip_shape_offsets( - trip_idx, cista::pair{shape_idx, task->result_.shape_offset_idx_}); + trip_idx, cista::pair{shape_idx, x->result_.shape_offset_idx_}); } } } void assign_bounding_boxes(timetable const& tt, shapes_storage& shapes_data, - vector_map& tasks, + vector_map const& tasks, shape_loader_state const& shape_states) { auto const new_routes = interval{static_cast(shapes_data.route_bboxes_.size()), @@ -254,9 +254,8 @@ void assign_bounding_boxes(timetable const& tt, offset_idx == shape_offset_idx_t::invalid()) { return; } - auto const& shape_tasks = - tasks[shape_states.get_relative_idx(shape_idx)]; - auto const it = utl::find_if(shape_tasks, [&](stop_seq_dist const& s) { + auto const& task = tasks[shape_states.get_relative_idx(shape_idx)]; + auto const it = utl::find_if(task, [&](stop_seq_dist const& s) { return s.result_.shape_offset_idx_ == offset_idx; }); From f5fdca7b8b8ea301306f106d4629d8b1464d79d9 Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:10:52 +0100 Subject: [PATCH 59/60] Add missing check for end element --- src/loader/gtfs/shape_prepare.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 0214d4bf..4a1a2967 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -205,8 +205,15 @@ void assign_shape_offsets(shapes_storage& shapes_data, task, trip.stop_seq_, [&](stop_seq_t const& a, stop_seq_t const& b) { return a < b; }, [](stop_seq_dist const& s) { return *s.stop_seq_; }); - shapes_data.add_trip_shape_offsets( - trip_idx, cista::pair{shape_idx, x->result_.shape_offset_idx_}); + if (x != end(task) && + x->result_.shape_offset_idx_ != shape_offset_idx_t::invalid()) { + shapes_data.add_trip_shape_offsets( + trip_idx, cista::pair{shape_idx, x->result_.shape_offset_idx_}); + } else { + shapes_data.add_trip_shape_offsets( + trip_idx, + cista::pair{shape_idx_t::invalid(), shape_offset_idx_t::invalid()}); + } } } } @@ -258,6 +265,9 @@ void assign_bounding_boxes(timetable const& tt, auto const it = utl::find_if(task, [&](stop_seq_dist const& s) { return s.result_.shape_offset_idx_ == offset_idx; }); + if (it == end(task)) { + return; + } auto const& res = it->result_; bounding_box.extend(res.trip_bbox_); From 3d83c8e6229db2b31095f3ff1c9d0d6fe13ab10b Mon Sep 17 00:00:00 2001 From: Michael Kutzner <174690291+MichaelKutzner@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:25:24 +0100 Subject: [PATCH 60/60] Update progress status text --- src/loader/gtfs/shape_prepare.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/loader/gtfs/shape_prepare.cc b/src/loader/gtfs/shape_prepare.cc index 4a1a2967..4b51a464 100644 --- a/src/loader/gtfs/shape_prepare.cc +++ b/src/loader/gtfs/shape_prepare.cc @@ -296,15 +296,18 @@ void calculate_shape_offsets_and_bboxes( shapes_storage& shapes_data, shape_loader_state const& shape_states, vector_map const& trips) { - utl::get_active_progress_tracker() - ->status("Creating trip offsets") - .out_bounds(98.F, 100.F); + auto const progress_tracker = utl::get_active_progress_tracker(); + progress_tracker->status("Creating trip offsets") + .out_bounds(98.F, 99.F) + .in_high(shape_states.id_map_.size()); auto tasks = create_offset_tasks(trips, shape_states); utl::parallel_for_run(tasks.size(), [&](std::size_t const i) { auto const s = relative_shape_idx_t{i}; process_task(tt, shapes_data, shape_states, s, tasks[s]); }); + progress_tracker->status("Writing offsets and bounding boxes") + .out_bounds(99.F, 100.F); assign_shape_offsets(shapes_data, trips, tasks, shape_states); assign_bounding_boxes(tt, shapes_data, tasks, shape_states); }