Remove duplicates

exe/import.cc

@@ -60,9 +60,14 @@ int main(int ac, char** av) {
       ("link_stop_distance",
        bpo::value(&c.link_stop_distance_)->default_value(c.link_stop_distance_),
        "the maximum distance at which stops in proximity will be linked")  //
-      ("merge_duplicates",
-       bpo::value(&c.merge_duplicates_)->default_value(c.merge_duplicates_),
-       "merge duplicates")  //
+      ("merge_dupes_intra_source",
+       bpo::value(&c.merge_dupes_intra_src_)
+           ->default_value(c.merge_dupes_intra_src_),
+       "merge duplicates within a source")  //
+      ("merge_dupes_inter_source",
+       bpo::value(&c.merge_dupes_inter_src_)
+           ->default_value(c.merge_dupes_inter_src_),
+       "merge duplicates between different sources")  //
       ("adjust_footpaths",
        bpo::value(&c.adjust_footpaths_)->default_value(c.adjust_footpaths_),
        "adjust footpath lengths")  //

include/nigiri/routing/for_each_meta.h → include/nigiri/for_each_meta.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "nigiri/routing/location_match_mode.h"
+#include "nigiri/location_match_mode.h"
 #include "nigiri/timetable.h"
 
 namespace nigiri::routing {

include/nigiri/loader/build_footpaths.h

@@ -6,7 +6,8 @@ namespace nigiri::loader {
 
 void build_footpaths(timetable& tt,
                      bool adjust_footpaths,
-                     bool merge_duplicates,
+                     bool merge_dupes_intra_src,
+                     bool merge_dupes_inter_src,
                      std::uint16_t max_footpath_length);
 
 }  // namespace nigiri::loader

include/nigiri/loader/init_finish.h

@@ -12,7 +12,8 @@ namespace nigiri::loader {
 void register_special_stations(timetable&);
 void finalize(timetable&,
               bool adjust_footpaths = false,
-              bool merge_duplicates = false,
+              bool merge_dupes_intra_src = false,
+              bool merge_dupes_inter_src = false,
               std::uint16_t max_footpath_length =
                   std::numeric_limits<std::uint16_t>::max());
 

include/nigiri/loader/link_nearby_stations.h

@@ -1,11 +1,9 @@
-#include "nigiri/loader/match_set.h"
-
 namespace nigiri {
 struct timetable;
 }
 
 namespace nigiri::loader {
 
-match_set_t link_nearby_stations(timetable&, bool const store_matches);
+void link_nearby_stations(timetable&);
 
 }  // namespace nigiri::loader

include/nigiri/loader/loader_interface.h

@@ -20,7 +20,8 @@ struct loader_config {
 
   // finalize options
   bool adjust_footpaths_{true};
-  bool merge_duplicates_{true};
+  bool merge_dupes_intra_src_{true};
+  bool merge_dupes_inter_src_{true};
   std::uint16_t max_footpath_length_{20};
 };
 

include/nigiri/loader/merge_duplicates.h

@@ -6,11 +6,6 @@ struct timetable;
 
 namespace nigiri::loader {
 
-using match_set_t = hash_set<pair<location_idx_t, location_idx_t>>;
-
-unsigned find_duplicates(timetable& tt,
-                         match_set_t const& matches,
-                         location_idx_t const a,
-                         location_idx_t const b);
+unsigned find_duplicates(timetable& tt, location_idx_t a, location_idx_t b);
 
 }  // namespace nigiri::loader

include/nigiri/query_generator/generator_settings.h

@@ -2,10 +2,10 @@
 
 #include "geo/box.h"
 
+#include "nigiri/location_match_mode.h"
 #include "nigiri/query_generator/transport_mode.h"
 #include "nigiri/routing/clasz_mask.h"
 #include "nigiri/routing/limits.h"
-#include "nigiri/routing/location_match_mode.h"
 #include "nigiri/routing/transfer_time_settings.h"
 #include "nigiri/timetable.h"
 

include/nigiri/routing/query.h

@@ -7,9 +7,9 @@
 
 #include "nigiri/common/interval.h"
 #include "nigiri/footpath.h"
+#include "nigiri/location_match_mode.h"
 #include "nigiri/routing/clasz_mask.h"
 #include "nigiri/routing/limits.h"
-#include "nigiri/routing/location_match_mode.h"
 #include "nigiri/routing/transfer_time_settings.h"
 #include "nigiri/td_footpath.h"
 #include "nigiri/types.h"

include/nigiri/routing/sanitize_via_stops.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "nigiri/routing/for_each_meta.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/routing/query.h"
 #include "nigiri/timetable.h"
 

include/nigiri/routing/search.h

@@ -6,9 +6,9 @@
 #include "utl/timing.h"
 #include "utl/to_vec.h"
 
+#include "nigiri/for_each_meta.h"
 #include "nigiri/logging.h"
 #include "nigiri/routing/dijkstra.h"
-#include "nigiri/routing/for_each_meta.h"
 #include "nigiri/routing/get_fastest_direct.h"
 #include "nigiri/routing/interval_estimate.h"
 #include "nigiri/routing/journey.h"

src/loader/build_footpaths.cc

@@ -407,13 +407,27 @@ void write_footpaths(timetable& tt) {
 
 void build_footpaths(timetable& tt,
                      bool const adjust_footpaths,
-                     bool const merge_duplicates,
+                     bool const merge_dupes_intra_src,
+                     bool const merge_dupes_inter_src,
                      std::uint16_t const max_footpath_length) {
   add_links_to_and_between_children(tt);
-  auto const matches = link_nearby_stations(tt, merge_duplicates);
-  if (merge_duplicates) {
-    for (auto const& [a, b] : matches) {
-      find_duplicates(tt, matches, a, b);
+  link_nearby_stations(tt);
+  if (merge_dupes_intra_src || merge_dupes_inter_src) {
+    for (auto l = location_idx_t{0U}; l != tt.n_locations(); ++l) {
+      if (tt.locations_.src_[l] == source_idx_t{source_idx_t::invalid()}) {
+        continue;
+      }
+      for (auto e : tt.locations_.equivalences_[l]) {
+        if (tt.locations_.src_[e] == source_idx_t{source_idx_t::invalid()} ||
+            (!merge_dupes_intra_src &&
+             tt.locations_.src_[l] == tt.locations_.src_[e]) ||
+            (!merge_dupes_inter_src &&
+             tt.locations_.src_[l] != tt.locations_.src_[e])) {
+          continue;
+        }
+
+        find_duplicates(tt, l, e);
+      }
     }
   }
   connect_components(tt, max_footpath_length, adjust_footpaths);

src/loader/init_finish.cc

@@ -28,7 +28,8 @@ void register_special_stations(timetable& tt) {
 
 void finalize(timetable& tt,
               bool const adjust_footpaths,
-              bool const merge_duplicates,
+              bool const merge_dupes_intra_src,
+              bool const merge_dupes_inter_src,
               std::uint16_t const max_footpath_length) {
   tt.location_routes_.resize(tt.n_locations());
 
@@ -47,7 +48,8 @@ void finalize(timetable& tt,
                             tt.trip_id_strings_[b.first].view());
         });
   }
-  build_footpaths(tt, adjust_footpaths, merge_duplicates, max_footpath_length);
+  build_footpaths(tt, adjust_footpaths, merge_dupes_intra_src,
+                  merge_dupes_inter_src, max_footpath_length);
   build_lb_graph<direction::kForward>(tt);
   build_lb_graph<direction::kBackward>(tt);
 }

src/loader/link_nearby_stations.cc

@@ -8,13 +8,12 @@
 
 namespace nigiri::loader {
 
-match_set_t link_nearby_stations(timetable& tt, bool const store_matches) {
+void link_nearby_stations(timetable& tt) {
   constexpr auto const kLinkNearbyMaxDistance = 300;  // [m];
 
   auto const locations_rtree =
       geo::make_point_rtree(tt.locations_.coordinates_);
 
-  auto matches = match_set_t{};
   for (auto l_from_idx = location_idx_t{0U};
        l_from_idx != tt.locations_.src_.size(); ++l_from_idx) {
     auto const from_pos = tt.locations_.coordinates_[l_from_idx];
@@ -55,14 +54,8 @@ match_set_t link_nearby_stations(timetable& tt, bool const store_matches) {
       tt.locations_.preprocessing_footpaths_in_[l_to_idx].emplace_back(
           l_from_idx, duration);
       tt.locations_.equivalences_[l_from_idx].emplace_back(l_to_idx);
-
-      if (store_matches) {
-        matches.emplace(make_match_pair(l_from_idx, l_to_idx));
-      }
     }
   }
-
-  return matches;
 }
 
 }  // namespace nigiri::loader

src/loader/load.cc

@@ -1,7 +1,5 @@
 #include "nigiri/loader/load.h"
 
-#include <tuple>
-
 #include "utl/enumerate.h"
 
 #include "nigiri/loader/dir.h"
@@ -50,8 +48,8 @@ timetable load(std::vector<std::filesystem::path> const& paths,
     }
   }
 
-  finalize(tt, c.adjust_footpaths_, c.merge_duplicates_,
-           c.max_footpath_length_);
+  finalize(tt, c.adjust_footpaths_, c.merge_dupes_intra_src_,
+           c.merge_dupes_inter_src_, c.max_footpath_length_);
 
   return tt;
 }

src/loader/merge_duplicates.cc

@@ -1,6 +1,6 @@
 #include "nigiri/loader/merge_duplicates.h"
 
-#include "nigiri/loader/match_set.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/timetable.h"
 
 namespace nigiri::loader {
@@ -33,27 +33,39 @@ bool merge(timetable& tt,
            stop_idx_t const size,
            transport_idx_t const a,
            transport_idx_t const b) {
+  assert(a != b);
+
   auto const bf_a = tt.bitfields_[tt.transport_traffic_days_[a]];
   auto const bf_b = tt.bitfields_[tt.transport_traffic_days_[b]];
   if ((bf_a & bf_b).none()) {
     return false;
   }
 
-  if ((bf_a & bf_b) == bf_a) {
-    tt.transport_traffic_days_[b] = bitfield_idx_t{0U};  // disable trip 'b'
+  auto const merge_and_nullify = [&tt](transport_idx_t const x,
+                                       transport_idx_t const y) {
+    tt.transport_traffic_days_[x] = bitfield_idx_t{0U};  // disable transport x
 
-    for (auto const merged_trips_idx_b : tt.transport_to_trip_section_[b]) {
-      for (auto const b_trp : tt.merged_trips_[merged_trips_idx_b]) {
-        for (auto& [t, range] : tt.trip_transport_ranges_[b_trp]) {
-          if (t == b) {
-            t = a;  // replace b with a in b's trip transport ranges
+    for (auto const merged_trips_idx_x : tt.transport_to_trip_section_[x]) {
+      for (auto const x_trp : tt.merged_trips_[merged_trips_idx_x]) {
+        for (auto& [t, range] : tt.trip_transport_ranges_[x_trp]) {
+          if (t == x) {
+            t = y;  // replace x with y in x's trip transport ranges
           }
         }
       }
     }
+  };
+
+  auto const is_superset = [](bitfield const& x, bitfield const& y) {
+    return (x & y) == x;
+  };
+
+  if (is_superset(bf_b, bf_a)) {
+    merge_and_nullify(a, b);
+  } else if (is_superset(bf_a, bf_b)) {
+    merge_and_nullify(b, a);
   } else {
-    tt.transport_traffic_days_[a] = tt.register_bitfield(bf_a | bf_b);
-    tt.transport_traffic_days_[b] = tt.register_bitfield(bf_b & ~bf_a);
+    tt.transport_traffic_days_[a] = tt.register_bitfield(bf_a & ~(bf_a & bf_b));
 
     hash_set<trip_idx_t> b_trips;
     for (auto const merged_trips_idx_b : tt.transport_to_trip_section_[b]) {
@@ -76,7 +88,6 @@ bool merge(timetable& tt,
 }
 
 unsigned find_duplicates(timetable& tt,
-                         match_set_t const& matches,
                          location_idx_t const a,
                          location_idx_t const b) {
   auto merged = 0U;
@@ -100,18 +111,29 @@ unsigned find_duplicates(timetable& tt,
         continue;
       }
 
-      for (auto const [x, y] : utl::zip(a_loc_seq, b_loc_seq)) {
-        if (!matches.contains(make_match_pair(stop{x}.location_idx(),
-                                              stop{y}.location_idx()))) {
-          continue;
-        }
+      auto const station_sequence_matches = [&]() {
+        return utl::all_of(utl::zip(a_loc_seq, b_loc_seq), [&](auto&& pair) {
+          auto const [x, y] = pair;
+          return matches(tt, routing::location_match_mode::kEquivalent,
+                         stop{x}.location_idx(), stop{y}.location_idx());
+        });
+      };
+
+      if (!station_sequence_matches()) {
+        continue;
       }
 
       auto const a_transport_range = tt.route_transport_ranges_[a_route];
       auto const b_transport_range = tt.route_transport_ranges_[b_route];
       auto a_t = begin(a_transport_range), b_t = begin(b_transport_range);
 
       while (a_t != end(a_transport_range) && b_t != end(b_transport_range)) {
+        if (*a_t == *b_t) {
+          ++a_t;
+          ++b_t;
+          continue;
+        }
+
         auto const time_a = tt.event_mam(a_route, *a_t, 0U, event_type::kDep);
         auto const time_b = tt.event_mam(b_route, *b_t, 0U, event_type::kDep);
 

src/query_generator/generator.cc

@@ -1,7 +1,7 @@
 #include "nigiri/query_generator/generator.h"
 
+#include "nigiri/location_match_mode.h"
 #include "nigiri/logging.h"
-#include "nigiri/routing/location_match_mode.h"
 #include "nigiri/routing/ontrip_train.h"
 #include "nigiri/special_stations.h"
 #include "nigiri/timetable.h"

src/routing/dijkstra.cc

@@ -6,7 +6,7 @@
 
 #include "nigiri/common/dial.h"
 #include "nigiri/footpath.h"
-#include "nigiri/routing/for_each_meta.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/routing/limits.h"
 #include "nigiri/routing/query.h"
 

src/routing/get_fastest_direct.cc

@@ -3,8 +3,8 @@
 #include "utl/get_or_create.h"
 
 #include "nigiri/common/dial.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/routing/dijkstra.h"
-#include "nigiri/routing/for_each_meta.h"
 #include "nigiri/special_stations.h"
 
 namespace nigiri::routing {

src/routing/raptor/optimize_footpaths.cc

@@ -2,7 +2,7 @@
 
 #include "utl/overloaded.h"
 
-#include "nigiri/routing/for_each_meta.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/routing/journey.h"
 #include "nigiri/rt/frun.h"
 #include "nigiri/rt/rt_timetable.h"

src/routing/raptor/reconstruct.cc

@@ -8,7 +8,7 @@
 #include "utl/overloaded.h"
 
 #include "nigiri/common/delta_t.h"
-#include "nigiri/routing/for_each_meta.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/routing/journey.h"
 #include "nigiri/routing/raptor/raptor_state.h"
 #include "nigiri/rt/frun.h"

src/routing/start_times.cc

@@ -1,6 +1,6 @@
 #include "nigiri/routing/start_times.h"
 
-#include "nigiri/routing/for_each_meta.h"
+#include "nigiri/for_each_meta.h"
 #include "nigiri/rt/rt_timetable.h"
 #include "nigiri/special_stations.h"
 #include "utl/enumerate.h"