vdv update (#119)

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* clang-tidy

* clang-format

* clang-format

* wip

* wip

* direction_id matching

* direction_id matching

* wip

* wip

* direction_id matching

* wip

* wip

* wip

* wip

* wip

* clang-format-17

* wip

* wip

* wip

* wip

* select all IstFahrt nodes in update messages, regardless of where they are located

* skip updates without stops, only log them

* skip updates without stops, only log them

* minimized timetables for case studies of unmatchable runs

* redirected update event output to file, adds test case for matching prevented by line name

* allow reference time differences of 1 minute

* use postfix matching for line names

* candidate struct to improve readability, discard subsequent matches if stop_idx is lower than the first matching stop_idx

* smd712 test case

* rbo920 test case

* maintain mapping vdv run id -> rt::run

* import run.h

* removes early termination when searching run

* removes early termination when searching run

* fix vdv run id parsing

* no result if there are multiple matches

* require at least half of the stops to match

* use matching ratio as tie breaker

* comment for test case ovo65

* rbo707 test case, requires iterating routes at equivalent stations to pass

* iterate routes at equivalent locations

* fix macos ci

* fix macos ci

* clang-tidy

* clang-tidy

* clang-format-17

* tolerate time discrepancies of up to 5 minutes between GTFS reference timetable and VDV update messages

* vgm 456 test case

* aggregate time error at matching stops

* allow time discrepancies of up to 10 min

* rvs347 & rbo512 test cases

* use total length of route as tiebreaker

* fix stats

* vgm418 test case

* reverse iteration

* monotonize rt times after vdv update

* vgm418: check for monotone rt times after update

* msvc fix: split update_vgm418

* optional debug logging

* optional debug logging

* wip

* rvs360 testcase

* scoring with quadratic error, refactoring

* early bailout if local_score < 0

* variable naming convention

* cleanup

* clang-format-17

* only count first update for unmatchable run in stats

* rm parameter name in declaration

* moves implementation of updater and vdv_stop to source file

* cleanup

* change name of limits for backward iteration

* const vdv_ev

* dbg output macro without std format

* remove time_since_epoch

* remove string append

* reverse iterator

* fix import

* rm vdv update exe

* fix reverse iterator

* fix reverse iterator

* fix dbg output, single update event function, fix monotonize

* rm unused import

* clang-tidy

* mam_dist

* descriptive test case names

* second return value signals if and in which direction midnight is traversed

* test cases with time discrepancies that traverse midnight

* handle time discrepancies that traverse midnight when matching

* switch off debug logging by default

* mam_dist descriptive types

* mam_dist comment

* rm empty line

* rm empty line

* parse_time function, tests for behavior on invalid input

* parse_time exception handling

* parse_time exception handling

* import sstream

* param name

* format

* refactor

* fwd declare pugixml structs

* refactor

* fix msvc ci

* rm pugi linkage

* class fwd declarations for pugi

---------

Co-authored-by: Felix Gündling <[email protected]>

.pkg

@@ -50,3 +50,7 @@
   [email protected]:motis-project/opentelemetry-cpp.git
   branch=main
   commit=ec4aef6b17b697052edef5417825ad71947b2ed1
+[pugixml]
+  [email protected]:motis-project/pugixml.git
+  branch=master
+  commit=60175e80e2f5e97e027ac78f7e14c5acc009ce50

CMakeLists.txt

@@ -83,11 +83,10 @@ endif ()
 file(GLOB_RECURSE nigiri-files src/*.cc)
 add_library(nigiri ${nigiri-files})
 target_include_directories(nigiri PUBLIC include)
-target_link_libraries(nigiri PUBLIC cista geo utl fmt date miniz date-tz wyhash unordered_dense gtfsrt oh opentelemetry_api)
+target_link_libraries(nigiri PUBLIC cista geo utl fmt date miniz date-tz wyhash unordered_dense gtfsrt oh opentelemetry_api pugixml)
 target_compile_features(nigiri PUBLIC cxx_std_23)
 target_compile_options(nigiri PRIVATE ${nigiri-compile-options})
 
-
 # --- MAIN ---
 file(GLOB_RECURSE nigiri-server-files server/main.cc)
 add_executable(nigiri-server ${nigiri-server-files})

include/nigiri/common/mam_dist.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdint>
+#include <utility>
+
+#include "nigiri/types.h"
+
+namespace nigiri {
+
+/*
+ * Computes the minimal distance between two minutes after midnight values
+ * within [0, 1440[, i.e., if the shorter distance between the two values spans
+ * midnight that distance is returned
+ * the first value of the returned pair contains the absolute distance between
+ * the two values
+ * the second value of the returned pair signals if and in which direction
+ * midnight is passed:
+ * -1: [actual -- midnight -- expected]
+ *  0: [actual -- expected] / [expected -- actual]
+ * +1: [expected -- midnight -- actual]
+ */
+std::pair<i32_minutes, date::days> mam_dist(i32_minutes expected,
+                                            i32_minutes actual);
+
+}  // namespace nigiri

include/nigiri/common/parse_time.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "nigiri/types.h"
+
+namespace nigiri {
+
+unixtime_t parse_time(std::string const&);
+
+}  // namespace nigiri

include/nigiri/rt/frun.h

@@ -91,13 +91,16 @@ struct frun : public run {
     using value_type = run_stop;
     using pointer = run_stop;
     using reference = run_stop;
-    using iterator_category = std::forward_iterator_tag;
+    using iterator_category = std::bidirectional_iterator_tag;
 
     iterator& operator++() noexcept;
     iterator operator++(int) noexcept;
 
-    bool operator==(iterator const o) const noexcept;
-    bool operator!=(iterator o) const noexcept;
+    iterator& operator--() noexcept;
+    iterator operator--(int) noexcept;
+
+    bool operator==(iterator const) const noexcept;
+    bool operator!=(iterator) const noexcept;
 
     run_stop operator*() const noexcept;
 
@@ -113,8 +116,14 @@ struct frun : public run {
   iterator begin() const noexcept;
   iterator end() const noexcept;
 
-  friend iterator begin(frun const& fr) noexcept;
-  friend iterator end(frun const& fr) noexcept;
+  friend iterator begin(frun const&) noexcept;
+  friend iterator end(frun const&) noexcept;
+
+  std::reverse_iterator<iterator> rbegin() const noexcept;
+  std::reverse_iterator<iterator> rend() const noexcept;
+
+  friend std::reverse_iterator<iterator> rbegin(frun const&) noexcept;
+  friend std::reverse_iterator<iterator> rend(frun const&) noexcept;
 
   stop_idx_t first_valid(stop_idx_t from = 0U) const;
   stop_idx_t last_valid() const;
@@ -127,7 +136,7 @@ struct frun : public run {
   trip_idx_t trip_idx() const;
   clasz get_clasz() const noexcept;
 
-  void print(std::ostream&, interval<stop_idx_t> stop_range);
+  void print(std::ostream&, interval<stop_idx_t>);
   friend std::ostream& operator<<(std::ostream&, frun const&);
 
   static frun from_rt(timetable const&,

include/nigiri/rt/vdv/vdv_update.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "nigiri/rt/run.h"
+#include "nigiri/types.h"
+
+namespace pugi {
+class xml_document;
+class xml_node;
+}  // namespace pugi
+
+namespace nigiri {
+struct rt_timetable;
+struct timetable;
+}  // namespace nigiri
+
+namespace nigiri::rt::vdv {
+
+struct statistics {
+  friend std::ostream& operator<<(std::ostream&, statistics const&);
+  friend statistics& operator+=(statistics&, statistics const&);
+
+  std::uint32_t unsupported_additional_runs_{0U};
+  std::uint32_t unsupported_cancelled_runs_{0U};
+  std::uint32_t total_stops_{0U};
+  std::uint32_t resolved_stops_{0U};
+  std::uint32_t unknown_stops_{0U};
+  std::uint32_t unsupported_additional_stops_{0U};
+  std::uint32_t total_runs_{0U};
+  std::uint32_t no_transport_found_at_stop_{0U};
+  std::uint32_t search_on_incomplete_{0U};
+  std::uint32_t found_runs_{0U};
+  std::uint32_t multiple_matches_{0U};
+  std::uint32_t matched_runs_{0U};
+  std::uint32_t unmatchable_runs_{0U};
+  std::uint32_t runs_without_stops_{0U};
+  std::uint32_t skipped_vdv_stops_{0U};
+  std::uint32_t excess_vdv_stops_{0U};
+  std::uint32_t updated_events_{0U};
+  std::uint32_t propagated_delays_{0U};
+};
+
+struct updater {
+  static constexpr auto const kExactMatchScore = 1000;
+  static constexpr auto const kAllowedTimeDiscrepancy = []() {
+    auto error = 0;
+    while (kExactMatchScore - error * error > 0) {
+      ++error;
+    }
+    return error - 1;
+  }();  // minutes
+
+  updater(timetable const&, source_idx_t);
+
+  void reset_vdv_run_ids_();
+
+  statistics const& get_stats() const;
+
+  void update(rt_timetable&, pugi::xml_document const&);
+
+private:
+  static std::optional<unixtime_t> get_opt_time(pugi::xml_node const&,
+                                                char const*);
+
+  struct vdv_stop {
+    explicit vdv_stop(location_idx_t, std::string_view id, pugi::xml_node);
+
+    std::optional<std::pair<unixtime_t, event_type>> get_event(
+        event_type et) const;
+
+    location_idx_t l_;
+    std::string_view id_;
+    std::optional<unixtime_t> dep_, arr_, rt_dep_, rt_arr_;
+  };
+
+  vector<vdv_stop> resolve_stops(pugi::xml_node vdv_run);
+
+  std::optional<rt::run> find_run(std::string_view vdv_run_id,
+                                  vector<vdv_stop> const&,
+                                  bool is_complete_run);
+
+  void update_run(rt_timetable&,
+                  run&,
+                  vector<vdv_stop> const&,
+                  bool is_complete_run);
+
+  void process_vdv_run(rt_timetable&, pugi::xml_node vdv_run);
+
+  timetable const& tt_;
+  source_idx_t src_idx_;
+  statistics stats_{};
+  hash_map<std::string, run> vdv_nigiri_{};
+};
+
+}  // namespace nigiri::rt::vdv

include/nigiri/timetable.h

@@ -90,6 +90,12 @@ struct timetable {
       return get(location_id_to_idx_.at(id));
     }
 
+    std::optional<location> find(location_id const& id) const {
+      auto const it = location_id_to_idx_.find(id);
+      return it == end(location_id_to_idx_) ? std::nullopt
+                                            : std::optional{get(it->second)};
+    }
+
     void resolve_timezones();
 
     // Station access: external station id -> internal station idx
@@ -485,4 +491,4 @@ struct timetable {
   hash_map<string, profile_idx_t> profiles_;
 };
 
-}  // namespace nigiri
+}  // namespace nigiri

src/common/mam_dist.cc

@@ -0,0 +1,23 @@
+#include "nigiri/common/mam_dist.h"
+
+#include <cassert>
+#include <cstdlib>
+
+namespace nigiri {
+
+std::pair<i32_minutes, date::days> mam_dist(i32_minutes const expected,
+                                            i32_minutes const actual) {
+  assert(i32_minutes{0} <= expected && expected < i32_minutes{1440});
+  assert(i32_minutes{0} <= actual && actual < i32_minutes{1440});
+
+  auto const diff = (expected - actual).count();
+  auto const abs = std::abs(diff);
+
+  if (abs > 1440 / 2) {
+    return {i32_minutes{1440 - abs}, date::days{diff < 0 ? -1 : +1}};
+  } else {
+    return {i32_minutes{abs}, date::days{0}};
+  }
+}
+
+}  // namespace nigiri

src/common/parse_time.cc

@@ -0,0 +1,15 @@
+#include "nigiri/common/parse_time.h"
+
+#include <sstream>
+
+namespace nigiri {
+
+unixtime_t parse_time(std::string const& str) {
+  unixtime_t parsed;
+  auto ss = std::stringstream{str};
+  ss.exceptions(std::ios::badbit | std::ios::failbit | std::ios::eofbit);
+  ss >> date::parse("%FT%T", parsed);
+  return parsed;
+}
+
+}  // namespace nigiri

src/loader/gtfs/load_timetable.cc

@@ -375,4 +375,4 @@ void load_timetable(loader_config const& config,
   }
 }
 
-}  // namespace nigiri::loader::gtfs
+}  // namespace nigiri::loader::gtfs

src/loader/gtfs/trip.cc

@@ -386,4 +386,4 @@ void read_frequencies(trip_data& trips, std::string_view file_content) {
          });
 }
 
-}  // namespace nigiri::loader::gtfs
+}  // namespace nigiri::loader::gtfs

src/rt/frun.cc

@@ -250,6 +250,21 @@ frun::iterator frun::iterator::operator++(int) noexcept {
   return r;
 }
 
+frun::iterator& frun::iterator::operator--() noexcept {
+  do {
+    --rs_.stop_idx_;
+  } while (rs_.stop_idx_ !=
+               static_cast<stop_idx_t>(rs_.fr_->stop_range_.from_ - 1U) &&
+           rs_.is_canceled());
+  return *this;
+}
+
+frun::iterator frun::iterator::operator--(int) noexcept {
+  auto r = *this;
+  --(*this);
+  return r;
+}
+
 bool frun::iterator::operator==(iterator const o) const noexcept {
   return rs_ == o.rs_;
 }
@@ -319,6 +334,21 @@ frun::iterator frun::end() const noexcept {
 frun::iterator begin(frun const& fr) noexcept { return fr.begin(); }
 frun::iterator end(frun const& fr) noexcept { return fr.end(); }
 
+std::reverse_iterator<frun::iterator> frun::rbegin() const noexcept {
+  return std::make_reverse_iterator(end());
+}
+
+std::reverse_iterator<frun::iterator> frun::rend() const noexcept {
+  return std::make_reverse_iterator(begin());
+}
+
+std::reverse_iterator<frun::iterator> rbegin(frun const& fr) noexcept {
+  return fr.rbegin();
+}
+std::reverse_iterator<frun::iterator> rend(frun const& fr) noexcept {
+  return fr.rend();
+}
+
 stop_idx_t frun::size() const noexcept {
   return static_cast<stop_idx_t>(
       (is_rt() && rtt_ != nullptr)