Update to latest mp4 rev (rerun-io#7470)

### What - Updates `mp4` to the latest commit on https://github.com/rerun-io/re_mp4/tree/jan/fixes We should land the following PR first: - rerun-io/re_mp4#4 (we can leave that one be and land this, but then we'd have to update the `rev` again once that lands) ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7470?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7470?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7470) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`.
grtlr · Sep 23, 2024 · de49fce · de49fce
1 parent f287587
commit de49fce
Show file tree

Hide file tree

Showing 10 changed files with 219 additions and 155 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -3481,19 +3481,6 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "mp4"
-version = "0.14.0"
-source = "git+https://github.com/rerun-io/mp4?rev=b019fcf3ac8e1befded54728686997cc168117c8#b019fcf3ac8e1befded54728686997cc168117c8"
-dependencies = [
- "byteorder",
- "bytes",
- "num-rational",
- "serde",
- "serde_json",
- "thiserror",
-]
-
 [[package]]
 name = "multimap"
 version = "0.10.0"
@@ -5239,6 +5226,20 @@ dependencies = [
  "web-time",
 ]
 
+[[package]]
+name = "re_mp4"
+version = "0.1.0"
+source = "git+https://github.com/rerun-io/re_mp4?rev=b2c39c374ed33c6e377dc368d6eba3fe051c61e3#b2c39c374ed33c6e377dc368d6eba3fe051c61e3"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "log",
+ "num-rational",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
 [[package]]
 name = "re_query"
 version = "0.19.0-alpha.1+dev"
@@ -5811,8 +5812,7 @@ name = "re_video"
 version = "0.19.0-alpha.1+dev"
 dependencies = [
  "itertools 0.13.0",
- "mp4",
- "ordered-float",
+ "re_mp4",
  "thiserror",
 ]
 

diff --git a/Cargo.toml b/Cargo.toml
@@ -196,7 +196,7 @@ memory-stats = "1.1"
 mimalloc = "0.1.37"
 mime_guess2 = "2.0" # infer MIME type by file extension, and map mime to file extension
 mint = "0.5.9"
-mp4 = "0.14.0"
+re_mp4 = "0.1.0"
 natord = "1.0.9"
 ndarray = "0.16"
 ndarray-rand = "0.15"
@@ -537,9 +537,9 @@ egui_tiles = { git = "https://github.com/rerun-io/egui_tiles", rev = "b2f5e23252
 
 egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark", rev = "7a9dc755bfa351a3796274cb8ca87129b051c084" } # https://github.com/lampsitter/egui_commonmark/pull/65
 
-# commit on `rerun-io/mp4` `main` branch
-# https://github.com/rerun-io/mp4/commit/b019fcf3ac8e1befded54728686997cc168117c8
-mp4 = { git = "https://github.com/rerun-io/mp4", rev = "b019fcf3ac8e1befded54728686997cc168117c8" }
+# commit on `rerun-io/mp4` `master` branch: https://github.com/rerun-io/re_mp4/tree/master
+# https://github.com/rerun-io/mp4/commit/b2c39c374ed33c6e377dc368d6eba3fe051c61e3
+re_mp4 = { git = "https://github.com/rerun-io/re_mp4", rev = "b2c39c374ed33c6e377dc368d6eba3fe051c61e3" }
 
 # commit on `rerun-io/re_arrow2` `main` branch
 # https://github.com/rerun-io/re_arrow2/commit/e4717d6debc6d4474ec10db8f629f823f57bad07

diff --git a/crates/store/re_video/Cargo.toml b/crates/store/re_video/Cargo.toml
@@ -27,6 +27,5 @@ features = ["all"]
 
 [dependencies]
 itertools.workspace = true
-mp4.workspace = true
-ordered-float.workspace = true
+re_mp4.workspace = true
 thiserror.workspace = true
diff --git a/crates/store/re_video/src/lib.rs b/crates/store/re_video/src/lib.rs
@@ -5,21 +5,33 @@
 
 mod mp4;
 
+use std::ops::Range;
+
 use itertools::Itertools;
-use ordered_float::OrderedFloat;
 
 /// Decoded video data.
 #[derive(Clone)]
 pub struct VideoData {
     pub config: Config,
 
-    /// Duration of the video, in milliseconds.
-    pub duration: TimeMs,
+    /// How many time units are there per second.
+    pub timescale: Timescale,
+
+    /// Duration of the video, in time units.
+    pub duration: Time,
 
     /// We split video into segments, each beginning with a key frame,
     /// followed by any number of delta frames.
     pub segments: Vec<Segment>,
 
+    /// Samples contain the byte offsets into `data` for each frame.
+    ///
+    /// This list is sorted in ascending order of decode timestamps.
+    ///
+    /// Samples must be decoded in decode-timestamp order,
+    /// and should be presented in composition-timestamp order.
+    pub samples: Vec<Sample>,
+
     /// This array stores all data used by samples.
     pub data: Vec<u8>,
 }
@@ -56,9 +68,9 @@ impl VideoData {
         // Segments are guaranteed to be sorted among each other, but within a segment,
         // presentation timestamps may not be sorted since this is sorted by decode timestamps.
         self.segments.iter().flat_map(|seg| {
-            seg.samples
+            self.samples[seg.range()]
                 .iter()
-                .map(|sample| sample.timestamp.as_nanos())
+                .map(|sample| sample.composition_timestamp.into_nanos(self.timescale))
                 .sorted()
         })
     }
@@ -67,23 +79,36 @@ impl VideoData {
 /// A segment of a video.
 #[derive(Clone)]
 pub struct Segment {
-    /// Time of the first sample in this segment, in milliseconds.
-    pub timestamp: TimeMs,
+    /// Decode timestamp of the first sample in this segment, in time units.
+    pub start: Time,
 
-    /// List of samples contained in this segment.
-    /// At least one sample per segment is guaranteed,
-    /// and the first sample is always a key frame.
-    pub samples: Vec<Sample>,
+    /// Range of samples contained in this segment.
+    pub sample_range: Range<u32>,
+}
+
+impl Segment {
+    /// The segment's `sample_range` mapped to `usize` for slicing.
+    pub fn range(&self) -> Range<usize> {
+        Range {
+            start: self.sample_range.start as usize,
+            end: self.sample_range.end as usize,
+        }
+    }
 }
 
 /// A single sample in a video.
 #[derive(Debug, Clone)]
 pub struct Sample {
-    /// Time at which this sample appears, in milliseconds.
-    pub timestamp: TimeMs,
+    /// Time at which this sample appears in the decoded bitstream, in time units.
+    pub decode_timestamp: Time,
 
-    /// Duration of the sample, in milliseconds.
-    pub duration: TimeMs,
+    /// Time at which this sample appears in the frame stream, in time units.
+    ///
+    /// `composition >= decode`
+    pub composition_timestamp: Time,
+
+    /// Duration of the sample, in time units.
+    pub duration: Time,
 
     /// Offset into [`VideoData::data`]
     pub byte_offset: u32,
@@ -108,51 +133,80 @@ pub struct Config {
     pub coded_width: u16,
 }
 
-#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub struct TimeMs(OrderedFloat<f64>);
+/// A value in time units.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Time(u64);
 
-impl TimeMs {
-    pub const ZERO: Self = Self(OrderedFloat(0.0));
+impl Time {
+    pub const ZERO: Self = Self(0);
 
+    /// Create a new value in _time units_.
+    ///
+    /// ⚠️ Don't use this for regular timestamps in seconds/milliseconds/etc.,
+    /// use the proper constructors for those instead!
+    /// This only exists for cases where you already have a value expressed in time units,
+    /// such as those received from the `WebCodecs` APIs.
     #[inline]
-    pub fn new(ms: f64) -> Self {
-        Self(OrderedFloat(ms))
+    pub fn new(v: u64) -> Self {
+        Self(v)
     }
 
     #[inline]
-    pub fn as_ms_f64(&self) -> f64 {
-        self.0.into_inner()
+    pub fn from_secs(v: f64, timescale: Timescale) -> Self {
+        Self((v * timescale.0 as f64).round() as u64)
     }
 
     #[inline]
-    pub fn as_nanos(self) -> i64 {
-        (self.0 * 1_000_000.0).round() as i64
+    pub fn from_millis(v: f64, timescale: Timescale) -> Self {
+        Self::from_secs(v / 1e3, timescale)
     }
-}
 
-impl std::ops::Add<Self> for TimeMs {
-    type Output = Self;
+    #[inline]
+    pub fn from_micros(v: f64, timescale: Timescale) -> Self {
+        Self::from_secs(v / 1e6, timescale)
+    }
 
     #[inline]
-    fn add(self, rhs: Self) -> Self::Output {
-        Self(self.0 + rhs.0)
+    pub fn from_nanos(v: i64, timescale: Timescale) -> Self {
+        Self::from_secs(v as f64 / 1e9, timescale)
+    }
+
+    #[inline]
+    pub fn into_secs(self, timescale: Timescale) -> f64 {
+        self.0 as f64 / timescale.0 as f64
     }
-}
 
-impl std::ops::Sub<Self> for TimeMs {
-    type Output = Self;
+    #[inline]
+    pub fn into_millis(self, timescale: Timescale) -> f64 {
+        self.into_secs(timescale) * 1e3
+    }
 
     #[inline]
-    fn sub(self, rhs: Self) -> Self::Output {
-        Self(self.0 - rhs.0)
+    pub fn into_micros(self, timescale: Timescale) -> f64 {
+        self.into_secs(timescale) * 1e6
+    }
+
+    #[inline]
+    pub fn into_nanos(self, timescale: Timescale) -> i64 {
+        (self.into_secs(timescale) * 1e9).round() as i64
+    }
+}
+
+/// The number of time units per second.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Timescale(u64);
+
+impl Timescale {
+    pub(crate) fn new(v: u64) -> Self {
+        Self(v)
     }
 }
 
 /// Errors that can occur when loading a video.
 #[derive(thiserror::Error, Debug)]
 pub enum VideoLoadError {
     #[error("Failed to determine media type from data: {0}")]
-    ParseMp4(#[from] ::mp4::Error),
+    ParseMp4(#[from] re_mp4::Error),
 
     #[error("Video file has no video tracks")]
     NoVideoTrack,
@@ -169,8 +223,9 @@ pub enum VideoLoadError {
     #[error("Video file has unsupported format")]
     UnsupportedVideoType,
 
-    #[error("Video file has unsupported codec {0}")]
-    UnsupportedCodec(String),
+    // `FourCC`'s debug impl doesn't quote the result
+    #[error("Video track uses unsupported codec \"{0}\"")] // NOLINT
+    UnsupportedCodec(re_mp4::FourCC),
 }
 
 impl std::fmt::Debug for VideoData {
@@ -187,8 +242,8 @@ impl std::fmt::Debug for VideoData {
 impl std::fmt::Debug for Segment {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("Segment")
-            .field("timestamp", &self.timestamp)
-            .field("samples", &self.samples.len())
+            .field("timestamp", &self.start)
+            .field("samples", &self.sample_range.len())
             .finish()
     }
 }