Bug 1642415 - Rerun mach vendor rust; r=platform-i18n-reviewers,gregt…

…atum

Differential Revision: https://phabricator.services.mozilla.com/D125236

.cargo/config.in

@@ -20,7 +20,7 @@ rev = "72eb355ddeada541d7e57dbe5fb60eb5124dea0d"
 [source."https://github.com/mozilla/l10nregistry-rs"]
 git = "https://github.com/mozilla/l10nregistry-rs"
 replace-with = "vendored-sources"
-rev = "a69df9836b1ef536727195209013b9ad6b132618"
+rev = "fd02134161505f978e132114cbabdac057ce3b71"
 
 [source."https://github.com/mozilla/cubeb-pulse-rs"]
 git = "https://github.com/mozilla/cubeb-pulse-rs"

third_party/rust/l10nregistry/.cargo-checksum.json

@@ -1 +1 @@
-{"files":{"Cargo.toml":"5874fa7dbce795209a9312250fee6db8c807ddfa53df4926bebcc9b718bc2281","LICENSE-APACHE":"5db2b182453ff32ed40f7da63589c9667a3f8bd8b16b1471b152caae56f77e45","LICENSE-MIT":"49c0b000c03731d9e3970dc059ad4ca345d773681f4a612b0024435b663e0220","benches/localization.rs":"879b46b5cc6b9c8bade51796acb0de0c5dbaa197c24b644592db06f976d39208","benches/preferences.rs":"8f5242b2166896da6219be57c5cb48f5b995a884b3bafa08358ce5206b3ad098","benches/solver.rs":"dbffd67c583b76b64499b9bf5163e40b7238afafa7edee5aabb2f04e956a15a6","benches/source.rs":"5efb5e562509aeab6a643739251d2b31994e0240316ecf407ca2188f14cbcf5d","src/env.rs":"213679dd9ef5d2aa8049f0351c2caa00e8890f788ff8a237cd8fccb01d061cdb","src/errors.rs":"f025bd1674c16b85b3097bf86831f0e8a0860f00b7151293e94206e456bd6959","src/fluent.rs":"ccf3ed58e1281e28ab36e75157778452d7c5323f5b08ff48fbea7bb45b70d361","src/lib.rs":"c2bba574aa4c1258d88695f7410175ccf04c3adf1ea3fb10769a0c9f4e07b133","src/registry/asynchronous.rs":"a392c5463caf7eeb9dc27c311d08361337b14fc6b05e83fedbe5683c66967a94","src/registry/mod.rs":"8b5077b30d3a0bde13270552009eaad0aece9c447f3c5fe210b56a1aedc6cddf","src/registry/synchronous.rs":"ca1554f88a492314be7b764b18fb39dcfcd5f47005f7364c05c1e0eb171831bf","src/solver/README.md":"bd93b3faef6adec5e71f06a7e06ca64b32f3877b513de099638fa86acf1e2d9e","src/solver/mod.rs":"f763519feca67ce6ce42c4387482fb10313f9f33f7d0fe687e0db51ef9592a6b","src/solver/parallel.rs":"c4c4edc38d8f639b0df057c12e0f4f0e0026e3f2fb3ad9563689d2620dd99fea","src/solver/serial.rs":"dcac1128fc87295b955c11b40b7adca5106a585de9adc2e4361cc6a92c968e27","src/solver/testing/mod.rs":"9f8b3d5b3436088d410d5544d2c2e33320bb56b0d364f1862790e4ab66fb7764","src/solver/testing/scenarios.rs":"2a11b2f3a6bbfe0055ea6125cae498391e28a4811c019e2f9479f959798e5767","src/source/fetcher.rs":"6220bf5010df94eb92f1adc56f76a65b41227b59312e27963d75b16ea0ee6393","src/source/mod.rs":"3e507f5775f71abc55acdd3e9a4be34f241c4e889244caae0142e0bc2ecae109","src/testing.rs":"4408fc78efb0d3dd52d4f8cb669914ef96531fd94796a5a2f4e1f65bc54f0e5f","tests/localization.rs":"5c2f4e75f03d4e42880adde4a6902e60b0c87d14160663e495cc6ed3091b4285","tests/registry.rs":"f9da785b408f0d54b2ca8f0bd7f52e86e75e9a642fbb91f57be870160024cbb6","tests/scenarios.rs":"ffe16cb382895bdbe809f63233fc1ac9a4be9022bc7b60d2f3210b49653569e8","tests/source.rs":"c43bbcc6f2e0c34766785cd1be94a3ed34f7f6327a1693f5dce62bcae7ac1d39"},"package":null}
+{"files":{"Cargo.toml":"24b85576fd6aa7ab9fc9522f0b1467b0030590e42bb8a03bdb9b8dcda68c8755","LICENSE-APACHE":"5db2b182453ff32ed40f7da63589c9667a3f8bd8b16b1471b152caae56f77e45","LICENSE-MIT":"49c0b000c03731d9e3970dc059ad4ca345d773681f4a612b0024435b663e0220","benches/localization.rs":"879b46b5cc6b9c8bade51796acb0de0c5dbaa197c24b644592db06f976d39208","benches/preferences.rs":"8f5242b2166896da6219be57c5cb48f5b995a884b3bafa08358ce5206b3ad098","benches/registry.rs":"fe34e5dd459991524e799aa00123e353ba122a94426ea187006aa4a514c8d308","benches/solver.rs":"dbffd67c583b76b64499b9bf5163e40b7238afafa7edee5aabb2f04e956a15a6","benches/source.rs":"1db1c142b019e0501538df1b4895e9669e4851ef8a59a40c641fd0c127bb34a4","src/env.rs":"213679dd9ef5d2aa8049f0351c2caa00e8890f788ff8a237cd8fccb01d061cdb","src/errors.rs":"f025bd1674c16b85b3097bf86831f0e8a0860f00b7151293e94206e456bd6959","src/fluent.rs":"ccf3ed58e1281e28ab36e75157778452d7c5323f5b08ff48fbea7bb45b70d361","src/lib.rs":"c2bba574aa4c1258d88695f7410175ccf04c3adf1ea3fb10769a0c9f4e07b133","src/registry/asynchronous.rs":"49eb65633ada3630922fe4e2845d7059d86ff238bbf06d9796281db696f06d95","src/registry/mod.rs":"0bd8162ca10585d2230607ee694a952ffb8501a971a756382d747279851141f9","src/registry/synchronous.rs":"a06e47df7f331c9d441bd6bcbbbf96f3634836d6d8ebc3f7cd01b75606b65c18","src/solver/README.md":"bd93b3faef6adec5e71f06a7e06ca64b32f3877b513de099638fa86acf1e2d9e","src/solver/mod.rs":"f763519feca67ce6ce42c4387482fb10313f9f33f7d0fe687e0db51ef9592a6b","src/solver/parallel.rs":"34cae8cf80f54d3f9c7e3eb60df6c33ab97a70f0e9ac0306a8d098d24cfe3c6a","src/solver/serial.rs":"dcac1128fc87295b955c11b40b7adca5106a585de9adc2e4361cc6a92c968e27","src/solver/testing/mod.rs":"3e1481bb6a73cd283d544553341df85f5adc88e6f28914ee10615b648bb927ff","src/solver/testing/scenarios.rs":"2a11b2f3a6bbfe0055ea6125cae498391e28a4811c019e2f9479f959798e5767","src/source/fetcher.rs":"6220bf5010df94eb92f1adc56f76a65b41227b59312e27963d75b16ea0ee6393","src/source/mod.rs":"b09d89dca3189405f9b37b86ab767307cf07ad0ce134c906aeb00f1cc0b889b9","src/testing.rs":"0c93632177c5c135bf8f26d25380b262f9b38314f434221a71b162aa2fe2335a","tests/localization.rs":"05a3a7612e6ccb9d7b119c9d7378b5da8229be506172dcf575ec1ebbbeb04256","tests/registry.rs":"a0b351d26eb2646eeba1d5e9eae947736304d7e4e1ebf1405d48e2efdb1ca389","tests/scenarios.rs":"ffe16cb382895bdbe809f63233fc1ac9a4be9022bc7b60d2f3210b49653569e8","tests/source.rs":"17dab145ef11ea9cbf8e986f049b5a8aca945e3dab7281ea1ae0f833831e48ba"},"package":null}

third_party/rust/l10nregistry/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "l10nregistry"
-version = "0.2.0"
+version = "0.3.0"
 authors = ["Zibi Braniecki <[email protected]>"]
 license = "Apache-2.0/MIT"
 edition = "2018"
@@ -29,18 +29,27 @@ tokio-io = ["tokio"]
 [[bench]]
 name = "preferences"
 harness = false
+required-features = ["tokio", "fluent-testing"]
 
 [[bench]]
 name = "localization"
 harness = false
+required-features = ["tokio", "fluent-testing"]
 
 [[bench]]
 name = "source"
 harness = false
+required-features = ["tokio", "fluent-testing"]
 
 [[bench]]
 name = "solver"
 harness = false
+required-features = ["tokio", "fluent-testing"]
+
+[[bench]]
+name = "registry"
+harness = false
+required-features = ["tokio", "fluent-testing"]
 
 [[test]]
 name = "source"

third_party/rust/l10nregistry/benches/registry.rs

@@ -0,0 +1,132 @@
+use criterion::criterion_group;
+use criterion::criterion_main;
+use criterion::Criterion;
+
+use futures::stream::StreamExt;
+use l10nregistry::testing::{FileSource, RegistrySetup, TestFileFetcher};
+use unic_langid::LanguageIdentifier;
+
+fn get_paths() -> Vec<String> {
+    let paths: Vec<&'static str> = vec![
+        "branding/brand.ftl",
+        "browser/sanitize.ftl",
+        "browser/preferences/blocklists.ftl",
+        "browser/preferences/colors.ftl",
+        "browser/preferences/selectBookmark.ftl",
+        "browser/preferences/connection.ftl",
+        "browser/preferences/addEngine.ftl",
+        "browser/preferences/siteDataSettings.ftl",
+        "browser/preferences/fonts.ftl",
+        "browser/preferences/languages.ftl",
+        "browser/preferences/preferences.ftl",
+        "security/certificates/certManager.ftl",
+        "security/certificates/deviceManager.ftl",
+        "toolkit/global/textActions.ftl",
+        "toolkit/printing/printUI.ftl",
+        "toolkit/updates/history.ftl",
+        "toolkit/featuregates/features.ftl",
+    ];
+
+    paths.iter().map(|s| s.to_string()).collect()
+}
+
+fn registry_bench(c: &mut Criterion) {
+    let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+    let mut group = c.benchmark_group("non-metasource");
+
+    let setup = RegistrySetup::new(
+        "test",
+        vec![
+            FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"),
+            FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"),
+            FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"),
+            FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"),
+        ],
+        vec![en_us.clone()],
+    );
+    let fetcher = TestFileFetcher::new();
+    let (_, reg) = fetcher.get_registry_and_environment(setup);
+
+    group.bench_function(&format!("serial",), |b| {
+        b.iter(|| {
+            let lang_ids = vec![en_us.clone()];
+            let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths());
+            while let Some(_) = i.next() {}
+        })
+    });
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    group.bench_function(&format!("parallel",), |b| {
+        b.iter(|| {
+            let lang_ids = vec![en_us.clone()];
+            let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths());
+            rt.block_on(async { while let Some(_) = i.next().await {} });
+        })
+    });
+
+    group.finish();
+}
+
+fn registry_metasource_bench(c: &mut Criterion) {
+    let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+    let mut group = c.benchmark_group("metasource");
+
+    let setup = RegistrySetup::new(
+        "test",
+        vec![
+            FileSource::new(
+                "toolkit",
+                Some("app"),
+                vec![en_us.clone()],
+                "toolkit/{locale}/",
+            ),
+            FileSource::new(
+                "browser",
+                Some("app"),
+                vec![en_us.clone()],
+                "browser/{locale}/",
+            ),
+            FileSource::new(
+                "toolkit",
+                Some("langpack"),
+                vec![en_us.clone()],
+                "toolkit/{locale}/",
+            ),
+            FileSource::new(
+                "browser",
+                Some("langpack"),
+                vec![en_us.clone()],
+                "browser/{locale}/",
+            ),
+        ],
+        vec![en_us.clone()],
+    );
+    let fetcher = TestFileFetcher::new();
+    let (_, reg) = fetcher.get_registry_and_environment(setup);
+
+    group.bench_function(&format!("serial",), |b| {
+        b.iter(|| {
+            let lang_ids = vec![en_us.clone()];
+            let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths());
+            while let Some(_) = i.next() {}
+        })
+    });
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    group.bench_function(&format!("parallel",), |b| {
+        b.iter(|| {
+            let lang_ids = vec![en_us.clone()];
+            let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths());
+            rt.block_on(async { while let Some(_) = i.next().await {} });
+        })
+    });
+
+    group.finish();
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(10);
+    targets = registry_bench, registry_metasource_bench
+);
+criterion_main!(benches);

third_party/rust/l10nregistry/benches/source.rs

@@ -27,7 +27,9 @@ fn source_bench(c: &mut Criterion) {
         let sources: Vec<_> = scenario
             .file_sources
             .iter()
-            .map(|s| fetcher.get_test_file_source(&s.name, get_locales(&s.locales), &s.path_scheme))
+            .map(|s| {
+                fetcher.get_test_file_source(&s.name, None, get_locales(&s.locales), &s.path_scheme)
+            })
             .collect();
 
         group.bench_function(format!("{}/has_file", scenario.name), |b| {

third_party/rust/l10nregistry/src/registry/asynchronous.rs

@@ -88,6 +88,7 @@ impl<P, B> State<P, B> {
 pub struct GenerateBundles<P, B> {
     reg: L10nRegistry<P, B>,
     locales: std::vec::IntoIter<LanguageIdentifier>,
+    current_metasource: usize,
     res_ids: Vec<String>,
     state: State<P, B>,
 }
@@ -101,6 +102,7 @@ impl<P, B> GenerateBundles<P, B> {
         Self {
             reg,
             locales,
+            current_metasource: 0,
             res_ids,
             state: State::Empty,
         }
@@ -133,7 +135,8 @@ impl<'l, P, B> AsyncTester for GenerateBundles<P, B> {
             .iter()
             .map(|(res_idx, source_idx)| {
                 let res = &self.res_ids[*res_idx];
-                lock.source_idx(*source_idx).fetch_file(locale, res)
+                lock.source_idx(self.current_metasource, *source_idx)
+                    .fetch_file(locale, res)
             })
             .collect::<FuturesOrdered<_>>();
         TestResult(stream.collect())
@@ -147,6 +150,23 @@ impl<P, B> BundleStream for GenerateBundles<P, B> {
     }
 }
 
+macro_rules! try_next_metasource {
+    ( $self:ident ) => {{
+        if $self.current_metasource > 0 {
+            $self.current_metasource -= 1;
+            let solver = ParallelProblemSolver::new(
+                $self.res_ids.len(),
+                $self.reg.lock().metasource_len($self.current_metasource),
+            );
+            $self.state = State::Solver {
+                locale: $self.state.get_locale().clone(),
+                solver,
+            };
+            continue;
+        }
+    }};
+}
+
 impl<P, B> Stream for GenerateBundles<P, B>
 where
     P: ErrorReporter,
@@ -164,6 +184,7 @@ where
                         Ok(Some(order)) => {
                             let locale = self.state.get_locale();
                             let bundle = self.reg.lock().bundle_from_order(
+                                self.current_metasource,
                                 locale.clone(),
                                 &order,
                                 &self.res_ids,
@@ -177,10 +198,14 @@ where
                             }
                         }
                         Ok(None) => {
+                            try_next_metasource!(self);
                             self.state = State::Empty;
                             continue;
                         }
                         Err(idx) => {
+                            try_next_metasource!(self);
+                            // Only signal an error if we run out of metasources
+                            // to try.
                             self.reg.shared.provider.report_errors(vec![
                                 L10nRegistryError::MissingResource {
                                     locale: self.state.get_locale().clone(),
@@ -197,7 +222,15 @@ where
                     }
                 }
             } else if let Some(locale) = self.locales.next() {
-                let solver = ParallelProblemSolver::new(self.res_ids.len(), self.reg.lock().len());
+                if self.reg.lock().number_of_metasources() == 0 {
+                    return None.into();
+                }
+                let number_of_metasources = self.reg.lock().number_of_metasources() - 1;
+                self.current_metasource = number_of_metasources;
+                let solver = ParallelProblemSolver::new(
+                    self.res_ids.len(),
+                    self.reg.lock().metasource_len(self.current_metasource),
+                );
                 self.state = State::Solver { locale, solver };
             } else {
                 return None.into();