nydus-image: prevent inconsistent RAFS version/compressor/digester of…

… parent bootstrap/chunk dict When specifying parent bootstrap or chunk dict, we need to make sure its rafs version, compressor and digester is consistent with current configuration. Signed-off-by: Qi Wang <[email protected]>
web-logs2 · Nov 19, 2022 · dd144ee · dd144ee
1 parent 371ea84
commit dd144ee
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 76 deletions.
diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs
@@ -7,7 +7,7 @@
 
 use std::any::Any;
 use std::collections::HashSet;
-use std::convert::TryFrom;
+use std::convert::{TryFrom, TryInto};
 use std::ffi::{OsStr, OsString};
 use std::fmt::{Debug, Display, Formatter, Result as FmtResult};
 use std::fs::OpenOptions;
@@ -324,6 +324,50 @@ impl From<compress::Algorithm> for RafsSuperFlags {
     }
 }
 
+#[derive(Clone, Copy, Debug)]
+pub struct RafsSuperConfig {
+    pub version: RafsVersion,
+    pub compressor: compress::Algorithm,
+    pub digester: digest::Algorithm,
+    pub explicit_uidgid: bool,
+}
+
+impl RafsSuperConfig {
+    pub fn check_compatibility(&self, meta: &RafsSuperMeta) -> Result<()> {
+        if self.compressor != meta.get_compressor() {
+            return Err(einval!(format!(
+                "Using inconsistent compressor {:?}, target compressor {:?}",
+                self.compressor,
+                meta.get_compressor()
+            )));
+        }
+
+        if self.digester != meta.get_digester() {
+            return Err(einval!(format!(
+                "Using inconsistent digester {:?}, target digester {:?}",
+                self.digester,
+                meta.get_digester()
+            )));
+        }
+
+        if self.explicit_uidgid != meta.explicit_uidgid() {
+            return Err(einval!(format!(
+                "Using inconsistent explicit_uidgid setting {:?}, target explicit_uidgid setting {:?}",
+                self.explicit_uidgid,
+                meta.explicit_uidgid()
+            )));
+        }
+
+        if u32::from(self.version) != meta.version {
+            return Err(einval!(format!(
+                "Using inconsistent RAFS version {:?}, target RAFS version {:?}",
+                self.version,
+                RafsVersion::try_from(meta.version)?
+            )));
+        }
+        Ok(())
+    }
+}
 /// Rafs filesystem meta-data cached from on disk RAFS super block.
 #[derive(Clone, Copy, Debug, Serialize)]
 pub struct RafsSuperMeta {
@@ -416,6 +460,15 @@ impl RafsSuperMeta {
             digest::Algorithm::Blake3
         }
     }
+
+    pub fn get_config(&self) -> RafsSuperConfig {
+        RafsSuperConfig {
+            version: self.version.try_into().unwrap_or_default(),
+            compressor: self.get_compressor(),
+            digester: self.get_digester(),
+            explicit_uidgid: self.explicit_uidgid(),
+        }
+    }
 }
 
 impl Default for RafsSuperMeta {
@@ -475,6 +528,15 @@ impl TryFrom<u32> for RafsVersion {
     }
 }
 
+impl From<RafsVersion> for u32 {
+    fn from(v: RafsVersion) -> Self {
+        match v {
+            RafsVersion::V5 => RAFS_SUPER_VERSION_V5,
+            RafsVersion::V6 => RAFS_SUPER_VERSION_V6,
+        }
+    }
+}
+
 impl RafsVersion {
     /// Check whether it's RAFS v5.
     pub fn is_v5(&self) -> bool {

diff --git a/src/bin/nydus-image/core/blob_compact.rs b/src/bin/nydus-image/core/blob_compact.rs
@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 use sha2::Digest;
 
 use nydus_rafs::metadata::chunk::ChunkWrapper;
-use nydus_rafs::metadata::{RafsMode, RafsSuper, RafsVersion};
+use nydus_rafs::metadata::{RafsSuper, RafsVersion};
 use nydus_storage::backend::BlobBackend;
 use nydus_storage::utils::alloc_buf;
 use nydus_utils::digest::RafsDigest;
@@ -552,14 +552,12 @@ impl BlobCompactor {
     }
 
     pub fn do_compact(
-        s_bootstrap: PathBuf,
+        rs: RafsSuper,
         d_bootstrap: PathBuf,
         chunk_dict: Option<Arc<dyn ChunkDict>>,
         backend: Arc<dyn BlobBackend + Send + Sync>,
         cfg: &Config,
     ) -> Result<Option<BuildOutput>> {
-        let rs = RafsSuper::load_from_metadata(&s_bootstrap, RafsMode::Direct, true)?;
-        info!("load bootstrap {:?} successfully", s_bootstrap);
         let mut build_ctx = BuildContext::new(
             "".to_string(),
             false,
@@ -599,7 +597,7 @@ impl BlobCompactor {
         compactor.compact(cfg)?;
         compactor.dump_new_blobs(&build_ctx, &cfg.blobs_dir, build_ctx.aligned_chunk)?;
         if compactor.new_blob_mgr.len() == 0 {
-            info!("blobs of {:?} have already been optimized", s_bootstrap);
+            info!("blobs of source bootstrap have already been optimized");
             return Ok(None);
         }
         info!("compact blob successfully");

diff --git a/src/bin/nydus-image/core/bootstrap.rs b/src/bin/nydus-image/core/bootstrap.rs
@@ -18,7 +18,7 @@ use nydus_rafs::metadata::layout::v6::{
     RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE,
 };
 use nydus_rafs::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE};
-use nydus_rafs::metadata::{RafsMode, RafsStore, RafsSuper};
+use nydus_rafs::metadata::{RafsMode, RafsStore, RafsSuper, RafsSuperConfig};
 use nydus_utils::digest::{DigestHasher, RafsDigest};
 
 use super::context::{
@@ -323,13 +323,13 @@ impl Bootstrap {
             return Err(Error::msg("bootstrap context's parent bootstrap is null"));
         };
 
-        let lower_compressor = rs.meta.get_compressor();
-        if ctx.compressor != lower_compressor {
-            return Err(Error::msg(format!(
-                "inconsistent compressor with the lower layer, current {}, lower: {}.",
-                ctx.compressor, lower_compressor
-            )));
-        }
+        let config = RafsSuperConfig {
+            compressor: ctx.compressor,
+            digester: ctx.digester,
+            explicit_uidgid: ctx.explicit_uidgid,
+            version: ctx.fs_version,
+        };
+        config.check_compatibility(&rs.meta)?;
 
         // Reuse lower layer blob table,
         // we need to append the blob entry of upper layer to the table

diff --git a/src/bin/nydus-image/core/chunk_dict.rs b/src/bin/nydus-image/core/chunk_dict.rs
@@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex};
 use anyhow::{Context, Result};
 use nydus_rafs::metadata::chunk::ChunkWrapper;
 use nydus_rafs::metadata::layout::v5::RafsV5ChunkInfo;
-use nydus_rafs::metadata::RafsSuper;
+use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig};
 use nydus_storage::device::BlobInfo;
 use nydus_utils::digest::RafsDigest;
 
@@ -91,7 +91,10 @@ impl ChunkDict for HashChunkDict {
 }
 
 impl HashChunkDict {
-    fn from_bootstrap_file(path: &Path) -> Result<Self> {
+    fn from_bootstrap_file(
+        path: &Path,
+        target_rafs_config: Option<RafsSuperConfig>,
+    ) -> Result<Self> {
         let rs = RafsSuper::load_chunk_dict_from_metadata(path)
             .with_context(|| format!("failed to open bootstrap file {:?}", path))?;
         let mut d = HashChunkDict {
@@ -100,6 +103,9 @@ impl HashChunkDict {
             blob_idx_m: Mutex::new(BTreeMap::new()),
         };
 
+        let config = target_rafs_config.unwrap_or_else(|| rs.meta.get_config());
+        config.check_compatibility(&rs.meta)?;
+
         if rs.meta.is_v5() {
             Tree::from_bootstrap(&rs, &mut d).context("failed to build tree from bootstrap")?;
         } else if rs.meta.is_v6() {
@@ -165,9 +171,13 @@ pub fn parse_chunk_dict_arg(arg: &str) -> Result<PathBuf> {
 }
 
 /// Load a chunk dictionary from external source.
-pub(crate) fn import_chunk_dict(arg: &str) -> Result<Arc<dyn ChunkDict>> {
+pub(crate) fn import_chunk_dict(
+    arg: &str,
+    config: Option<RafsSuperConfig>,
+) -> Result<Arc<dyn ChunkDict>> {
     let file_path = parse_chunk_dict_arg(arg)?;
-    HashChunkDict::from_bootstrap_file(&file_path).map(|d| Arc::new(d) as Arc<dyn ChunkDict>)
+    HashChunkDict::from_bootstrap_file(&file_path, config)
+        .map(|d| Arc::new(d) as Arc<dyn ChunkDict>)
 }
 
 #[cfg(test)]
@@ -193,7 +203,7 @@ mod tests {
         let mut source_path = PathBuf::from(root_dir);
         source_path.push("tests/texture/bootstrap/rafs-v5.boot");
         let path = source_path.to_str().unwrap();
-        let dict = import_chunk_dict(path).unwrap();
+        let dict = import_chunk_dict(path, None).unwrap();
 
         assert!(dict.get_chunk(&RafsDigest::default()).is_none());
         assert_eq!(dict.get_blobs().len(), 18);

diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs
@@ -23,7 +23,7 @@ use clap::{Arg, ArgAction, ArgMatches, Command as App};
 use nix::unistd::{getegid, geteuid};
 use nydus_api::http::BackendConfig;
 use nydus_app::{setup_logging, BuildTimeInfo};
-use nydus_rafs::metadata::RafsVersion;
+use nydus_rafs::metadata::{RafsMode, RafsSuper, RafsSuperConfig, RafsVersion};
 use nydus_rafs::RafsIoReader;
 use nydus_storage::factory::BlobFactory;
 use nydus_storage::meta::{
@@ -677,8 +677,14 @@ impl Command {
 
         let mut blob_mgr = BlobManager::new();
         if let Some(chunk_dict_arg) = matches.get_one::<String>("chunk-dict") {
+            let config = RafsSuperConfig {
+                version,
+                compressor,
+                digester,
+                explicit_uidgid: !repeatable,
+            };
             blob_mgr.set_chunk_dict(timing_tracer!(
-                { import_chunk_dict(chunk_dict_arg) },
+                { import_chunk_dict(chunk_dict_arg, Some(config)) },
                 "import_chunk_dict"
             )?);
         }
@@ -778,9 +784,11 @@ impl Command {
             Some(s) => PathBuf::from(s),
         };
 
+        let rs = RafsSuper::load_from_metadata(&bootstrap_path, RafsMode::Direct, true)?;
+        info!("load bootstrap {:?} successfully", bootstrap_path);
         let chunk_dict = match matches.get_one::<String>("chunk-dict") {
             None => None,
-            Some(args) => Some(import_chunk_dict(args)?),
+            Some(args) => Some(import_chunk_dict(args, Some(rs.meta.get_config()))?),
         };
 
         let backend_type = matches
@@ -801,7 +809,7 @@ impl Command {
             .with_context(|| format!("invalid config file {}", config_file_path))?;
 
         if let Some(build_output) =
-            BlobCompactor::do_compact(bootstrap_path, dst_bootstrap, chunk_dict, backend, &config)?
+            BlobCompactor::do_compact(rs, dst_bootstrap, chunk_dict, backend, &config)?
         {
             OutputSerializer::dump(matches, build_output, build_info)?;
         }

diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs
@@ -8,9 +8,7 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 
 use anyhow::{Context, Result};
-use nydus_rafs::metadata::{RafsInodeExt, RafsMode, RafsSuper, RafsSuperMeta, RafsVersion};
-use nydus_utils::compress;
-use nydus_utils::digest;
+use nydus_rafs::metadata::{RafsInodeExt, RafsMode, RafsSuper, RafsVersion};
 
 use crate::core::bootstrap::Bootstrap;
 use crate::core::chunk_dict::HashChunkDict;
@@ -20,29 +18,6 @@ use crate::core::context::{
 use crate::core::node::{ChunkSource, Overlay, WhiteoutSpec};
 use crate::core::tree::{MetadataTreeBuilder, Tree};
 
-#[derive(Clone, Debug, Eq, PartialEq)]
-struct Flags {
-    compressor: compress::Algorithm,
-    digester: digest::Algorithm,
-    explicit_uidgid: bool,
-}
-
-impl Flags {
-    fn from_meta(meta: &RafsSuperMeta) -> Self {
-        Self {
-            compressor: meta.get_compressor(),
-            digester: meta.get_digester(),
-            explicit_uidgid: meta.explicit_uidgid(),
-        }
-    }
-
-    fn set_to_ctx(&self, ctx: &mut BuildContext) {
-        ctx.compressor = self.compressor;
-        ctx.digester = self.digester;
-        ctx.explicit_uidgid = self.explicit_uidgid;
-    }
-}
-
 /// Struct to generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps.
 ///
 /// A container image contains one or more layers, a RAFS bootstrap is built for each layer.
@@ -79,50 +54,32 @@ impl Merger {
 
         // Get the blobs come from chunk dict bootstrap.
         let mut chunk_dict_blobs = HashSet::new();
+        let mut config = None;
         if let Some(chunk_dict_path) = &chunk_dict {
             let rs = RafsSuper::load_from_metadata(chunk_dict_path, RafsMode::Direct, true)
                 .context(format!("load chunk dict bootstrap {:?}", chunk_dict_path))?;
+            config = Some(rs.meta.get_config());
             for blob in rs.superblock.get_blob_infos() {
                 chunk_dict_blobs.insert(blob.blob_id().to_string());
             }
         }
 
         let mut fs_version = None;
-        let mut flags: Option<Flags> = None;
         let mut chunk_size = None;
         let mut tree: Option<Tree> = None;
         let mut blob_mgr = BlobManager::new();
         for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
             let rs = RafsSuper::load_from_metadata(bootstrap_path, RafsMode::Direct, true)
                 .context(format!("load bootstrap {:?}", bootstrap_path))?;
 
-            match fs_version {
-                None => fs_version = Some(rs.meta.version),
-                Some(version) => {
-                    if version != rs.meta.version {
-                        bail!(
-                            "can not merge bootstraps with inconsistent RAFS version {} and {}",
-                            version,
-                            rs.meta.version
-                        );
-                    }
-                }
-            }
-
-            let current_flags = Flags::from_meta(&rs.meta);
-            if let Some(flags) = &flags {
-                if flags != &current_flags {
-                    bail!(
-                        "can not merge bootstraps with inconsistent RAFS flags, current bootstrap {:?}, flags {:?}, expected flags {:?}",
-                        bootstrap_path,
-                        current_flags,
-                        flags,
-                    );
-                }
+            if let Some(config) = &config {
+                config.check_compatibility(&rs.meta)?;
             } else {
-                // Keep final bootstrap following superblock flags of source bootstraps.
-                current_flags.set_to_ctx(ctx);
-                flags = Some(current_flags);
+                config = Some(rs.meta.get_config());
+                fs_version = Some(rs.meta.version);
+                ctx.compressor = rs.meta.get_compressor();
+                ctx.digester = rs.meta.get_digester();
+                ctx.explicit_uidgid = rs.meta.explicit_uidgid();
             }
 
             let blob_hash = Self::get_blob_hash(bootstrap_path)?;