fix broken native and zran image support

1. We should ignore to handle TOC when only bootstrap be provided; 2. Get zran rafs blob digest/size from CLI options for merge subcommand; Signed-off-by: Yan Song <[email protected]>
jiangliu · Dec 28, 2022 · 955bf74 · 955bf74
1 parent c4090c9
commit 955bf74
Show file tree

Hide file tree

Showing 9 changed files with 63 additions and 41 deletions.
diff --git a/api/src/config.rs b/api/src/config.rs
@@ -145,9 +145,12 @@ impl ConfigV2 {
 
     /// Get configuration information for RAFS filesystem.
     pub fn get_rafs_config(&self) -> Result<&RafsConfigV2> {
-        self.rafs
-            .as_ref()
-            .ok_or_else(|| einval!("no configuration information for rafs"))
+        self.rafs.as_ref().ok_or_else(|| {
+            Error::new(
+                ErrorKind::InvalidInput,
+                "no configuration information for rafs",
+            )
+        })
     }
 
     /// Clone the object with all secrets removed.
@@ -644,6 +647,9 @@ pub struct RafsConfigV2 {
     /// Filesystem prefetching configuration.
     #[serde(default)]
     pub prefetch: PrefetchConfigV2,
+    /// Probe TOC list included in the blob.
+    #[serde(default)]
+    pub probe_toc: bool,
 }
 
 impl RafsConfigV2 {
@@ -969,6 +975,9 @@ struct RafsConfig {
     // ZERO value means, amplifying user io is not enabled.
     #[serde(default = "default_batch_size")]
     pub amplify_io: usize,
+    /// Probe TOC list included in the blob.
+    #[serde(default)]
+    pub probe_toc: bool,
 }
 
 impl TryFrom<RafsConfig> for ConfigV2 {
@@ -986,6 +995,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             access_pattern: v.access_pattern,
             latest_read_files: v.latest_read_files,
             prefetch: v.fs_prefetch.into(),
+            probe_toc: v.probe_toc,
         };
         if !cache.prefetch.enable && rafs.prefetch.enable {
             cache.prefetch = rafs.prefetch.clone();

diff --git a/rafs/src/metadata/direct_v6.rs b/rafs/src/metadata/direct_v6.rs
@@ -1152,7 +1152,7 @@ impl RafsInodeExt for OndiskInodeWrapper {
             + (idx as usize * size_of::<RafsV6InodeChunkAddr>());
         let chunk_addr = state.map.get_ref::<RafsV6InodeChunkAddr>(offset)?;
 
-        if state.meta.has_inlined_chunk_digest() {
+        if state.meta.probe_toc && state.meta.has_inlined_chunk_digest() {
             let blob_index = chunk_addr.blob_index();
             let chunk_index = chunk_addr.blob_ci_index();
             let device = self.mapping.device.lock().unwrap();

diff --git a/rafs/src/metadata/md_v5.rs b/rafs/src/metadata/md_v5.rs
@@ -40,6 +40,8 @@ impl RafsSuper {
         self.meta.prefetch_table_entries = sb.prefetch_table_entries();
         self.meta.prefetch_table_offset = sb.prefetch_table_offset();
 
+        self.meta.probe_toc = self.probe_toc;
+
         match self.mode {
             RafsMode::Direct => {
                 let mut inodes = DirectSuperBlockV5::new(&self.meta, self.validate_digest);

diff --git a/rafs/src/metadata/md_v6.rs b/rafs/src/metadata/md_v6.rs
@@ -55,6 +55,7 @@ impl RafsSuper {
             self.meta.prefetch_table_offset,
             self.meta.prefetch_table_entries
         );
+        self.meta.probe_toc = self.probe_toc;
 
         match self.mode {
             RafsMode::Direct => {

diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs
@@ -431,6 +431,9 @@ pub struct RafsSuperMeta {
     pub chunk_table_offset: u64,
     /// Size  of the chunk table for RAFS v6.
     pub chunk_table_size: u64,
+
+    /// Probe TOC list included in the blob.
+    pub probe_toc: bool,
 }
 
 impl RafsSuperMeta {
@@ -519,6 +522,7 @@ impl Default for RafsSuperMeta {
             is_chunk_dict: false,
             chunk_table_offset: 0,
             chunk_table_size: 0,
+            probe_toc: false,
         }
     }
 }
@@ -618,6 +622,8 @@ pub struct RafsSuper {
     pub meta: RafsSuperMeta,
     /// Rafs filesystem super block.
     pub superblock: Arc<dyn RafsSuperBlock>,
+    /// Probe TOC list included in the blob.
+    pub probe_toc: bool,
 }
 
 impl Default for RafsSuper {
@@ -627,6 +633,7 @@ impl Default for RafsSuper {
             validate_digest: false,
             meta: RafsSuperMeta::default(),
             superblock: Arc::new(NoopSuperBlock::new()),
+            probe_toc: false,
         }
     }
 }
@@ -637,6 +644,7 @@ impl RafsSuper {
         Ok(Self {
             mode: RafsMode::from_str(conf.mode.as_str())?,
             validate_digest: conf.validate,
+            probe_toc: conf.probe_toc,
             ..Default::default()
         })
     }
@@ -669,23 +677,33 @@ impl RafsSuper {
             .open(path.as_ref())?;
         let mut reader = Box::new(file) as RafsIoReader;
 
-        if let Err(e) = rs.load(&mut reader) {
-            let id = BlobInfo::get_blob_id_from_meta_path(path.as_ref())?;
-            let new_path = match TocEntryList::extract_rafs_meta(&id, config.clone()) {
-                Ok(v) => v,
-                Err(_e) => {
-                    debug!("failed to load inlined RAFS meta, {}", _e);
-                    return Err(e);
-                }
-            };
-            let file = OpenOptions::new().read(true).write(false).open(new_path)?;
-            reader = Box::new(file) as RafsIoReader;
+        let mut probe_toc = false;
+        if let Ok(rafs_config) = config.get_rafs_config() {
+            probe_toc = rafs_config.probe_toc;
+        }
+
+        if probe_toc {
+            if let Err(e) = rs.load(&mut reader) {
+                let id = BlobInfo::get_blob_id_from_meta_path(path.as_ref())?;
+                let new_path = match TocEntryList::extract_rafs_meta(&id, config.clone()) {
+                    Ok(v) => v,
+                    Err(_e) => {
+                        debug!("failed to load inlined RAFS meta, {}", _e);
+                        return Err(e);
+                    }
+                };
+                let file = OpenOptions::new().read(true).write(false).open(new_path)?;
+                reader = Box::new(file) as RafsIoReader;
+                rs.load(&mut reader)?;
+                rs.set_blob_id_from_meta_path(path.as_ref())?;
+            }
+        } else {
             rs.load(&mut reader)?;
-            rs.set_blob_id_from_meta_path(path.as_ref())?;
         }
 
         if (validate_digest || config.is_chunk_validation_enabled())
             && rs.meta.has_inlined_chunk_digest()
+            && probe_toc
         {
             rs.create_blob_device(config)?;
         }

diff --git a/src/bin/nydus-image/builder/mod.rs b/src/bin/nydus-image/builder/mod.rs
@@ -84,11 +84,7 @@ fn dump_bootstrap(
             }
         }
         if !ctx.conversion_type.is_to_ref() {
-            if ctx.blob_inline_meta {
-                blob_ctx.compressed_blob_size = 0;
-            } else {
-                blob_ctx.compressed_blob_size = blob_writer.pos()?;
-            }
+            blob_ctx.compressed_blob_size = blob_writer.pos()?;
         }
     }
 
@@ -174,11 +170,7 @@ fn finalize_blob(
         dump_toc(ctx, blob_ctx, blob_writer)?;
 
         if !ctx.conversion_type.is_to_ref() {
-            if ctx.blob_inline_meta {
-                blob_ctx.compressed_blob_size = 0;
-            } else {
-                blob_ctx.compressed_blob_size = blob_writer.pos()?;
-            }
+            blob_ctx.compressed_blob_size = blob_writer.pos()?;
         }
         if ctx.blob_inline_meta && blob_ctx.blob_id == "x".repeat(64) {
             blob_ctx.blob_id = String::new();

diff --git a/src/bin/nydus-image/core/context.rs b/src/bin/nydus-image/core/context.rs
@@ -472,8 +472,14 @@ impl BlobContext {
         let mut blob_id = blob.raw_blob_id().to_string();
         let mut features = blob.features();
 
+        let mut probe_toc = false;
+        if let Ok(rafs_config) = ctx.configuration.get_rafs_config() {
+            probe_toc = rafs_config.probe_toc;
+        }
+
         // Fixes up blob info objects from inlined-meta blobs.
-        if features.contains(BlobFeatures::INLINED_META)
+        if probe_toc
+            && features.contains(BlobFeatures::INLINED_META)
             && (chunk_source == ChunkSource::Dict || chunk_source == ChunkSource::Parent)
         {
             let backend_config = ctx

diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs
@@ -31,7 +31,6 @@ use crate::core::tree::{MetadataTreeBuilder, Tree};
 pub struct Merger {}
 
 impl Merger {
-    /*
     /// Merge assumes the bootstrap name is in `$blob_digest` format.
     fn get_raw_blob_digest(bootstrap_path: &Path) -> Result<String> {
         let file_name = bootstrap_path
@@ -41,7 +40,6 @@ impl Merger {
             .ok_or_else(|| anyhow!("convert to string"))?;
         Ok(file_name.to_string())
     }
-     */
 
     fn get_digest_from_list(digests: &Option<Vec<String>>, idx: usize) -> Result<Option<[u8; 32]>> {
         Ok(if let Some(digests) = &digests {
@@ -163,7 +161,7 @@ impl Merger {
 
                     // The blob id (blob sha256 hash) in parent bootstrap is invalid for nydusd
                     // runtime, should change it to the hash of whole tar blob.
-                    blob_ctx.blob_id = blob.blob_id().to_owned();
+                    blob_ctx.blob_id = Self::get_raw_blob_digest(bootstrap_path)?;
                     if let Some(digest) = Self::get_digest_from_list(&blob_digests, layer_idx)? {
                         if blob.has_feature(BlobFeatures::ZRAN) {
                             blob_ctx.rafs_blob_digest = digest;

diff --git a/storage/src/device.rs b/storage/src/device.rs
@@ -196,9 +196,7 @@ impl BlobInfo {
 
     /// Get the id of the blob, with special handling of `inlined-meta` case.
     pub fn blob_id(&self) -> String {
-        if self.has_feature(BlobFeatures::INLINED_META) && !self.has_feature(BlobFeatures::ZRAN)
-            || !self.meta_ci_is_valid()
-        {
+        if self.has_feature(BlobFeatures::INLINED_META) && !self.has_feature(BlobFeatures::ZRAN) {
             let guard = self.meta_path.lock().unwrap();
             if !guard.is_empty() {
                 return guard.deref().clone();
@@ -451,16 +449,13 @@ impl BlobInfo {
     /// Get RAFS blob id for ZRan.
     pub fn get_rafs_blob_id(&self) -> Result<String, Error> {
         assert!(self.has_feature(BlobFeatures::ZRAN));
-        let id = if self.has_feature(BlobFeatures::INLINED_META) {
+        if self.has_feature(BlobFeatures::INLINED_META) {
             let guard = self.meta_path.lock().unwrap();
-            if guard.is_empty() {
-                return Err(einval!("failed to get blob id from meta file name"));
+            if !guard.is_empty() {
+                return Ok(guard.deref().clone());
             }
-            guard.deref().clone()
-        } else {
-            hex::encode(&self.rafs_blob_digest)
-        };
-        Ok(id)
+        }
+        Ok(hex::encode(&self.rafs_blob_digest))
     }
 }