stroage: fix a bug in handling last chunk of blob

Fix a bug in handling last chunk of blob.

Signed-off-by: Jiang Liu <[email protected]>

rafs/src/fs.rs

@@ -438,7 +438,7 @@ impl Rafs {
                 let batch_size = 1024 * 1024 * 2;
 
                 for blob in &blob_infos {
-                    let blob_size = blob.compressed_size();
+                    let blob_size = blob.compressed_data_size();
                     let count = div_round_up(blob_size, batch_size);
 
                     let mut pre_offset = 0u64;

rafs/src/metadata/mod.rs

@@ -693,7 +693,7 @@ impl RafsSuper {
             let blobs = rs.superblock.get_blob_infos();
             for blob in blobs.iter() {
                 // Fix blob id for new images with old converters.
-                if blob.has_feature(BlobFeatures::INLINED_META) {
+                if blob.has_feature(BlobFeatures::INLINED_FS_META) {
                     blob.set_blob_id_from_meta_path(path.as_ref())?;
                     fixed = true;
                 }
@@ -702,7 +702,7 @@ impl RafsSuper {
                 // Fix blob id for old images with old converters.
                 let last = blobs.len() - 1;
                 let blob = &blobs[last];
-                if !blob.has_feature(BlobFeatures::CAP_INLINED_META) {
+                if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) {
                     rs.set_blob_id_from_meta_path(path.as_ref())?;
                 }
             }
@@ -738,8 +738,8 @@ impl RafsSuper {
     pub fn set_blob_id_from_meta_path(&self, meta_path: &Path) -> Result<()> {
         let blobs = self.superblock.get_blob_infos();
         for blob in blobs.iter() {
-            if blob.has_feature(BlobFeatures::INLINED_META)
-                || !blob.has_feature(BlobFeatures::CAP_INLINED_META)
+            if blob.has_feature(BlobFeatures::INLINED_FS_META)
+                || !blob.has_feature(BlobFeatures::CAP_TAR_TOC)
             {
                 blob.set_blob_id_from_meta_path(meta_path)?;
             }

src/bin/nydus-image/core/context.rs

@@ -449,7 +449,7 @@ impl BlobContext {
             .set_4k_aligned(features.contains(BlobFeatures::ALIGNED));
         blob_ctx
             .blob_meta_header
-            .set_inlined_meta(features.contains(BlobFeatures::INLINED_META));
+            .set_inlined_fs_meta(features.contains(BlobFeatures::INLINED_FS_META));
         blob_ctx
             .blob_meta_header
             .set_chunk_info_v2(features.contains(BlobFeatures::CHUNK_INFO_V2));
@@ -461,7 +461,7 @@ impl BlobContext {
             .set_inlined_chunk_digest(features.contains(BlobFeatures::INLINED_CHUNK_DIGEST));
         blob_ctx
             .blob_meta_header
-            .set_cap_inlined_meta(features.contains(BlobFeatures::CAP_INLINED_META));
+            .set_cap_tar_toc(features.contains(BlobFeatures::CAP_TAR_TOC));
 
         blob_ctx
     }
@@ -477,8 +477,8 @@ impl BlobContext {
 
         // Fixes up blob info objects from inlined-meta blobs.
         if chunk_source == ChunkSource::Dict || chunk_source == ChunkSource::Parent {
-            if features.contains(BlobFeatures::INLINED_META) {
-                features &= !BlobFeatures::INLINED_META;
+            if features.contains(BlobFeatures::INLINED_FS_META) {
+                features &= !BlobFeatures::INLINED_FS_META;
 
                 if !features.contains(BlobFeatures::ZRAN) {
                     blob_id = blob.blob_id();
@@ -538,9 +538,7 @@ impl BlobContext {
                         }
                     }
                 }
-            } else if !blob.has_feature(BlobFeatures::CAP_INLINED_META)
-                && !ctx.can_access_data_blobs
-            {
+            } else if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) && !ctx.can_access_data_blobs {
                 blob_id = blob.blob_id();
             }
         }
@@ -583,7 +581,7 @@ impl BlobContext {
 
     // TODO: check the logic to reset prefetch size
     pub fn set_blob_prefetch_size(&mut self, ctx: &BuildContext) {
-        if (self.compressed_blob_size > 0
+        if (self.uncompressed_blob_size > 0
             || (ctx.conversion_type == ConversionType::EStargzIndexToRef
                 && !self.blob_id.is_empty()))
             && ctx.prefetch.policy != PrefetchPolicy::Blob
@@ -652,7 +650,7 @@ impl BlobContext {
 
     /// Get blob id if the blob has some chunks.
     pub fn blob_id(&mut self) -> Option<String> {
-        if self.compressed_blob_size > 0 {
+        if self.uncompressed_blob_size > 0 {
             Some(self.blob_id.to_string())
         } else {
             None
@@ -1070,9 +1068,9 @@ impl BuildContext {
         features: Features,
     ) -> Self {
         let blob_features = if blob_inline_meta {
-            BlobFeatures::INLINED_META | BlobFeatures::CAP_INLINED_META
+            BlobFeatures::INLINED_FS_META | BlobFeatures::CAP_TAR_TOC
         } else {
-            BlobFeatures::CAP_INLINED_META
+            BlobFeatures::CAP_TAR_TOC
         };
         BuildContext {
             blob_id,

src/bin/nydus-image/main.rs

@@ -995,9 +995,10 @@ impl Command {
         let mut blob_ids = Vec::new();
         for (idx, blob) in blobs.iter().enumerate() {
             println!(
-                "\t {}: {}, compressed size 0x{:x}, uncompressed size 0x{:x}, chunks: 0x{:x}, features: {}",
+                "\t {}: {}, compressed data size 0x{:x}, compressed file size 0x{:x}, uncompressed file size 0x{:x}, chunks: 0x{:x}, features: {}",
                 idx,
                 blob.blob_id(),
+                blob.compressed_data_size(),
                 blob.compressed_size(),
                 blob.uncompressed_size(),
                 blob.chunk_count(),

src/bin/nydusd/fs_cache.rs

@@ -510,7 +510,7 @@ impl FsCacheHandler {
             Some(1) => nydus_api::default_batch_size() as u64,
             Some(s) => s as u64,
         };
-        let blob_size = blob_info.compressed_size();
+        let blob_size = blob_info.compressed_data_size();
         let count = (blob_size + size - 1) / size;
         let mut blob_req = Vec::with_capacity(count as usize);
         let mut pre_offset = 0u64;

storage/src/cache/filecache/mod.rs

@@ -221,7 +221,7 @@ impl FileCacheEntry {
         // Set cache file to its expected size.
         let file_size = file.metadata()?.len();
         let cached_file_size = if mgr.cache_raw_data {
-            blob_info.compressed_size()
+            blob_info.compressed_data_size()
         } else {
             blob_info.uncompressed_size()
         };

storage/src/device.rs

@@ -52,15 +52,16 @@ bitflags! {
         /// Uncompressed chunk data is 4K aligned.
         const ALIGNED = 0x0000_0001;
         /// RAFS meta data is inlined in the data blob.
-        const INLINED_META = 0x0000_0002;
+        const INLINED_FS_META = 0x0000_0002;
         /// Blob chunk information format v2.
         const CHUNK_INFO_V2 = 0x0000_0004;
         /// Blob compression information data include context data for zlib random access.
         const ZRAN = 0x0000_0008;
         /// Chunk digest array is inlined in the data blob.
         const INLINED_CHUNK_DIGEST = 0x0000_0010;
-        /// Inlined-meta capability, used to support backward compatibility for legacy converters.
-        const CAP_INLINED_META = 0x4000_0000;
+        /// Data blob are encoded with Tar header and optionally ToC.
+        /// It's also a flag indicating that images are generated with `nydus-image` v2.2 or newer.
+        const CAP_TAR_TOC = 0x4000_0000;
         /// Rafs V5 image without extended blob table, this is an internal flag.
         const _V5_NO_EXT_BLOB_TABLE = 0x8000_0000;
     }
@@ -198,8 +199,9 @@ impl BlobInfo {
 
     /// Get the id of the blob, with special handling of `inlined-meta` case.
     pub fn blob_id(&self) -> String {
-        if (self.has_feature(BlobFeatures::INLINED_META) && !self.has_feature(BlobFeatures::ZRAN))
-            || !self.has_feature(BlobFeatures::CAP_INLINED_META)
+        if (self.has_feature(BlobFeatures::INLINED_FS_META)
+            && !self.has_feature(BlobFeatures::ZRAN))
+            || !self.has_feature(BlobFeatures::CAP_TAR_TOC)
         {
             let guard = self.meta_path.lock().unwrap();
             if !guard.is_empty() {
@@ -214,7 +216,20 @@ impl BlobInfo {
         &self.blob_id
     }
 
-    /// Get size of the compressed blob.
+    /// Get size of compressed chunk data, not including `blob.meta`, `blob.chunk`, `toc` etc.
+    pub fn compressed_data_size(&self) -> u64 {
+        if self.has_feature(BlobFeatures::ZRAN) {
+            // It's the size of OCIv1 targz blob.
+            self.compressed_size
+        } else if self.has_feature(BlobFeatures::CAP_TAR_TOC) && self.meta_ci_is_valid() {
+            // There's a tar header between chunk data and compression information.
+            self.meta_ci_offset - 0x200
+        } else {
+            self.compressed_size
+        }
+    }
+
+    /// Get size of the compressed blob, including `blob.meta`, `blob.chunk`, `toc` etc.
     pub fn compressed_size(&self) -> u64 {
         self.compressed_size
     }
@@ -453,7 +468,7 @@ impl BlobInfo {
     /// Get RAFS blob id for ZRan.
     pub fn get_rafs_blob_id(&self) -> Result<String, Error> {
         assert!(self.has_feature(BlobFeatures::ZRAN));
-        let id = if self.has_feature(BlobFeatures::INLINED_META) {
+        let id = if self.has_feature(BlobFeatures::INLINED_FS_META) {
             let guard = self.meta_path.lock().unwrap();
             if guard.is_empty() {
                 return Err(einval!("failed to get blob id from meta file name"));