Avoid dedup chunks that are relatively new

To prevent deduplicating chunks from a previous failed upload attempt, only consider deduping chunks that are older than a specific threshold. related to https://bugzilla.redhat.com/show_bug.cgi?id=2256223 Signed-off-by: Danny Zaken <[email protected]>
noobaa · Oct 28, 2024 · 948e450 · 948e450
1 parent bfaa477
commit 948e450
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 8 deletions.
diff --git a/config.js b/config.js
@@ -392,6 +392,9 @@ config.CHUNK_CODER_EC_PARITY_TYPE = 'cm256';
 config.CHUNK_CODER_EC_TOLERANCE_THRESHOLD = 2;
 config.CHUNK_CODER_EC_IS_DEFAULT = false;
 
+// DEDUP
+config.MIN_CHUNK_AGE_FOR_DEDUP = 60 * 60 * 1000; // 1 hour
+
 //////////////////////////
 // DEDUP INDEXER CONFIG //
 //////////////////////////
@@ -870,7 +873,7 @@ config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
 config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;
 config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1;
 config.ALLOW_HTTP = false;
-// config files should allow access to the owner of the files 
+// config files should allow access to the owner of the files
 config.BASE_MODE_CONFIG_FILE = 0o600;
 config.BASE_MODE_CONFIG_DIR = 0o700;
 
@@ -1054,10 +1057,10 @@ function _get_config_root() {
 }
 
 /**
- * validate_nc_master_keys_config validates the following - 
+ * validate_nc_master_keys_config validates the following -
  * 1. if type is file -
  *    1.1. no GET/PUT executables provided
- * 2. if type is executable - 
+ * 2. if type is executable -
  *    2.1. no file location provided
  *    2.2. GET & PUT executables exist and executables
  */

diff --git a/src/server/object_services/mapper.js b/src/server/object_services/mapper.js
@@ -186,7 +186,7 @@ function map_chunk(chunk, tier, tiering, tiering_status, location_info) {
         }
 
         /**
-         * @param {nb.TierMirror} mirror 
+         * @param {nb.TierMirror} mirror
          */
         function map_frag_in_mirror(mirror) {
             const used_blocks = [];
@@ -291,6 +291,13 @@ function is_chunk_good_for_dedup(chunk) {
     if (!chunk.is_accessible) return false;
     if (chunk.is_building_blocks) return false;
     if (chunk.is_building_frags) return false;
+
+    // reject chunks that are not at least config.MIN_CHUNK_AGE_FOR_DEDUP old
+    // this is to avoid deduping chunks that might be from a previous attempt to write the same object
+    // see https://bugzilla.redhat.com/show_bug.cgi?id=2256223
+    const chunk_age = Date.now() - chunk._id.getTimestamp().getTime();
+    if (chunk_age < config.MIN_CHUNK_AGE_FOR_DEDUP) return false;
+
     return true;
 }
 
@@ -360,15 +367,15 @@ function _block_sort_newer_first(block1, block2) {
 
 
 /**
- * @param {nb.Pool} pool 
+ * @param {nb.Pool} pool
  * @returns {boolean}
  */
 function _pool_has_redundancy(pool) {
     return Boolean(pool.cloud_pool_info || pool.mongo_pool_info);
 }
 
 // /**
-//  * 
+//  *
 //  * @param {nb.Chunk} chunk
 //  * @param {nb.LocationInfo} [location_info]
 //  */

diff --git a/src/test/unit_tests/test_encryption.js b/src/test/unit_tests/test_encryption.js
@@ -29,6 +29,8 @@ let wrapped_coretest_secret_key;
 const BKT = `bucket.example`;
 const key_rotator = new KeyRotator({ name: 'kr'});
 
+config.MIN_CHUNK_AGE_FOR_DEDUP = 0;
+
 mocha.describe('Encryption tests', function() {
     const { rpc_client, EMAIL, SYSTEM } = coretest;
     let response_account;
@@ -991,10 +993,10 @@ mocha.describe('Rotation tests', function() {
             compare_secrets(secrets, system_store_account.master_key_id._id);
         });
 });
-// TODO: 
+// TODO:
         // 1. add more tests for checking namespace resources
         // 2. add tests for enable/disable account that has pool/namespace resource
-////////////// HELPERS 
+////////////// HELPERS
 
 async function multipart_upload(bucket, key, s3_conf) {
     let res = await s3_conf.createMultipartUpload({Bucket: bucket, Key: key, ContentType: 'text/plain'});