diff --git a/config.js b/config.js index 513309dff2..8dcb737d71 100644 --- a/config.js +++ b/config.js @@ -408,6 +408,9 @@ config.CHUNK_CODER_EC_PARITY_TYPE = 'cm256'; config.CHUNK_CODER_EC_TOLERANCE_THRESHOLD = 2; config.CHUNK_CODER_EC_IS_DEFAULT = false; +// DEDUP +config.MIN_CHUNK_AGE_FOR_DEDUP = 60 * 60 * 1000; // 1 hour + ////////////////////////// // DEDUP INDEXER CONFIG // ////////////////////////// @@ -886,7 +889,7 @@ config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443; config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1; config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1; config.ALLOW_HTTP = false; -// config files should allow access to the owner of the files +// config files should allow access to the owner of the files config.BASE_MODE_CONFIG_FILE = 0o600; config.BASE_MODE_CONFIG_DIR = 0o700; @@ -1069,10 +1072,10 @@ function _get_config_root() { } /** - * validate_nc_master_keys_config validates the following - + * validate_nc_master_keys_config validates the following - * 1. if type is file - * 1.1. no GET/PUT executables provided - * 2. if type is executable - + * 2. if type is executable - * 2.1. no file location provided * 2.2. GET & PUT executables exist and executables */ diff --git a/src/server/object_services/mapper.js b/src/server/object_services/mapper.js index e434c9253d..d5f0fddef6 100644 --- a/src/server/object_services/mapper.js +++ b/src/server/object_services/mapper.js @@ -186,7 +186,7 @@ function map_chunk(chunk, tier, tiering, tiering_status, location_info) { } /** - * @param {nb.TierMirror} mirror + * @param {nb.TierMirror} mirror */ function map_frag_in_mirror(mirror) { const used_blocks = []; @@ -291,6 +291,13 @@ function is_chunk_good_for_dedup(chunk) { if (!chunk.is_accessible) return false; if (chunk.is_building_blocks) return false; if (chunk.is_building_frags) return false; + + // reject chunks that are not at least config.MIN_CHUNK_AGE_FOR_DEDUP old + // this is to avoid deduping chunks that might be from a previous attempt to write the same object + // see https://bugzilla.redhat.com/show_bug.cgi?id=2256223 + const chunk_age = Date.now() - chunk._id.getTimestamp().getTime(); + if (chunk_age < config.MIN_CHUNK_AGE_FOR_DEDUP) return false; + return true; } @@ -360,7 +367,7 @@ function _block_sort_newer_first(block1, block2) { /** - * @param {nb.Pool} pool + * @param {nb.Pool} pool * @returns {boolean} */ function _pool_has_redundancy(pool) { @@ -368,7 +375,7 @@ function _pool_has_redundancy(pool) { } // /** -// * +// * // * @param {nb.Chunk} chunk // * @param {nb.LocationInfo} [location_info] // */ diff --git a/src/test/unit_tests/test_encryption.js b/src/test/unit_tests/test_encryption.js index 0a60c1ec47..fcedf99186 100644 --- a/src/test/unit_tests/test_encryption.js +++ b/src/test/unit_tests/test_encryption.js @@ -29,6 +29,8 @@ let wrapped_coretest_secret_key; const BKT = `bucket.example`; const key_rotator = new KeyRotator({ name: 'kr'}); +config.MIN_CHUNK_AGE_FOR_DEDUP = 0; + mocha.describe('Encryption tests', function() { const { rpc_client, EMAIL, SYSTEM } = coretest; let response_account; @@ -991,10 +993,10 @@ mocha.describe('Rotation tests', function() { compare_secrets(secrets, system_store_account.master_key_id._id); }); }); -// TODO: +// TODO: // 1. add more tests for checking namespace resources // 2. add tests for enable/disable account that has pool/namespace resource -////////////// HELPERS +////////////// HELPERS async function multipart_upload(bucket, key, s3_conf) { let res = await s3_conf.createMultipartUpload({Bucket: bucket, Key: key, ContentType: 'text/plain'});