NC | lifecycle | add noncurrent days rule

Signed-off-by: nadav mizrahi <[email protected]>

Author: nadavMiz

src/manage_nsfs/nc_lifecycle.js

@@ -580,7 +580,7 @@ class NCLifecycle {
 
     /**
      * check if object is delete candidate based on newer noncurrent versions rule
-     * @param {Object} object_info
+     * @param {nb.ObjectInfo} object_info
      * @param {Object} newer_noncurrent_state
      * @param {Number} num_newer_versions
      * @returns
@@ -600,21 +600,22 @@ class NCLifecycle {
 
     /**
      * check if object is delete candidate based on number of noncurrent days rule
-     * @param {Object} object_info
+     * @param {nb.ObjectInfo} object_info
      * @param {Number} num_non_current_days
      * @returns
      */
     filter_noncurrent_days(object_info, num_non_current_days) {
-        //TODO implement
-        return true;
+        if (object_info.is_latest) return false;
+        const noncurrent_time = object_info.nc_noncurrent_time;
+        return this._get_file_age_days(noncurrent_time) >= num_non_current_days;
     }
 
     /**
      * get_candidates_by_noncurrent_version_expiration_rule processes the noncurrent version expiration rule
      * TODO:
      * POSIX - need to support both noncurrent_days and newer_noncurrent_versions
      * GPFS - implement noncurrent_days using GPFS ILM policy as an optimization
-     * @param {*} lifecycle_rule
+     * @param {Object} lifecycle_rule
      * @param {Object} bucket_json
      * @returns {Promise<Object[]>}
      */

src/sdk/namespace_fs.js

@@ -2634,6 +2634,8 @@ class NamespaceFS {
         const storage_class = Glacier.storage_class_from_xattr(stat.xattr);
         const size = Number(stat.xattr?.[XATTR_DIR_CONTENT] || stat.size);
         const tag_count = stat.xattr ? this._number_of_tags_fs_xttr(stat.xattr) : 0;
+        const nc_noncurrent_time = (stat.xattr?.[XATTR_NON_CURRENT_TIMESTASMP] && Number(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP])) ||
+            stat.ctime.getTime();
 
         return {
             obj_id: etag,
@@ -2652,6 +2654,7 @@ class NamespaceFS {
             xattr: to_xattr(stat.xattr),
             tag_count,
             tagging: get_tags_from_xattr(stat.xattr),
+            nc_noncurrent_time,
 
             // temp:
             lock_settings: undefined,

src/sdk/nb.d.ts

@@ -442,6 +442,7 @@ interface ObjectInfo {
     restore_status?: RestoreStatus;
     checksum?: Checksum;
     object_parts?: GetObjectAttributesParts;
+    nc_noncurrent_time ?: number;
 }
 
 

src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js

@@ -21,6 +21,7 @@ const buffer_utils = require('../../../util/buffer_utils');
 const crypto = require('crypto');
 const NsfsObjectSDK = require('../../../sdk/nsfs_object_sdk');
 const nb_native = require('../../../util/nb_native');
+const native_fs_utils = require('../../../util/native_fs_utils');
 
 const LIFECYCLE_RULE_STATUS_ENUM = Object.freeze({
     ENABLED: 'Enabled',
@@ -578,6 +579,93 @@ describe('noobaa nc - lifecycle versioning ENABLE', () => {
             });
             expect(has_delete_marker).toBe(true);
         });
+
+        it('nc lifecycle - noncurrent expiration rule - expire older versions by number of days with filter - regular key', async () => {
+            const lifecycle_rule = [{
+                "id": "expire noncurrent versions after 3 days with size ",
+                "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED,
+                "filter": {
+                    "prefix": prefix,
+                    "object_size_greater_than": 80,
+                },
+                "noncurrent_version_expiration": {
+                    "noncurrent_days": 3
+                }
+            }];
+            await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+            let res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            await update_version_xattr(test_bucket, test_key1_regular, res.version_id);
+
+            const expected_res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false);
+            res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 60, false);
+            await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false);
+            await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false);
+            await update_version_xattr(test_bucket, test_prefix_key_regular, expected_res.version_id);
+            await update_version_xattr(test_bucket, test_prefix_key_regular, res.version_id);
+
+            await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined);
+            const object_list = await object_sdk.list_object_versions({ bucket: test_bucket });
+            expect(object_list.objects.length).toBe(5);
+            object_list.objects.forEach(element => {
+                expect(element.version_id).not.toBe(expected_res.version_id);
+            });
+        });
+
+        it('nc lifecycle - noncurrent expiration rule - both noncurrent days and older versions', async () => {
+            const lifecycle_rule = [{
+                "id": "expire noncurrent versions after 3 days with size ",
+                "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED,
+                "filter": {
+                    "prefix": '',
+                },
+                "noncurrent_version_expiration": {
+                    "noncurrent_days": 3,
+                    "newer_noncurrent_versions": 1
+                }
+            }];
+            await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+            const expected_res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            const res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            // older than 3 days but no more than one noncurrent version - don't delete
+            await update_version_xattr(test_bucket, test_key1_regular, expected_res.version_id);
+            // both older than 3 days and more than one noncurrent version - delete
+            await update_version_xattr(test_bucket, test_key1_regular, res.version_id);
+
+            await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined);
+            const object_list = await object_sdk.list_object_versions({ bucket: test_bucket });
+            expect(object_list.objects.length).toBe(2);
+            object_list.objects.forEach(element => {
+                expect(element.version_id).not.toBe(expected_res.version_id);
+            });
+        });
+
+        it('nc lifecycle - noncurrent expiration rule - older versions valid but noncurrent_days not valid', async () => {
+            const lifecycle_rule = [{
+                "id": "expire noncurrent versions after 3 days with size ",
+                "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED,
+                "filter": {
+                    "prefix": '',
+                },
+                "noncurrent_version_expiration": {
+                    "noncurrent_days": 3,
+                    "newer_noncurrent_versions": 1
+                }
+            }];
+            await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+            await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+            // more than one noncurrent version but not older than 3 days - don't delete
+            await create_object(object_sdk, test_bucket, test_key1_regular, 100, false);
+
+            await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined);
+            const object_list = await object_sdk.list_object_versions({ bucket: test_bucket });
+            expect(object_list.objects.length).toBe(3);
+        });
     });
 
     describe('noobaa nc - lifecycle versioning ENABLE - expiration rule - delete marker', () => {
@@ -1682,16 +1770,16 @@ describe('noobaa nc - lifecycle notifications', () => {
 /**
  * create_object creates an object with random data in the bucket
  * Note: is_old - if true, would update the mtime of the file.
- * @param {object} sdk
+ * @param {object} object_sdk
  * @param {string} bucket
  * @param {string} key
  * @param {number} size
  * @param {boolean} [is_old]
  * @param {{ key: string; value: string; }[]} [tagging]
  */
-async function create_object(sdk, bucket, key, size, is_old, tagging) {
+async function create_object(object_sdk, bucket, key, size, is_old, tagging) {
     const data = crypto.randomBytes(size);
-    const res = await sdk.upload_object({
+    const res = await object_sdk.upload_object({
         bucket,
         key,
         source_stream: buffer_utils.buffer_to_read_stream(data),
@@ -1720,6 +1808,33 @@ async function update_file_mtime(target_path) {
     await os_utils.exec(update_file_mtime_cmp, { return_stdout: true });
 }
 
+/**
+ * updates the number of noncurrent days xattr of target path to be 5 days older. use only on noncurrent objects.
+ * is use this function on latest object the xattr will be changed when the object turns noncurrent
+ * how to use this function:
+ * 1. create a new object but don't change its mtime (changing mtime will cause versioning functions to fail)
+ * 2. create a new object with the same key to make the object noncurrent
+ * 3. call this function to change the xattr of the noncurrent object
+ * @param {String} bucket
+ * @param {String} key
+ * @param {String} version_id
+ * @returns {Promise<Void>}
+ */
+async function update_version_xattr(bucket, key, version_id) {
+    const older_time = new Date();
+    older_time.setDate(yesterday.getDate() - 5); // 5 days ago
+
+    const target_path = path.join(root_path, bucket, path.dirname(key), '.versions', `${path.basename(key)}_${version_id}`);
+    const file = await nb_native().fs.open(config_fs.fs_context, target_path, config.NSFS_OPEN_READ_MODE,
+        native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE));
+    const stat = await file.stat(config_fs.fs_context);
+    const xattr = Object.assign(stat.xattr, {
+        'user.noobaa.non_current_timestamp': older_time.getTime(),
+    });
+    await file.replacexattr(config_fs.fs_context, xattr, undefined);
+    await file.close(config_fs.fs_context);
+}
+
 /**
  * date_to_run_time_format coverts a date to run time format HH:MM
  * @param {Date} date