Merge pull request #8504 from tangledbytes/utkarsh/backport/5.17.1

[Backport 5.17] Backport of DB and Scale fixes to 4.17.1

Author: nimrod-becker

config.js

@@ -198,6 +198,22 @@ config.S3_RESTORE_REQUEST_MAX_DAYS = 30;
  */
 config.S3_RESTORE_REQUEST_MAX_DAYS_BEHAVIOUR = 'TRUNCATE';
 
+/**
+ * S3_MAX_KEY_LENGTH controls the maximum key length that will be accepted
+ * by NooBaa endpoints.
+ * 
+ * This value is 1024 bytes for S3 but the default is `Infinity`
+ */
+config.S3_MAX_KEY_LENGTH = Infinity;
+
+/**
+ * S3_MAX_BUCKET_NAME_LENGTH controls the maximum bucket name length that
+ * will be accepted by NooBaa endpoints.
+ * 
+ * This value is 63 bytes for S3 but the default is `Infinity`
+ */
+config.S3_MAX_BUCKET_NAME_LENGTH = Infinity;
+
 /////////////////////
 // SECRETS CONFIG  //
 /////////////////////

src/endpoint/s3/ops/s3_get_bucket_lifecycle.js

@@ -47,6 +47,7 @@ async function get_bucket_lifecycle(req) {
             current_rule.Expiration = {
                 Days: rule.expiration.days,
                 Date: rule.expiration.date ? new Date(rule.expiration.date).toISOString() : undefined,
+                ExpiredObjectDeleteMarker: rule.expiration.expired_object_delete_marker,
             };
             _.omitBy(current_rule.Expiration, _.isUndefined);
         }

src/endpoint/s3/ops/s3_put_bucket_lifecycle.js

@@ -46,6 +46,13 @@ function parse_filter(filter) {
     return current_rule_filter;
 }
 
+function reject_empty_field(field) {
+    if (_.isEmpty(field)) {
+        dbg.error('Invalid field - empty', field);
+        throw new S3Error(S3Error.MalformedXML);
+    }
+}
+
 // parse lifecycle rule expiration
 function parse_expiration(expiration) {
     const output_expiration = {};
@@ -111,12 +118,14 @@ async function put_bucket_lifecycle(req) {
 
         if (rule.Expiration?.length === 1) {
             current_rule.expiration = parse_expiration(rule.Expiration[0]);
+            reject_empty_field(current_rule.expiration);
         }
 
         if (rule.AbortIncompleteMultipartUpload?.length === 1) {
             current_rule.abort_incomplete_multipart_upload = _.omitBy({
                 days_after_initiation: parse_lifecycle_field(rule.AbortIncompleteMultipartUpload[0].DaysAfterInitiation),
             }, _.isUndefined);
+            reject_empty_field(current_rule.abort_incomplete_multipart_upload);
         }
 
         if (rule.Transition?.length === 1) {
@@ -125,13 +134,15 @@ async function put_bucket_lifecycle(req) {
                 date: parse_lifecycle_field(rule.Transition[0].Date, s => new Date(s)),
                 days: parse_lifecycle_field(rule.Transition[0].Days),
             }, _.isUndefined);
+            reject_empty_field(current_rule.transition);
         }
 
         if (rule.NoncurrentVersionExpiration?.length === 1) {
             current_rule.noncurrent_version_expiration = _.omitBy({
                 noncurrent_days: parse_lifecycle_field(rule.NoncurrentVersionExpiration[0].NoncurrentDays),
                 newer_noncurrent_versions: parse_lifecycle_field(rule.NoncurrentVersionExpiration[0].NewerNoncurrentVersions),
             }, _.isUndefined);
+            reject_empty_field(current_rule.noncurrent_version_expiration);
         }
 
         if (rule.NoncurrentVersionTransition?.length === 1) {
@@ -140,6 +151,7 @@ async function put_bucket_lifecycle(req) {
                 noncurrent_days: parse_lifecycle_field(rule.NoncurrentVersionTransition[0].NoncurrentDays),
                 newer_noncurrent_versions: parse_lifecycle_field(rule.NoncurrentVersionTransition[0].NewerNoncurrentVersions),
             }, _.isUndefined);
+            reject_empty_field(current_rule.noncurrent_version_transition);
         }
 
         return current_rule;

src/endpoint/s3/s3_errors.js

@@ -249,8 +249,8 @@ S3Error.InvalidURI = Object.freeze({
     message: 'Couldn\'t parse the specified URI.',
     http_code: 400,
 });
-S3Error.KeyTooLong = Object.freeze({
-    code: 'KeyTooLong',
+S3Error.KeyTooLongError = Object.freeze({
+    code: 'KeyTooLongError',
     message: 'Your key is too long.',
     http_code: 400,
 });

src/endpoint/s3/s3_rest.js

@@ -340,6 +340,14 @@ function get_bucket_and_key(req) {
             key = suffix;
         }
     }
+
+    if (key?.length > config.S3_MAX_KEY_LENGTH) {
+        throw new S3Error(S3Error.KeyTooLongError);
+    }
+    if (bucket?.length > config.S3_MAX_BUCKET_NAME_LENGTH) {
+        throw new S3Error(S3Error.InvalidBucketName);
+    }
+
     return {
         bucket,
         // decode and replace hadoop _$folder$ in key

src/sdk/namespace_fs.js

@@ -963,6 +963,7 @@ class NamespaceFS {
                     dbg.warn(`NamespaceFS.read_object_md: retrying retries=${retries} file_path=${file_path}`, err);
                     retries -= 1;
                     if (retries <= 0 || !native_fs_utils.should_retry_link_unlink(is_gpfs, err)) throw err;
+                    await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50));
                 }
             }
             this._throw_if_delete_marker(stat, params);
@@ -1037,6 +1038,7 @@ class NamespaceFS {
                             {bucket_path: this.bucket_path, object_name: params.key}, err);
                         throw err;
                     }
+                    await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50));
                 }
             }
             this._throw_if_delete_marker(stat, params);
@@ -2847,6 +2849,13 @@ class NamespaceFS {
                 const deleted_latest = file_path === latest_version_path;
                 if (deleted_latest) {
                     gpfs_options = await this._open_files_gpfs(fs_context, file_path, undefined, undefined, undefined, undefined, true);
+                    if (gpfs_options) {
+                        const src_stat = await gpfs_options.delete_version.src_file.stat(fs_context);
+                        if (this._is_mismatch_version_id(src_stat, version_id)) {
+                            dbg.warn('NamespaceFS._delete_single_object_versioned mismatch version_id', file_path, version_id, this._get_version_id_by_xattr(src_stat));
+                            throw error_utils.new_error_code('MISMATCH_VERSION', 'file version does not match the version we asked for');
+                        }
+                    }
                     const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
                     await native_fs_utils.safe_unlink(fs_context, file_path, version_info,
                         gpfs_options?.delete_version, bucket_tmp_dir_path);
@@ -3007,15 +3016,17 @@ class NamespaceFS {
         let latest_ver_info;
         for (;;) {
             try {
-                // get latest version_id if exists
+                // TODO get latest version from file in POSIX like in GPFS path
                 latest_ver_info = await this._get_version_info(fs_context, latest_ver_path);
-                const versioned_path = latest_ver_info && this._get_version_path(params.key, latest_ver_info.version_id_str);
-                const versioned_info = latest_ver_info && await this._get_version_info(fs_context, versioned_path);
-
-                dbg.log1('Namespace_fs._delete_latest_version:', latest_ver_info, versioned_path, versioned_info);
+                dbg.log1('Namespace_fs._delete_latest_version:', latest_ver_info);
                 if (latest_ver_info) {
+                    if (is_gpfs) {
                     gpfs_options = await this._open_files_gpfs(fs_context, latest_ver_path, undefined, undefined, undefined,
-                        undefined, true, versioned_info);
+                            undefined, true);
+                        const latest_fd = gpfs_options?.move_to_dst?.dst_file;
+                        latest_ver_info = latest_fd && await this._get_version_info(fs_context, undefined, latest_fd);
+                    }
+                    const versioned_path = latest_ver_info && this._get_version_path(params.key, latest_ver_info.version_id_str);
 
                     const suspended_and_latest_is_not_null = this._is_versioning_suspended() &&
                         latest_ver_info.version_id_str !== NULL_VERSION_ID;
@@ -3172,7 +3183,7 @@ class NamespaceFS {
         let dir_file;
         let versioned_file;
         try {
-            // open /versions/key_ver file if exists
+            // open /versions/key_ver file if exists. TODO is versioned_file needed
             versioned_file = versioned_info && await native_fs_utils.open_file(fs_context, this.bucket_path, versioned_info.path, 'r');
 
             // open files for deletion flow

src/sdk/nsfs_glacier_backend/tapecloud.js

@@ -51,7 +51,7 @@ class TapeCloudUtils {
                 throw new Error('process exited with non-zero exit code:', errcode);
             }
 
-            reader = new NewlineReader(fs_context, tmp);
+            reader = new NewlineReader(fs_context, tmp, { skip_overflow_lines: true });
             await reader.forEach(async line => {
                 const failure_case = line.startsWith("Fail");
                 const success_case = line.startsWith("Success");