diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index a8bc6db067..14b716cc58 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -631,7 +631,7 @@ class MDStore { * @property {1|-1} [order] * @property {boolean} [pagination] * - * @returns {nb.ObjectMD[]} + * @returns {Promise} */ async find_objects({ bucket_id, @@ -695,6 +695,80 @@ class MDStore { }); } + /** + * TODO add support for versioning or add another function to support versioning. + * @typedef {Object} DeleteObjectsParams + * @property {nb.ID} bucket_id + * @property {RegExp} key + * @property {number} [max_create_time] + * @property {number} [max_size] + * @property {number} [min_size] + * @property {Array<{ key: string; value: string; }>} [tagging] + * @property {number} [limit] + * @property {boolean} [return_results] + * + * @param {DeleteObjectsParams} params + * @returns {Promise} + */ + async delete_objects_by_query({ + bucket_id, + key, + max_create_time, + max_size, + min_size, + tagging, + limit, + return_results = false, + }) { + const params = [new Date()]; + const sql_conditions = []; + if (key) { + params.push(key.source); + sql_conditions.push(`data->>'key' ~ $${params.length}`); + } + if (max_size !== undefined) { + params.push(max_size); + sql_conditions.push(`(data->>'size')::BIGINT < $${params.length}`); + } + if (min_size !== undefined) { + params.push(min_size); + sql_conditions.push(`(data->>'size')::BIGINT > $${params.length}`); + } + if (tagging && tagging.length) { + params.push(JSON.stringify(tagging)); + sql_conditions.push(`(data->>'tagging')::jsonb @> $${params.length}::jsonb`); + } + if (max_create_time) { + params.push(new Date(moment.unix(max_create_time).toISOString()).toISOString()); + sql_conditions.push(`data->>'create_time' < $${params.length}`); + } + + const sql_limit = limit === undefined ? "" : `LIMIT ${limit}`; + + let query = ` + WITH rows AS ( + SELECT _id + FROM ${this._objects.name} + WHERE + ${sql_and_conditions( + `data->>'bucket' = '${bucket_id}'`, + ...sql_conditions, + `data->'deleted' IS NULL`, + `data->'upload_started' IS NULL`, + `data->'version_enabled' IS NULL`, + )} + ${sql_limit} + ) + UPDATE ${this._objects.name} + SET data = jsonb_set(data, '{deleted}', to_jsonb($1::text), true) + WHERE _id IN ( + SELECT rows._id FROM rows + )`; + query += return_results ? ' RETURNING *;' : ';'; + const result = await this._objects.executeSQL(query, params); + return return_results ? result.rows : []; + } + async find_unreclaimed_objects(limit) { const results = await this._objects.find({ deleted: { $exists: true }, diff --git a/src/server/object_services/object_server.js b/src/server/object_services/object_server.js index 512d870caa..f63c979b15 100644 --- a/src/server/object_services/object_server.js +++ b/src/server/object_services/object_server.js @@ -971,7 +971,7 @@ async function delete_multiple_objects_by_filter(req) { const key = new RegExp('^' + _.escapeRegExp(req.rpc_params.prefix)); const bucket_id = req.bucket._id; const reply_objects = req.rpc_params.reply_objects; - // TODO: change it to perform changes in batch. Won't scale. + const query = { bucket_id, key, @@ -983,18 +983,26 @@ async function delete_multiple_objects_by_filter(req) { min_size: req.rpc_params.size_greater, limit: req.rpc_params.limit, }; - - const objects = await MDStore.instance().find_objects(query); - - const delete_results = await delete_multiple_objects(_.assign(req, { - rpc_params: { - bucket: req.bucket.name, - objects: _.map(objects, obj => ({ - key: obj.key, - version_id: req.rpc_params.delete_version ? MDStore.instance().get_object_version_id(obj) : '', - })) - } - })); + let delete_results; + let objects; + // TODO: Add support to delete_objects_by_query also for versioning or add another function to support versioning. + if (req.bucket.versioning === 'DISABLED' && config.DB_TYPE !== 'mongodb') /* only for postgres */ { + query.return_results = true; // we want to return the objects that were deleted + objects = await MDStore.instance().delete_objects_by_query(query); + } else { + // TODO: change it to perform changes in batch. Won't scale + objects = await MDStore.instance().find_objects(query); + + delete_results = await delete_multiple_objects(_.assign(req, { + rpc_params: { + bucket: req.bucket.name, + objects: _.map(objects, obj => ({ + key: obj.key, + version_id: req.rpc_params.delete_version ? MDStore.instance().get_object_version_id(obj) : '', + })) + } + })); + } const reply = { num_objects_deleted: objects.length }; if (reply_objects) { @@ -1004,7 +1012,7 @@ async function delete_multiple_objects_by_filter(req) { //or incude the error if deletion failed reply.deleted_objects = []; for (let i = 0; i < objects.length; ++i) { - if (delete_results[i].err_code) { + if (delete_results && delete_results[i].err_code) { reply.deleted_objects[i] = { err_code: delete_results[i].err_code, err_message: delete_results[i].err_message diff --git a/src/test/integration_tests/api/s3/test_lifecycle.js b/src/test/integration_tests/api/s3/test_lifecycle.js index e0f16fad3a..74d1b37455 100644 --- a/src/test/integration_tests/api/s3/test_lifecycle.js +++ b/src/test/integration_tests/api/s3/test_lifecycle.js @@ -287,18 +287,16 @@ mocha.describe('lifecycle', () => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { - Rules: [ - { - AbortIncompleteMultipartUpload: { - // 10 less days have gone by since upload created - DaysAfterInitiation: parts_age + 10, - }, - Filter: { - Prefix: '' - }, - Status: 'Enabled', - } - ], + Rules: [{ + AbortIncompleteMultipartUpload: { + // 10 less days have gone by since upload created + DaysAfterInitiation: parts_age + 10, + }, + Filter: { + Prefix: '' + }, + Status: 'Enabled', + }], }, }; @@ -329,18 +327,16 @@ mocha.describe('lifecycle', () => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { - Rules: [ - { - AbortIncompleteMultipartUpload: { - // 10 less days have gone by since upload created - DaysAfterInitiation: parts_age + 10, - }, - Filter: { - Prefix: prefix, - }, - Status: 'Enabled', - } - ], + Rules: [{ + AbortIncompleteMultipartUpload: { + // 10 less days have gone by since upload created + DaysAfterInitiation: parts_age + 10, + }, + Filter: { + Prefix: prefix, + }, + Status: 'Enabled', + }], }, }; @@ -407,7 +403,7 @@ mocha.describe('lifecycle', () => { const mp_list_after = await rpc_client.object.list_multiparts({ obj_id, bucket, key }); assert.strictEqual(mp_list_after.multiparts.length, num_parts, - `list_multiparts actual ${mp_list_after.multiparts.length} !== ${num_parts}`); + `list_multiparts actual ${mp_list_after.multiparts.length} !== ${num_parts}`); return obj_id; } @@ -495,17 +491,15 @@ mocha.describe('lifecycle', () => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { - Rules: [ - { - NoncurrentVersionExpiration: { - NoncurrentDays: age + 10, - }, - Filter: { - Prefix: '' - }, - Status: 'Enabled', - } - ], + Rules: [{ + NoncurrentVersionExpiration: { + NoncurrentDays: age + 10, + }, + Filter: { + Prefix: '' + }, + Status: 'Enabled', + }], }, }; @@ -535,17 +529,15 @@ mocha.describe('lifecycle', () => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { - Rules: [ - { - NoncurrentVersionExpiration: { - NoncurrentDays: age + 10, - }, - Filter: { - Prefix: prefix, - }, - Status: 'Enabled', - } - ], + Rules: [{ + NoncurrentVersionExpiration: { + NoncurrentDays: age + 10, + }, + Filter: { + Prefix: prefix, + }, + Status: 'Enabled', + }], }, }; @@ -576,20 +568,18 @@ mocha.describe('lifecycle', () => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { - Rules: [ - { - NoncurrentVersionExpiration: { - // Age exceeds but newernoncurrent versions doesn't - // so no expiration should happen - NoncurrentDays: age - 10, - NewerNoncurrentVersions: 10 - }, - Filter: { - Prefix: '' - }, - Status: 'Enabled', - } - ], + Rules: [{ + NoncurrentVersionExpiration: { + // Age exceeds but newernoncurrent versions doesn't + // so no expiration should happen + NoncurrentDays: age - 10, + NewerNoncurrentVersions: 10 + }, + Filter: { + Prefix: '' + }, + Status: 'Enabled', + }], }, }; @@ -634,7 +624,7 @@ mocha.describe('lifecycle', () => { } } - mocha.it('lifecyle - version object expired', async () => { + mocha.it('lifecycle - version object expired', async () => { const age = 30; const version_count = 10; const version_bucket_key = 'test-lifecycle-version1-1'; @@ -646,7 +636,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(version_count + 1, version_bucket_key); }); - mocha.it('lifecyle - version object not expired', async () => { + mocha.it('lifecycle - version object not expired', async () => { const age = 5; const version_count = 10; const version_bucket_key = 'test-lifecycle-version2-0'; @@ -658,7 +648,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(version_count, version_bucket_key); }); - mocha.it('lifecyle - version not expiration', async () => { + mocha.it('lifecycle - version not expiration', async () => { const days = 30; const version_count = 10; const newnon_current_version = 1; @@ -667,7 +657,7 @@ mocha.describe('lifecycle', () => { // create_time updated to 29 days and expire is 30 days and noncurrent expire in 15 await create_mock_version(version_bucket_key, version_bucket, days - 1, version_count); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, newnon_current_version, noncurrent_days); + version_bucket_key, days, newnon_current_version, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); @@ -676,7 +666,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(2, version_bucket_key); }); - mocha.it('lifecyle - version expiration - only NewerNoncurrentVersions exceeded', async () => { + mocha.it('lifecycle - version expiration - only NewerNoncurrentVersions exceeded', async () => { const days = 35; const version_count = 10; const newnon_current_version = 5; @@ -685,7 +675,7 @@ mocha.describe('lifecycle', () => { // create_time updated to 36 days and expire is 35 days await create_mock_version(version_bucket_key, version_bucket, days + 1, version_count, true); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, newnon_current_version, noncurrent_days); + version_bucket_key, days, newnon_current_version, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); @@ -693,7 +683,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(7, version_bucket_key); }); - mocha.it('lifecyle - version expiration - only NoncurrentDays exceeded', async () => { + mocha.it('lifecycle - version expiration - only NoncurrentDays exceeded', async () => { const days = 45; const version_count = 10; const newnon_current_version = 100; @@ -702,7 +692,7 @@ mocha.describe('lifecycle', () => { // create_time updated to 45+30+1= 76 days and expire is 45 days await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count, true); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, newnon_current_version, noncurrent_days); + version_bucket_key, days, newnon_current_version, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); @@ -711,7 +701,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(11, version_bucket_key); }); - mocha.it('lifecyle - version expiration - both NoncurrentDays and NewerNoncurrentVersions exceeded', async () => { + mocha.it('lifecycle - version expiration - both NoncurrentDays and NewerNoncurrentVersions exceeded', async () => { const days = 30; const version_count = 10; const newnon_current_version = 1; @@ -720,7 +710,7 @@ mocha.describe('lifecycle', () => { // create_time updated to 46 days and expire is 30 days await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, newnon_current_version, noncurrent_days); + version_bucket_key, days, newnon_current_version, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); @@ -728,7 +718,7 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(2, version_bucket_key); }); - mocha.it('lifecyle - version not expiration - delete marker true', async () => { + mocha.it('lifecycle - version not expiration - delete marker true', async () => { const days = 30; const version_count = 10; const noncurrent_days = 15; @@ -736,7 +726,7 @@ mocha.describe('lifecycle', () => { // create_time updated to 29 days and expire is 30 days, await create_mock_version(version_bucket_key, version_bucket, days - 1, version_count, true); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, undefined, noncurrent_days); + version_bucket_key, days, undefined, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); @@ -744,14 +734,14 @@ mocha.describe('lifecycle', () => { await verify_version_deleted(1, version_bucket_key); }); - mocha.it('lifecyle - version expiration all - delete marker true', async () => { + mocha.it('lifecycle - version expiration all - delete marker true', async () => { const days = 30; const version_count = 10; const noncurrent_days = 15; const version_bucket_key = 'test-lifecycle-version6'; await create_mock_version(version_bucket_key, version_bucket, days + noncurrent_days + 1, version_count, true); const putLifecycleParams = commonTests.version_lifecycle_configuration(version_bucket, - version_bucket_key, days, undefined, noncurrent_days); + version_bucket_key, days, undefined, noncurrent_days); await s3.putBucketLifecycleConfiguration(putLifecycleParams); await lifecycle.background_worker(); await verify_version_deleted(2, version_bucket_key); @@ -776,7 +766,7 @@ mocha.describe('lifecycle', () => { prefix: key, }; const obj_ids = []; - const {objects} = await rpc_client.object.list_object_versions(obj_params); + const { objects } = await rpc_client.object.list_object_versions(obj_params); for (const object of objects) { obj_ids.push(object.obj_id); } @@ -804,7 +794,7 @@ mocha.describe('lifecycle', () => { mocha.describe('bucket-lifecycle-expiration-header', function() { const bucket = Bucket; - const run_expiration_test = async ({ rules, expected_id, expected_days, key, tagging = undefined, size = 1000}) => { + const run_expiration_test = async ({ rules, expected_id, expected_days, key, tagging = undefined, size = 1000 }) => { const putLifecycleParams = { Bucket: bucket, LifecycleConfiguration: { Rules: rules } diff --git a/src/test/integration_tests/db/test_md_store.js b/src/test/integration_tests/db/test_md_store.js index 3bc1c4fbe8..2b77b9df03 100644 --- a/src/test/integration_tests/db/test_md_store.js +++ b/src/test/integration_tests/db/test_md_store.js @@ -50,6 +50,91 @@ mocha.describe('md_store', function() { assert_equal(res.obj.num_parts, 88); }); + const date_now = Date.now(); + const now = new Date(date_now); + const info1 = { + _id: md_store.make_md_id(), + system: system_id, + bucket: bucket_id, + key: 'lala_1' + date_now.toString(36), + create_time: now, + content_type: 'lulu_' + date_now.toString(36), + }; + const info2 = { + _id: md_store.make_md_id(), + system: system_id, + bucket: bucket_id, + key: 'lala_2' + date_now.toString(36), + create_time: now, + content_type: 'lulu_' + date_now.toString(36), + }; + const max_create_time = now.getTime() / 1000 - 60; // 1 minute ago + + mocha.it('delete_objects_by_query - key', async function() { + if (config.DB_TYPE !== 'postgres') this.skip(); // eslint-disable-line no-invalid-this + info1._id = md_store.make_md_id(); + await md_store.insert_object(info1); + info2._id = md_store.make_md_id(); + await md_store.insert_object(info2); + let obj = await md_store.find_object_by_id(info1._id); + assert_equal(obj, info1); + obj = await md_store.find_object_by_id(info2._id); + assert_equal(obj, info2); + const objects = await md_store.delete_objects_by_query({ + bucket_id: bucket_id, + key: /^lala_/, + return_results: true, + }); + console.log('deleted objects:', objects); + assert_equal(objects.length, 2); + }); + + mocha.it('delete_objects_by_query - max_create_time', async function() { + if (config.DB_TYPE !== 'postgres') this.skip(); // eslint-disable-line no-invalid-this + info1._id = md_store.make_md_id(); + info1.create_time = new Date(max_create_time - 100); + await md_store.insert_object(info1); + info2._id = md_store.make_md_id(); + await md_store.insert_object(info2); + let obj = await md_store.find_object_by_id(info1._id); + assert_equal(obj, info1); + obj = await md_store.find_object_by_id(info2._id); + assert_equal(obj, info2); + const query = { + key: /^lala_/, + bucket_id: bucket_id, + max_create_time, + return_results: true, + }; + let objects = await md_store.delete_objects_by_query(query); + console.log('deleted objects:', objects); + assert_equal(objects.length, 1); + query.max_create_time = now.getTime() / 1000 + 1; // 1 second later + objects = await md_store.delete_objects_by_query(query); + console.log('deleted objects:', objects); + assert_equal(objects.length, 1); + }); + + mocha.it('delete_objects_by_query - limit', async function() { + if (config.DB_TYPE !== 'postgres') this.skip(); // eslint-disable-line no-invalid-this + info1._id = md_store.make_md_id(); + await md_store.insert_object(info1); + info2._id = md_store.make_md_id(); + await md_store.insert_object(info2); + let obj = await md_store.find_object_by_id(info1._id); + assert_equal(obj, info1); + obj = await md_store.find_object_by_id(info2._id); + assert_equal(obj, info2); + const objects = await md_store.delete_objects_by_query({ + key: /^lala_/, + bucket_id: bucket_id, + limit: 1, + return_results: true, + }); + console.log('deleted objects:', objects); + assert_equal(objects.length, 1); + }); + mocha.it('insert_object() detects missing key (bad schema)', async function() { const info = { _id: md_store.make_md_id(), @@ -140,35 +225,36 @@ mocha.describe('md_store', function() { mocha.describe('parts', function() { const parts = [{ - _id: md_store.make_md_id(), - system: system_id, - bucket: md_store.make_md_id(), - obj: md_store.make_md_id(), - chunk: md_store.make_md_id(), - start: 0, - end: 10, - seq: 0, - }, - { - _id: md_store.make_md_id(), - system: system_id, - bucket: md_store.make_md_id(), - obj: md_store.make_md_id(), - chunk: md_store.make_md_id(), - start: 0, - end: 20, - seq: 0, - }, - { - _id: md_store.make_md_id(), - system: system_id, - bucket: md_store.make_md_id(), - obj: md_store.make_md_id(), - chunk: md_store.make_md_id(), - start: 0, - end: 20, - seq: 0, - }]; + _id: md_store.make_md_id(), + system: system_id, + bucket: md_store.make_md_id(), + obj: md_store.make_md_id(), + chunk: md_store.make_md_id(), + start: 0, + end: 10, + seq: 0, + }, + { + _id: md_store.make_md_id(), + system: system_id, + bucket: md_store.make_md_id(), + obj: md_store.make_md_id(), + chunk: md_store.make_md_id(), + start: 0, + end: 20, + seq: 0, + }, + { + _id: md_store.make_md_id(), + system: system_id, + bucket: md_store.make_md_id(), + obj: md_store.make_md_id(), + chunk: md_store.make_md_id(), + start: 0, + end: 20, + seq: 0, + } + ]; mocha.it('insert_parts()', async function() { return md_store.insert_parts(parts); @@ -214,7 +300,7 @@ mocha.describe('md_store', function() { }); mocha.it('delete_parts_by_ids()', async function() { - const part_ids = [ parts[1]._id ]; + const part_ids = [parts[1]._id]; return md_store.delete_parts_by_ids(part_ids); }); @@ -225,7 +311,7 @@ mocha.describe('md_store', function() { mocha.it('has_any_parts_for_object deleted', async function() { const obj = { _id: parts[2].obj }; - const part_ids = [ parts[2]._id ]; + const part_ids = [parts[2]._id]; await md_store.delete_parts_by_ids(part_ids); assert.equal(await md_store.has_any_parts_for_object(obj), false); });