Skip to content

NSFS | Tagging bugs #8417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 7, 2024
Merged

NSFS | Tagging bugs #8417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions src/sdk/namespace_fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1165,7 +1165,6 @@ class NamespaceFS {
await this._throw_if_storage_class_not_supported(params.storage_class);

upload_params = await this._start_upload(fs_context, object_sdk, file_path, params, open_mode);

if (!params.copy_source || upload_params.copy_res === copy_status_enum.FALLBACK) {
// We are taking the buffer size closest to the sized upload
const bp = multi_buffer_pool.get_buffers_pool(params.size);
Expand Down Expand Up @@ -1332,6 +1331,13 @@ class NamespaceFS {
await this.append_to_migrate_wal(file_path);
}
}
if (params.tagging) {
for (const { key, value } of params.tagging) {
fs_xattr = Object.assign(fs_xattr || {}, {
[XATTR_TAG + key]: value
});
}
}
if (fs_xattr && !is_dir_content && should_replace_xattr) await target_file.replacexattr(fs_context, fs_xattr);
// fsync
if (config.NSFS_TRIGGER_FSYNC) await target_file.fsync(fs_context);
Expand Down Expand Up @@ -1952,15 +1958,17 @@ class NamespaceFS {
async get_object_tagging(params, object_sdk) {
const tag_set = [];
let file_path;
if (params.version_id && this._is_versioning_enabled()) {
file_path = this._get_version_path(params.key, params.version_id);
let file;
const fs_context = this.prepare_fs_context(object_sdk);
// Version specific tag will return even if versioning suspended.
if (params.version_id) {
file_path = await this._find_version_path(fs_context, params, true);
} else {
file_path = this._get_file_path(params);
file_path = this._get_file_md_path(params);
}
try {
const fs_context = this.prepare_fs_context(object_sdk);
dbg.log0('NamespaceFS.get_object_tagging: param ', params, 'file_path :', file_path);
const file = await nb_native().fs.open(fs_context, file_path);
file = await nb_native().fs.open(fs_context, file_path);
const stat = await file.stat(fs_context);
if (stat.xattr) {
for (const [xattr_key, xattr_value] of Object.entries(stat.xattr)) {
Expand All @@ -1975,6 +1983,8 @@ class NamespaceFS {
} catch (err) {
dbg.error(`NamespaceFS.get_object_tagging: failed in dir ${file_path} with error: `, err);
throw native_fs_utils.translate_error_codes(err, native_fs_utils.entity_enum.OBJECT);
} finally {
if (file) await file.close(fs_context);
}
dbg.log0('NamespaceFS.get_object_tagging: return tagging ', tag_set, 'file_path :', file_path);
return { tagging: tag_set };
Expand Down
259 changes: 259 additions & 0 deletions src/test/unit_tests/test_namespace_fs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/* Copyright (C) 2020 NooBaa */
/*eslint max-lines-per-function: ["error", 900]*/
/*eslint max-lines: ["error", 2100]*/
'use strict';

const _ = require('lodash');
Expand Down Expand Up @@ -1764,8 +1765,266 @@ mocha.describe('namespace_fs copy object', function() {
});
});

mocha.describe('namespace_fs upload object with tagging', function() {

const upload_bkt_tagging = 'test_ns_uploads_object_tagging';
const tmp_fs_path_tagging = path.join(TMP_PATH, 'test_namespace_fs_tagging');
const ns_tmp_bucket_path_tagging = `${tmp_fs_path_tagging}/${src_bkt}`;
const ns_tmp_tagging = new NamespaceFS({ bucket_path: ns_tmp_bucket_path_tagging, bucket_id: '3', namespace_resource_id: undefined });
mocha.before(async () => {
await P.all(_.map([src_bkt, upload_bkt_tagging], async buck =>
fs_utils.create_fresh_path(`${tmp_fs_path_tagging}/${buck}`)));
});
mocha.after(async () => {
await P.all(_.map([src_bkt, upload_bkt_tagging], async buck =>
fs_utils.folder_delete(`${tmp_fs_path_tagging}/${buck}`)));
await fs_utils.folder_delete(tmp_fs_path_tagging);
});
mocha.describe('upload_object (tagging)', function() {
const upload_key = 'upload_key_1';
const upload_folder_key = 'upload_folder_key1/';
const upload_folder_key_without_slash = 'upload_folder_key';
const data = crypto.randomBytes(100);
const tagging_1 = [{ key: 'tag1', value: 'val1' }];
const xattr = { key: 'key1', key2: 'value1' };

mocha.it('Upload object with tagging', async function() {
const params = {
bucket: upload_bkt_tagging,
key: upload_key,
xattr,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_1,
};
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_1);
});

mocha.it('Upload object with multiple tagging', async function() {
const tagging_multi = [{ key: 'tag1', value: 'val1' },
{ key: 'tag2', value: 'val2' },
{ key: 'tag3', value: 'val3' }
];
const params = {
bucket: upload_bkt_tagging,
key: upload_key,
xattr,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_multi,
};
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_multi);
});

mocha.it('Upload object with tagging, update tag', async function() {
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
const params = {
bucket: upload_bkt_tagging,
key: upload_key,
xattr,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_1,
};
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_1);

const update_params = {
bucket: upload_bkt_tagging,
key: upload_key,
xattr,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_2,
};
const update_upload_res = await ns_tmp_tagging.upload_object(update_params, dummy_object_sdk);
console.log('upload_object (tag) response after update', inspect(update_upload_res));
const update_tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key,
}, dummy_object_sdk);
console.log('get_object_tagging response after update', inspect(tag_res));
await validate_tagging(update_tag_res.tagging, tagging_2);
});

mocha.it('Upload folder object with tagging', async function() {
const params = {
bucket: upload_bkt_tagging,
key: upload_folder_key,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_1,
};
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_folder_key,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_1);
});

mocha.it('Upload folder object with tagging, get object tagging key without slash', async function() {
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
const params = {
bucket: upload_bkt_tagging,
key: upload_folder_key_without_slash,
xattr,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_2,
};
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_folder_key_without_slash,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_2);
});

mocha.it('Upload folder object with tagging and version enabled', async function() {
ns_tmp_tagging.set_bucket_versioning('ENABLED', dummy_object_sdk);
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
const tagging_3 = [{ key: 'tag3', value: 'val3' }];
const upload_key_2 = 'tagging_upload_key_2';
const params1 = {
bucket: upload_bkt_tagging,
key: upload_key_2,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_1,
};
const upload_res = await ns_tmp_tagging.upload_object(params1, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res));
const params2 = {
bucket: upload_bkt_tagging,
key: upload_key_2,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_2,
};
const upload_res1 = await ns_tmp_tagging.upload_object(params2, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res1));

//get tag for first version
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
version_id: upload_res.version_id,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res));
await validate_tagging(tag_res.tagging, tagging_1);

// suspend versioning, verion specific tag should retun.
ns_tmp_tagging.set_bucket_versioning('SUSPENDED', dummy_object_sdk);
const params3 = {
bucket: upload_bkt_tagging,
key: upload_key_2,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_3,
};
const upload_res2 = await ns_tmp_tagging.upload_object(params3, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res2));

//get tag for second version, after disabling version
const tag_res1 = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
version_id: upload_res1.version_id,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res1));
await validate_tagging(tag_res1.tagging, tagging_2);

//get tag for last item without version id
const tag_res2 = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
}, dummy_object_sdk);
console.log('get_object_tagging response', inspect(tag_res2));
await validate_tagging(tag_res2.tagging, tagging_3);
});

mocha.it('get tag with last version', async function() {
ns_tmp_tagging.set_bucket_versioning('ENABLED', dummy_object_sdk);
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
const upload_key_2 = 'tagging_upload_key_2';
const params1 = {
bucket: upload_bkt_tagging,
key: upload_key_2,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_1,
};
const upload_res = await ns_tmp_tagging.upload_object(params1, dummy_object_sdk);
console.log('upload_object (tag) response@@@', inspect(upload_res));
//get tag for first version, with version id
const tag_res = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
version_id: upload_res.version_id,
}, dummy_object_sdk);
await validate_tagging(tag_res.tagging, tagging_1);
//get tag for first version, without version id
const tag_res_1 = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
}, dummy_object_sdk);
await validate_tagging(tag_res_1.tagging, tagging_1);
const params2 = {
bucket: upload_bkt_tagging,
key: upload_key_2,
source_stream: buffer_utils.buffer_to_read_stream(data),
tagging: tagging_2,
};
const upload_res1 = await ns_tmp_tagging.upload_object(params2, dummy_object_sdk);
console.log('upload_object (tag) response', inspect(upload_res1));

//get tag for lates version, with version id
const tag_res1 = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
version_id: upload_res1.version_id,
}, dummy_object_sdk);
await validate_tagging(tag_res1.tagging, tagging_2);
//get tag for lates version, without version id
const tag_res1_1 = await ns_tmp_tagging.get_object_tagging({
bucket: upload_bkt_tagging,
key: upload_key_2,
}, dummy_object_sdk);
await validate_tagging(tag_res1_1.tagging, tagging_2);

});

});
});
});

async function validate_tagging(tag_res, tag_req) {
let count = 0;
if (tag_res.length === 0) {
assert.fail('object tag should not be empty');
}
for (const tagging of tag_res) {
assert.deepStrictEqual(tagging.key, tag_req[count].key);
assert.deepStrictEqual(tagging.value, tag_req[count].value);
count += 1;
}
}

//simulates object_sdk.fix_copy_source_params filtering of source xattr for copy object tests
async function _get_source_copy_xattr(copy_source, source_ns, object_sdk) {
const read_md_res = await source_ns.read_object_md({
Expand Down