Skip to content

NC | NSFS | CLI | Improve Performance of List with Name Filter #8272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 13, 2024
Merged

NC | NSFS | CLI | Improve Performance of List with Name Filter #8272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions src/cmd/manage_nsfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,10 @@ async function main(argv = minimist(process.argv.slice(2))) {
}

async function bucket_management(action, user_input) {
const data = await fetch_bucket_data(action, user_input);
let data;
if (action !== ACTIONS.LIST) {
data = await fetch_bucket_data(action, user_input);
}
await manage_bucket_operations(action, data, user_input);
}

Expand Down Expand Up @@ -265,7 +268,10 @@ async function account_management(action, user_input) {
// init if actions is add/update (require encryption) or show_secrets = true (require decryption)
if ([ACTIONS.ADD, ACTIONS.UPDATE].includes(action) || show_secrets) await nc_mkm.init();

const data = await fetch_account_data(action, user_input);
let data;
if (action !== ACTIONS.LIST) {
data = await fetch_account_data(action, user_input);
}
await manage_account_operations(action, data, show_secrets, user_input);
}

Expand Down Expand Up @@ -571,10 +577,18 @@ function filter_bucket(bucket, filters) {
* @param {boolean} [show_secrets]
* @param {object} [filters]
*/
async function list_config_files(type, wide, show_secrets, filters) {
const entries = type === TYPES.ACCOUNT ?
async function list_config_files(type, wide, show_secrets, filters = {}) {
let entries;
// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
const is_filter_by_name = filters.name !== undefined;
if (is_filter_by_name) {
entries = [{'name': filters.name + JSON_SUFFIX}];
} else {
entries = type === TYPES.ACCOUNT ?
await config_fs.list_root_accounts() :
await config_fs.list_buckets();
}

const should_filter = Object.keys(filters).length > 0;
// decryption causing mkm initalization
Expand Down
27 changes: 27 additions & 0 deletions src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1133,6 +1133,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
});
Expand All @@ -1141,6 +1142,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: true };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
// added additional properties that we can see with wide option (uid, new_buckets_path)
Expand All @@ -1155,6 +1157,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: 'true' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
// added additional properties that we can see with wide option (uid, new_buckets_path)
Expand All @@ -1169,6 +1172,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: 'TRUE' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
// added additional properties that we can see with wide option (uid, new_buckets_path)
Expand All @@ -1183,6 +1187,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: 'false' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
});
Expand All @@ -1191,6 +1196,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: 'FALSE' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
});
Expand Down Expand Up @@ -1218,6 +1224,7 @@ describe('manage nsfs cli account flow', () => {
} catch (e) {
res = e;
}
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
});
Expand All @@ -1226,6 +1233,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, uid: 999 };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account1']));
});
Expand All @@ -1234,6 +1242,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, gid: 999 };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account1']));
});
Expand All @@ -1242,6 +1251,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, uid: 999, gid: 999 };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account1']));
});
Expand All @@ -1250,6 +1260,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, uid: 999, gid: 888 };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3']));
});
Expand All @@ -1265,6 +1276,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, user: 'root' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account4']));
});
Expand All @@ -1273,6 +1285,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, user: 'shaul' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual([]);
});
Expand All @@ -1281,6 +1294,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, access_key: 'GIGiFAnjaaE7OKD5N7hA' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account1']));
});
Expand All @@ -1289,14 +1303,25 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, name: 'account3' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3']));
});

it('cli list filter by name (non-existing-account) - (none)', async () => {
const account_options = { config_root, name: 'non-existing-account' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual([]);
});

it('cli list filter by access key (of account1) and name (of account3) - (none)', async () => {
const account_options = { config_root, name: 'account3', access_key: 'GIGiFAnjaaE7OKD5N7hA' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual([]);
});
Expand All @@ -1305,6 +1330,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, name: 'account3', access_key: 'non-existing-access-key' };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual([]);
});
Expand All @@ -1320,6 +1346,7 @@ describe('manage nsfs cli account flow', () => {
const account_options = { config_root, wide: true, show_secrets: true };
const action = ACTIONS.LIST;
const res = await exec_manage_cli(type, action, account_options);
expect(Array.isArray(JSON.parse(res).response.reply)).toBe(true);
expect(JSON.parse(res).response.reply.map(item => item.name))
.toEqual(expect.arrayContaining(['account3', 'account2', 'account1']));
const res_arr = JSON.parse(res).response.reply;
Expand Down