Skip to content

NC | Config Dir Restructure #8279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 19, 2024
Merged

NC | Config Dir Restructure #8279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 35 additions & 37 deletions src/cmd/manage_nsfs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const minimist = require('minimist');
const config = require('../../config');
const P = require('../util/promise');
const nb_native = require('../util/nb_native');
const { ConfigFS, JSON_SUFFIX } = require('../sdk/config_fs');
const { ConfigFS } = require('../sdk/config_fs');
const cloud_utils = require('../util/cloud_utils');
const native_fs_utils = require('../util/native_fs_utils');
const mongo_utils = require('../util/mongo_utils');
Expand Down Expand Up @@ -387,8 +387,8 @@ async function add_account(data) {
await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.ADD, undefined);

const access_key = has_access_keys(data.access_keys) ? data.access_keys[0].access_key : undefined;
const name_exists = await config_fs.is_account_exists({ name: data.name });
const access_key_exists = access_key && await config_fs.is_account_exists({ access_key });
const name_exists = await config_fs.is_account_exists_by_name(data.name);
const access_key_exists = access_key && await config_fs.is_account_exists_by_access_key(access_key);

const event_arg = data.name ? data.name : access_key;
if (name_exists || access_key_exists) {
Expand All @@ -405,7 +405,7 @@ async function add_account(data) {
// for validating against the schema we need an object, hence we parse it back to object
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await config_fs.create_account_config_file(data.name, account, true);
await config_fs.create_account_config_file(account);
write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg });
}

Expand Down Expand Up @@ -434,7 +434,7 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
// for validating against the schema we need an object, hence we parse it back to object
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await config_fs.update_account_config_file(data.name, account, undefined, undefined);
await config_fs.update_account_config_file(account);
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
return;
}
Expand All @@ -446,9 +446,9 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
secret_key: data.access_keys[0].secret_key,
};

const name_exists = update_name && await config_fs.is_account_exists({ name: data.name });
const name_exists = update_name && await config_fs.is_account_exists_by_name(data.name, undefined);
const access_key_exists = update_access_key &&
await config_fs.is_account_exists({ access_key: data.access_keys[0].access_key.unwrap() });
await config_fs.is_account_exists_by_access_key(data.access_keys[0].access_key.unwrap());

if (name_exists || access_key_exists) {
const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
Expand All @@ -465,18 +465,17 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
// for validating against the schema we need an object, hence we parse it back to object
const parsed_data = JSON.parse(encrypted_data);
nsfs_schema_utils.validate_account_schema(parsed_data);
if (update_name) {
await config_fs.create_account_config_file(new_name, parsed_data, true, [cur_access_key]);
await config_fs.delete_account_config_file(cur_name, data.access_keys);
} else if (update_access_key) {
await config_fs.update_account_config_file(cur_name, parsed_data, parsed_data.access_keys, [cur_access_key]);
}
await config_fs.update_account_config_file(parsed_data, {
old_name: update_name && cur_name,
new_access_keys_to_link: update_access_key && parsed_data.access_keys,
access_keys_to_delete: update_access_key && [{ access_key: cur_access_key }]
});
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
}

async function delete_account(data) {
await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.DELETE, undefined);
await config_fs.delete_account_config_file(data.name, data.access_keys);
await config_fs.delete_account_config_file(data);
write_stdout_response(ManageCLIResponse.AccountDeleted, '', { account: data.name });
}

Expand Down Expand Up @@ -578,19 +577,10 @@ function filter_bucket(bucket, filters) {
* @param {object} [filters]
*/
async function list_config_files(type, wide, show_secrets, filters = {}) {
let entries;
// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
let entries = [];
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;
if (is_filter_by_name) {
entries = [{'name': filters.name + JSON_SUFFIX}];
} else {
entries = type === TYPES.ACCOUNT ?
await config_fs.list_root_accounts() :
await config_fs.list_buckets();
}

const should_filter = Object.keys(filters).length > 0;
// decryption causing mkm initalization
// decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true)
const options = {
Expand All @@ -599,19 +589,27 @@ async function list_config_files(type, wide, show_secrets, filters = {}) {
silent_if_missing: true
};

// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entries = [filters.name];
} else if (type === TYPES.ACCOUNT) {
entries = await config_fs.list_accounts();
} else if (type === TYPES.BUCKET) {
entries = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entries, async entry => {
if (entry.name.endsWith(JSON_SUFFIX)) {
if (wide || should_filter) {
const data = type === TYPES.ACCOUNT ?
await config_fs.get_account_by_name(entry.name, options) :
await config_fs.get_bucket_by_name(entry.name, options);
if (!data) return undefined;
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) };
} else {
return { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) };
}
if (wide || should_filter) {
const data = type === TYPES.ACCOUNT ?
await config_fs.get_account_by_name(entry, options) :
await config_fs.get_bucket_by_name(entry, options);
if (!data) return undefined;
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry };
} else {
return { name: entry };
}
});
// it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
Expand Down
5 changes: 0 additions & 5 deletions src/deploy/NVA_build/standalone_deploy_nsfs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,8 @@ function execute() {

# Please note that the command we use here are without "sudo" because we are running from the container with Root permissions
function main() {
# Add accounts to run ceph tests
execute "node src/cmd/manage_nsfs account add --name cephalt --new_buckets_path ${FS_ROOT_1} --uid 1000 --gid 1000" nsfs_cephalt.log
execute "node src/cmd/manage_nsfs account add --name cephtenant --new_buckets_path ${FS_ROOT_2} --uid 2000 --gid 2000" nsfs_cephtenant.log

# Start noobaa service
execute "node src/cmd/nsfs" nsfs.log

# Wait for sometime to process to start
sleep 10
}
Expand Down
20 changes: 10 additions & 10 deletions src/manage_nsfs/health.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ const nb_native = require('../util/nb_native');
const native_fs_utils = require('../util/native_fs_utils');
const { read_stream_join } = require('../util/buffer_utils');
const { make_https_request } = require('../util/http_utils');
const { JSON_SUFFIX } = require('../sdk/config_fs');
const { TYPES } = require('./manage_nsfs_constants');
const { get_boolean_or_string_value, throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils');
const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
Expand Down Expand Up @@ -327,7 +326,7 @@ class NSFSHealth {
config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
} else if (type === TYPES.ACCOUNT) {
// TODO - handle iam accounts when directory structure changes - read_account_by_id
config_dir_path = this.config_fs.accounts_dir_path;
config_dir_path = this.config_fs.accounts_by_name_dir_path;
config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
}
// TODO - this is not a good handling for that - we need to take it to an upper level
Expand All @@ -339,15 +338,16 @@ class NSFSHealth {
};
}

const entries = type === TYPES.BUCKET ?
await this.config_fs.list_buckets() :
await this.config_fs.list_root_accounts();

const config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(JSON_SUFFIX));
let config_files;
if (type === TYPES.BUCKET) {
config_files = await this.config_fs.list_buckets();
} else {
config_files = await this.config_fs.list_accounts();
}
for (const config_file of config_files) {
// config_file get data or push error
const { config_data = undefined, err_obj = undefined } =
await this.get_config_file_data_or_error_object(type, config_file.name);
await this.get_config_file_data_or_error_object(type, config_file);
if (!config_data && err_obj) {
invalid_storages.push(err_obj.invalid_storage);
continue;
Expand Down Expand Up @@ -395,9 +395,9 @@ class NSFSHealth {
} catch (err) {
let err_code;
const config_file_path = type === TYPES.BUCKET ?
await this.config_fs.get_bucket_path_by_name(config_file_name) :
this.config_fs.get_bucket_path_by_name(config_file_name) :
// TODO - should be changed to id when moving to new structure for supporting iam accounts
await this.config_fs.get_account_path_by_name(config_file_name);
this.config_fs.get_account_path_by_name(config_file_name);

if (err.code === 'ENOENT') {
dbg.log1(`Error: Config file path should be a valid path`, config_file_path, err);
Expand Down
49 changes: 20 additions & 29 deletions src/manage_nsfs/manage_nsfs_validations.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,9 @@

const config = require('../../config');
const dbg = require('../util/debug_module')(__filename);
const path = require('path');
const net = require('net');
const P = require('../util/promise');
const nb_native = require('../util/nb_native');
const string_utils = require('../util/string_utils');
const { JSON_SUFFIX } = require('../sdk/config_fs');
const native_fs_utils = require('../util/native_fs_utils');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
Expand Down Expand Up @@ -354,7 +351,7 @@ async function validate_bucket_args(config_fs, data, action) {
if (data.s3_policy) {
try {
await bucket_policy_utils.validate_s3_policy(data.s3_policy, data.name,
async principal => config_fs.is_account_exists({ name: principal })
async principal => config_fs.is_account_exists_by_name(principal, account.owner)
);
} catch (err) {
dbg.error('validate_bucket_args invalid bucket policy err:', err);
Expand Down Expand Up @@ -482,16 +479,14 @@ function _validate_access_keys(access_key, secret_key) {
* @param {string} account_name
*/
async function validate_account_not_owns_buckets(config_fs, account_name) {
const entries = await config_fs.list_buckets();
await P.map_with_concurrency(10, entries, async entry => {
if (entry.name.endsWith(JSON_SUFFIX)) {
const data = await config_fs.get_bucket_by_name(entry.name, { silent_if_missing: true });
if (data && data.bucket_owner === account_name) {
const detail_msg = `Account ${account_name} has bucket ${data.name}`;
throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasBuckets, detail_msg);
}
return data;
const bucket_names = await config_fs.list_buckets();
await P.map_with_concurrency(10, bucket_names, async bucket_name => {
const data = await config_fs.get_bucket_by_name(bucket_name, { silent_if_missing: true });
if (data && data.bucket_owner === account_name) {
const detail_msg = `Account ${account_name} has bucket ${data.name}`;
throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasBuckets, detail_msg);
}
return data;
});
}

Expand All @@ -503,24 +498,20 @@ async function validate_account_not_owns_buckets(config_fs, account_name) {
* @param {string} action
*/
async function check_if_root_account_does_not_have_IAM_users(config_fs, account_to_check, action) {
const fs_context = config_fs.fs_context;
const entries = await nb_native().fs.readdir(fs_context, config_fs.accounts_dir_path);
await P.map_with_concurrency(10, entries, async entry => {
if (entry.name.endsWith(JSON_SUFFIX)) {
const full_path = path.join(config_fs.accounts_dir_path, entry.name);
const account_data = await config_fs.get_config_data(full_path);
if (entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME)) return undefined;
const is_root_account_owns_user = check_root_account_owns_user(account_to_check, account_data);
if (is_root_account_owns_user) {
const detail_msg = `Account ${account_to_check.name} has IAM account ${account_data.name}`;
if (action === ACTIONS.DELETE) {
throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasIAMAccounts, detail_msg);
}
// else it is called with action ACTIONS.UPDATE
throw_cli_error(ManageCLIError.AccountCannotBeRootAccountsManager, detail_msg);
// TODO - For supporting IAM, we need to check if {config_dir}/identities/{account_id}/users/ has anything inside
const account_names = await config_fs.list_accounts();
await P.map_with_concurrency(10, account_names, async account_name => {
const account_data = await config_fs.get_account_by_name(account_name);
const is_root_account_owns_user = check_root_account_owns_user(account_to_check, account_data);
if (is_root_account_owns_user) {
const detail_msg = `Account ${account_to_check.name} has IAM account ${account_data.name}`;
if (action === ACTIONS.DELETE) {
throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasIAMAccounts, detail_msg);
}
return account_data;
// else it is called with action ACTIONS.UPDATE
throw_cli_error(ManageCLIError.AccountCannotBeRootAccountsManager, detail_msg);
}
return account_data;
});
}

Expand Down
Loading