Skip to content

NC | Lifecycle | backports to 5.18 #8882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Mar 24, 2025
24 changes: 24 additions & 0 deletions config.js
Original file line number Diff line number Diff line change
Expand Up @@ -998,6 +998,30 @@ config.NC_DISABLE_HEALTH_ACCESS_CHECK = false;
config.NC_DISABLE_POSIX_MODE_ACCESS_CHECK = true;
config.NC_DISABLE_SCHEMA_CHECK = false;

////////// NC LIFECYLE //////////

config.NC_LIFECYCLE_LOGS_DIR = '/var/log/noobaa/lifecycle';
config.NC_LIFECYCLE_CONFIG_DIR_NAME = 'lifecycle';
config.NC_LIFECYCLE_TIMEOUT_MS = 8 * 60 * 60 * 1000;

// NC_LIFECYCLE_RUN_TIME must be of the format hh:mm which specifies
// when NooBaa should allow running nc lifecycle process
// NOTE: This will also be in the same timezone as specified in
// NC_LIFECYCLE_TZ
config.NC_LIFECYCLE_RUN_TIME = '00:00';

// NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS configures the delay
// tolerance in minutes.
//
// eg. If the expiry run time is set to 00:00 and the tolerance is
// set to be 2 mins then the expiry can trigger till 00:02 (unless
// already triggered between 00:00 - 00:02
config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = 2;

/** @type {'UTC' | 'LOCAL'} */
config.NC_LIFECYCLE_TZ = 'LOCAL';

config.NC_LIFECYCLE_GPFS_ILM_ENABLED = false;
////////// GPFS //////////
config.GPFS_DOWN_DELAY = 1000;

Expand Down
48 changes: 44 additions & 4 deletions src/cmd/manage_nsfs.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,18 @@
/* Copyright (C) 2020 NooBaa */
'use strict';

// DO NOT PUT NEW REQUIREMENTS BEFORE SETTING process.env.NC_NSFS_NO_DB_ENV = 'true'
// NC nsfs deployments specifying process.env.LOCAL_MD_SERVER=true deployed together with a db
// when a system_store object is initialized VaccumAnalyzer is being called once a day.
// when NC nsfs deployed without db we would like to avoid running VaccumAnalyzer in any flow there is
// because running it will cause a panic.
if (process.env.LOCAL_MD_SERVER !== 'true') {
process.env.NC_NSFS_NO_DB_ENV = 'true';
}

const dbg = require('../util/debug_module')(__filename);
if (!dbg.get_process_name()) dbg.set_process_name('noobaa-cli');

const _ = require('lodash');
const path = require('path');
const minimist = require('minimist');
Expand All @@ -17,6 +28,7 @@ const { account_id_cache } = require('../sdk/accountspace_fs');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse;
const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier');
const noobaa_cli_lifecycle = require('../manage_nsfs/nc_lifecycle');
const manage_nsfs_logging = require('../manage_nsfs/manage_nsfs_logging');
const noobaa_cli_diagnose = require('../manage_nsfs/diagnose');
const noobaa_cli_upgrade = require('../manage_nsfs/upgrade');
Expand Down Expand Up @@ -76,6 +88,8 @@ async function main(argv = minimist(process.argv.slice(2))) {
await notification_management();
} else if (type === TYPES.CONNECTION) {
await connection_management(action, user_input);
} else if (type === TYPES.LIFECYCLE) {
await lifecycle_management(argv);
} else {
throw_cli_error(ManageCLIError.InvalidType);
}
Expand Down Expand Up @@ -252,7 +266,7 @@ async function delete_bucket(data, force) {
}
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
await config_fs.delete_bucket_config_file(data.name);
return { code: ManageCLIResponse.BucketDeleted, detail: '', event_arg: { bucket: data.name } };
return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
} catch (err) {
if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
throw err;
Expand Down Expand Up @@ -461,7 +475,7 @@ async function update_account(data) {
*/
async function delete_account(data) {
await config_fs.delete_account_config_file(data);
return { code: ManageCLIResponse.AccountDeleted, detail: '', event_arg: { account: data.name } };
return { code: ManageCLIResponse.AccountDeleted, detail: { name: data.name }, event_arg: { account: data.name } };
}

/**
Expand Down Expand Up @@ -754,12 +768,12 @@ async function connection_management(action, user_input) {
break;
case ACTIONS.DELETE:
await config_fs.delete_connection_config_file(user_input.name);
response = { code: ManageCLIResponse.ConnectionDeleted };
response = { code: ManageCLIResponse.ConnectionDeleted, detail: {name: user_input.name} };
break;
case ACTIONS.UPDATE:
await notifications_util.update_connect_file(user_input.name, user_input.key,
user_input.value, user_input.remove_key, config_fs);
response = { code: ManageCLIResponse.ConnectionUpdated };
response = { code: ManageCLIResponse.ConnectionUpdated, detail: {name: user_input.name} };
break;
case ACTIONS.STATUS:
data = await new notifications_util.Notificator({
Expand All @@ -780,5 +794,31 @@ async function connection_management(action, user_input) {
write_stdout_response(response.code, response.detail, response.event_arg);
}

////////////////////
///// LIFECYCLE ////
////////////////////

/**
* lifecycle_management runs the nc lifecycle management
* @returns {Promise<void>}
*/
async function lifecycle_management(args) {
const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
const short_status = get_boolean_or_string_value(args.short_status);
try {
const options = { disable_service_validation, disable_runtime_validation, short_status };
const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options);
if (should_run) {
write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);
} else {
write_stdout_response(ManageCLIResponse.LifecycleWorkerNotRunning);
}
} catch (err) {
dbg.error('manage_nsfs.lifecycle_management: Error while running run_lifecycle_under_lock', config_fs.config_json_path, err);
throw_cli_error(err);
}
}

exports.main = main;
if (require.main === module) main();
96 changes: 1 addition & 95 deletions src/cmd/nsfs.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,14 @@ if (process.env.LOCAL_MD_SERVER === 'true') {
//const js_utils = require('../util/js_utils');
const nb_native = require('../util/nb_native');
//const schema_utils = require('../util/schema_utils');
const RpcError = require('../rpc/rpc_error');
const ObjectSDK = require('../sdk/object_sdk');
const { cluster } = require('../util/fork_utils');
const NamespaceFS = require('../sdk/namespace_fs');
const BucketSpaceSimpleFS = require('../sdk/bucketspace_simple_fs');
const BucketSpaceFS = require('../sdk/bucketspace_fs');
const SensitiveString = require('../util/sensitive_string');
const endpoint_stats_collector = require('../sdk/endpoint_stats_collector');
//const { RPC_BUFFERS } = require('../rpc');
const AccountSDK = require('../sdk/account_sdk');
const NsfsObjectSDK = require('../sdk/nsfs_object_sdk');
const AccountSpaceFS = require('../sdk/accountspace_fs');
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
const { set_debug_level } = require('../manage_nsfs/manage_nsfs_cli_utils');
Expand Down Expand Up @@ -121,98 +119,6 @@ function print_usage() {

let nsfs_config_root;

class NsfsObjectSDK extends ObjectSDK {
constructor(fs_root, fs_config, account, versioning, config_root, nsfs_system) {
// const rpc_client_hooks = new_rpc_client_hooks();
// rpc_client_hooks.account.read_account_by_access_key = async ({ access_key }) => {
// if (access_key) {
// return { access_key };
// }
// };
// rpc_client_hooks.bucket.read_bucket_sdk_info = async ({ name }) => {
// if (name) {
// return { name };
// }
// };
let bucketspace;
if (config_root) {
bucketspace = new BucketSpaceFS({ config_root }, endpoint_stats_collector.instance());
} else {
bucketspace = new BucketSpaceSimpleFS({ fs_root });
}
super({
rpc_client: null,
internal_rpc_client: null,
object_io: null,
bucketspace,
stats: endpoint_stats_collector.instance(),
});
this.nsfs_config_root = nsfs_config_root;
this.nsfs_fs_root = fs_root;
this.nsfs_fs_config = fs_config;
this.nsfs_account = account;
this.nsfs_versioning = versioning;
this.nsfs_namespaces = {};
this.nsfs_system = nsfs_system;
if (!config_root) {
this._get_bucket_namespace = bucket_name => this._simple_get_single_bucket_namespace(bucket_name);
this.load_requesting_account = auth_req => this._simple_load_requesting_account(auth_req);
this.read_bucket_sdk_policy_info = bucket_name => this._simple_read_bucket_sdk_policy_info(bucket_name);
this.read_bucket_sdk_config_info = () => undefined;
this.read_bucket_usage_info = () => undefined;
this.read_bucket_sdk_website_info = () => undefined;
this.read_bucket_sdk_namespace_info = () => undefined;
this.read_bucket_sdk_caching_info = () => undefined;
}
}

async _simple_get_single_bucket_namespace(bucket_name) {
const existing_ns = this.nsfs_namespaces[bucket_name];
if (existing_ns) return existing_ns;
const ns_fs = new NamespaceFS({
fs_backend: this.nsfs_fs_config.backend,
bucket_path: this.nsfs_fs_root + '/' + bucket_name,
bucket_id: 'nsfs',
namespace_resource_id: undefined,
access_mode: undefined,
versioning: this.nsfs_versioning,
stats: endpoint_stats_collector.instance(),
force_md5_etag: false,
});
this.nsfs_namespaces[bucket_name] = ns_fs;
return ns_fs;
}

async _simple_load_requesting_account(auth_req) {
const access_key = this.nsfs_account.access_keys?.[0]?.access_key;
if (access_key) {
const token = this.get_auth_token();
if (!token) {
throw new RpcError('UNAUTHORIZED', `Anonymous access to bucket not allowed`);
}
if (token.access_key !== access_key.unwrap()) {
throw new RpcError('INVALID_ACCESS_KEY_ID', `Account with access_key not found`);
}
}
this.requesting_account = this.nsfs_account;
}

async _simple_read_bucket_sdk_policy_info(bucket_name) {
return {
s3_policy: {
Version: '2012-10-17',
Statement: [{
Effect: 'Allow',
Action: ['*'],
Resource: ['*'],
Principal: [new SensitiveString('*')],
}]
},
bucket_owner: new SensitiveString('nsfs'),
owner_account: new SensitiveString('nsfs-id'), // temp
};
}
}

// NsfsAccountSDK was based on NsfsObjectSDK
// simple flow was not implemented
Expand Down
24 changes: 7 additions & 17 deletions src/manage_nsfs/health.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ const native_fs_utils = require('../util/native_fs_utils');
const { read_stream_join } = require('../util/buffer_utils');
const { make_https_request } = require('../util/http_utils');
const { TYPES } = require('./manage_nsfs_constants');
const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils');
const { get_boolean_or_string_value, throw_cli_error, write_stdout_response,
get_bucket_owner_account_by_id, get_service_status, NOOBAA_SERVICE_NAME } = require('./manage_nsfs_cli_utils');
const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError;


const HOSTNAME = 'localhost';
const NOOBAA_SERVICE = 'noobaa';

const health_errors = {
NOOBAA_SERVICE_FAILED: {
Expand Down Expand Up @@ -116,7 +116,7 @@ class NSFSHealth {
async nc_nsfs_health() {
let endpoint_state;
let memory;
const noobaa_service_state = await this.get_service_state(NOOBAA_SERVICE);
const noobaa_service_state = await this.get_service_state(NOOBAA_SERVICE_NAME);
const { service_status, pid } = noobaa_service_state;
if (pid !== '0') {
endpoint_state = await this.get_endpoint_response();
Expand All @@ -135,7 +135,7 @@ class NSFSHealth {
if (this.all_bucket_details) bucket_details = await this.get_bucket_status();
if (this.all_account_details) account_details = await this.get_account_status();
const health = {
service_name: NOOBAA_SERVICE,
service_name: NOOBAA_SERVICE_NAME,
status: service_health,
memory: memory,
error: error_code,
Expand Down Expand Up @@ -204,18 +204,8 @@ class NSFSHealth {
}

async get_service_state(service_name) {
let service_status;
let pid;
try {
service_status = await os_util.exec('systemctl show -p ActiveState --value ' + service_name, {
ignore_rc: false,
return_stdout: true,
trim_stdout: true,
});
} catch (err) {
dbg.warn('could not receive service active state', service_name, err);
service_status = 'missing service status info';
}
const service_status = await get_service_status(service_name);
try {
pid = await os_util.exec('systemctl show --property MainPID --value ' + service_name, {
ignore_rc: false,
Expand Down Expand Up @@ -302,13 +292,13 @@ class NSFSHealth {
async get_service_memory_usage() {
let memory_status;
try {
memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE + ' | grep Memory ', {
memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE_NAME + ' | grep Memory ', {
ignore_rc: false,
return_stdout: true,
trim_stdout: true,
});
} catch (err) {
dbg.warn('could not receive service active state', NOOBAA_SERVICE, err);
dbg.warn('could not receive service active state', NOOBAA_SERVICE_NAME, err);
memory_status = 'Memory: missing memory info';
}
if (memory_status) {
Expand Down
35 changes: 33 additions & 2 deletions src/manage_nsfs/manage_nsfs_cli_errors.js
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,34 @@ ManageCLIError.MissingCliParam = Object.freeze({
http_code: 400,
});

//////////////////////////////
// LIFECYCLE ERRORS //
//////////////////////////////

ManageCLIError.SystemJsonIsMissing = Object.freeze({
code: 'SystemJsonIsMissing',
message: 'Lifecycle worker can not run when system.json is missing.',
http_code: 400,
});

ManageCLIError.NooBaaServiceIsNotActive = Object.freeze({
code: 'NooBaaServiceIsNotActive',
message: 'Lifecycle worker can not run when NooBaa service is not active.',
http_code: 400,
});

ManageCLIError.LifecycleFailed = Object.freeze({
code: 'LifecycleFailed',
message: 'Lifecycle worker run failed.',
http_code: 400,
});

ManageCLIError.LifecycleWorkerReachedTimeout = Object.freeze({
code: 'LifecycleWorkerReachedTimeout',
message: `Lifecycle worker reached timeout - configured timeout is ${config.NC_LIFECYCLE_TIMEOUT_MS} ms`,
http_code: 400,
});

///////////////////////////////
// ERRORS MAPPING //
///////////////////////////////
Expand All @@ -522,7 +550,8 @@ ManageCLIError.RPC_ERROR_TO_MANAGE = Object.freeze({
NO_SUCH_USER: ManageCLIError.InvalidAccountDistinguishedName,
INVALID_MASTER_KEY: ManageCLIError.InvalidMasterKey,
INVALID_BUCKET_NAME: ManageCLIError.InvalidBucketName,
CONFIG_DIR_VERSION_MISMATCH: ManageCLIError.ConfigDirUpdateBlocked
CONFIG_DIR_VERSION_MISMATCH: ManageCLIError.ConfigDirUpdateBlocked,
LIFECYCLE_WORKER_TIMEOUT: ManageCLIError.LifecycleWorkerReachedTimeout
});

const NSFS_CLI_ERROR_EVENT_MAP = {
Expand All @@ -535,7 +564,9 @@ const NSFS_CLI_ERROR_EVENT_MAP = {
BucketSetForbiddenBucketOwnerNotExists: NoobaaEvent.UNAUTHORIZED, // GAP - add event
BucketSetForbiddenBucketOwnerIsIAMAccount: NoobaaEvent.UNAUTHORIZED, // // GAP - add event
LoggingExportFailed: NoobaaEvent.LOGGING_FAILED,
UpgradeFailed: NoobaaEvent.CONFIG_DIR_UPGRADE_FAILED
UpgradeFailed: NoobaaEvent.CONFIG_DIR_UPGRADE_FAILED,
LifecycleFailed: NoobaaEvent.LIFECYCLE_FAILED,
LifecycleWorkerReachedTimeout: NoobaaEvent.LIFECYCLE_TIMEOUT
};

exports.ManageCLIError = ManageCLIError;
Expand Down
Loading