From 4247c09b32479ed347f401ca8d35284b38ec8cd7 Mon Sep 17 00:00:00 2001 From: naveenpaul1 Date: Wed, 4 Jun 2025 16:55:54 +0530 Subject: [PATCH] Code Refactor | Remove Lambda/func_services Signed-off-by: naveenpaul1 --- src/agent/agent.js | 15 +- src/agent/func_services/func_node.js | 150 ------ src/agent/func_services/func_proc.js | 105 ---- src/api/api.js | 2 - src/api/bucket_api.js | 142 ----- src/api/events_api.js | 12 - src/api/func_api.js | 472 ---------------- src/api/func_node_api.js | 99 ---- src/api/object_api.js | 20 - src/api/system_api.js | 6 - src/endpoint/endpoint.js | 9 +- src/endpoint/lambda/lambda_errors.js | 132 ----- src/endpoint/lambda/lambda_rest.js | 204 ------- src/endpoint/lambda/lambda_utils.js | 24 - src/endpoint/lambda/ops/lambda_create_func.js | 42 -- src/endpoint/lambda/ops/lambda_delete_func.js | 19 - src/endpoint/lambda/ops/lambda_get_func.js | 29 - src/endpoint/lambda/ops/lambda_invoke_func.js | 25 - src/endpoint/lambda/ops/lambda_list_funcs.js | 22 - src/sdk/func_sdk.js | 37 -- src/sdk/namespace_cache.js | 86 --- src/sdk/namespace_merge.js | 95 +--- src/sdk/namespace_nb.js | 112 ---- src/sdk/nb.d.ts | 1 - src/sdk/object_sdk.js | 15 - src/server/func_services/func_indexes.js | 14 - src/server/func_services/func_schema.js | 84 --- src/server/func_services/func_server.js | 446 --------------- .../func_services/func_stats_indexes.js | 13 - src/server/func_services/func_stats_schema.js | 37 -- src/server/func_services/func_stats_store.js | 83 --- src/server/func_services/func_store.js | 108 ---- src/server/md_server.js | 1 - src/server/notifications/dispatcher.js | 235 ++++---- src/server/notifications/event_server.js | 4 +- .../object_services/events_dispatcher.js | 99 ---- src/server/object_services/object_server.js | 10 - src/server/server_rpc.js | 8 - src/server/system_services/account_server.js | 7 - src/server/system_services/bucket_server.js | 155 +----- src/server/system_services/pool_server.js | 8 - .../system_services/schemas/bucket_schema.js | 37 -- src/server/system_services/system_server.js | 13 - src/server/web_server.js | 1 - src/test/framework/system_tests_list.js | 8 - src/test/lambda/create_backup_file_func.js | 39 -- src/test/lambda/delete_backup_file_func.js | 18 - src/test/lambda/denial_of_service_func.js | 40 -- src/test/lambda/dos_event_sample.json | 8 - src/test/lambda/test_lambda.sh | 37 -- src/test/lambda/word_count_func.js | 69 --- src/test/system_tests/mongodb_defaults.js | 1 - src/test/system_tests/sanity_build_test.js | 37 -- .../test_bucket_lambda_triggers.js | 509 ------------------ src/test/unit_tests/coretest.js | 1 - src/test/unit_tests/test_namespace_cache.js | 1 - src/test/unit_tests/test_system_servers.js | 46 -- src/tools/mongodb_bucket_blow.js | 1 - src/upgrade/migration_to_postgres.js | 26 - 59 files changed, 115 insertions(+), 3964 deletions(-) delete mode 100644 src/agent/func_services/func_node.js delete mode 100644 src/agent/func_services/func_proc.js delete mode 100644 src/api/func_api.js delete mode 100644 src/api/func_node_api.js delete mode 100644 src/endpoint/lambda/lambda_errors.js delete mode 100644 src/endpoint/lambda/lambda_rest.js delete mode 100644 src/endpoint/lambda/lambda_utils.js delete mode 100644 src/endpoint/lambda/ops/lambda_create_func.js delete mode 100644 src/endpoint/lambda/ops/lambda_delete_func.js delete mode 100644 src/endpoint/lambda/ops/lambda_get_func.js delete mode 100644 src/endpoint/lambda/ops/lambda_invoke_func.js delete mode 100644 src/endpoint/lambda/ops/lambda_list_funcs.js delete mode 100644 src/sdk/func_sdk.js delete mode 100644 src/server/func_services/func_indexes.js delete mode 100644 src/server/func_services/func_schema.js delete mode 100644 src/server/func_services/func_server.js delete mode 100644 src/server/func_services/func_stats_indexes.js delete mode 100644 src/server/func_services/func_stats_schema.js delete mode 100644 src/server/func_services/func_stats_store.js delete mode 100644 src/server/func_services/func_store.js delete mode 100644 src/server/object_services/events_dispatcher.js delete mode 100644 src/test/lambda/create_backup_file_func.js delete mode 100644 src/test/lambda/delete_backup_file_func.js delete mode 100644 src/test/lambda/denial_of_service_func.js delete mode 100644 src/test/lambda/dos_event_sample.json delete mode 100755 src/test/lambda/test_lambda.sh delete mode 100644 src/test/lambda/word_count_func.js delete mode 100644 src/test/system_tests/test_bucket_lambda_triggers.js diff --git a/src/agent/agent.js b/src/agent/agent.js index b08a92d010..48c68103ce 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -17,7 +17,6 @@ const pkg = require('../../package.json'); const DebugLogger = require('../util/debug_module'); const diag = require('./agent_diagnostics'); const config = require('../../config'); -const FuncNode = require('./func_services/func_node'); const os_utils = require('../util/os_utils'); const js_utils = require('../util/js_utils'); const net_utils = require('../util/net_utils'); @@ -150,11 +149,6 @@ class Agent { this.block_store = new BlockStoreMem(block_store_options); } - this.func_node = new FuncNode({ - rpc_client: this.client, - storage_path: this.storage_path, - }); - // AGENT API methods - bind to self // (rpc registration requires bound functions) js_utils.self_bind(this, [ @@ -190,12 +184,6 @@ class Agent { } ); } - this.rpc.register_service( - this.rpc.schema.func_node_api, - this.func_node, { - middleware: [req => this._authenticate_agent_api(req)] - } - ); // register rpc n2n this.n2n_agent = this.rpc.register_n2n_agent((...args) => this.client.node.n2n_signal(...args)); @@ -602,8 +590,7 @@ class Agent { // agent_api requests allowed only on server connection if (!req.method_api.auth?.n2n && - req.api !== this.rpc.schema.block_store_api && - req.api !== this.rpc.schema.func_node_api + req.api !== this.rpc.schema.block_store_api ) { // delayed close the connection to give a chance to send the thrown error response setTimeout(() => req.connection.close(), 1000); diff --git a/src/agent/func_services/func_node.js b/src/agent/func_services/func_node.js deleted file mode 100644 index a70d42bace..0000000000 --- a/src/agent/func_services/func_node.js +++ /dev/null @@ -1,150 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const fs = require('fs'); -const path = require('path'); -const child_process = require('child_process'); - -const P = require('../../util/promise'); -const dbg = require('../../util/debug_module')(__filename); -const { RpcError, RPC_BUFFERS } = require('../../rpc'); -const fs_utils = require('../../util/fs_utils'); -const semaphore = require('../../util/semaphore'); -const zip_utils = require('../../util/zip_utils'); - -const FUNC_PROC_PATH = path.resolve(__dirname, 'func_proc.js'); -const FUNC_NODE_PATH = path.resolve(__dirname, '..', '..', '..', 'node_modules'); - -class FuncNode { - - constructor(params) { - this.rpc_client = params.rpc_client; - this.storage_path = params.storage_path || '.'; - this.functions_path = path.join(this.storage_path, 'functions'); - this.functions_loading_path = path.join(this.storage_path, 'functions_loading'); - this.loading_serial = new semaphore.Semaphore(1); - } - - async invoke_func(req) { - const func = await this._load_func_code(req); - let res; - try { - // const res = await new Promise((resolve, reject) => { - res = await new Promise((resolve, reject) => { - const proc = child_process.fork(FUNC_PROC_PATH, [], { - cwd: func.code_dir, - stdio: 'inherit', - // main node root modules library for the forked lambda function, so function can use modules (like aws-s3) - // from wherever located (func.code_dir) - env: { - NODE_PATH: FUNC_NODE_PATH, - container: process.env.container - } - }) - .once('error', reject) - .once('exit', code => resolve({ - error: { - message: `Func process exit unexpectedly (should use callback function) with code ${code}`, - code: String(code), - } - })) - .once('message', msg => { - dbg.log1('invoke_func: received message', msg); - if (msg.error) { - return resolve({ - error: { - message: msg.error.message || 'Unknown error from func process', - stack: msg.error.stack, - code: String(msg.error.code), - } - }); - } - return resolve({ - result: msg.result - }); - }); - const msg = { - config: req.params.config, - event: req.params.event, - aws_config: req.params.aws_config, - rpc_options: req.params.rpc_options, - AWS_EXECUTION_ENV: 'NOOBAA_FUNCTION' - }; - dbg.log1('invoke_func: send message', msg); - proc.send(msg); - }); - } catch (e) { - dbg.error('invoke_func:: got error:', e); - throw e; - } - return res; - } - - async _load_func_code(req) { - const name = req.params.config.name; - const version = req.params.config.version; - const code_sha256 = req.params.config.code_sha256; - const version_dir = path.join(this.functions_path, name, version); - const func_json_path = path.join(version_dir, 'func.json'); - // replacing the base64 encoded sha256 from using / to - in order to use as folder name - const code_dir = path.join(version_dir, code_sha256.replace(/\//g, '-')); - return this.loading_serial.surround(async () => { - let func; - try { - try { - await fs.promises.stat(code_dir); - const func_json_buf = await fs.promises.readFile(func_json_path, 'utf8'); - func = JSON.parse(func_json_buf); - //if we can't load the function from the code dir (or it is not exist) we will create the dir and put the code there - } catch (err) { - if (err.code !== 'ENOENT') throw err; - func = await this._write_func_into_dir(code_dir, name, version, code_sha256, version_dir, func_json_path, req); - } - func.code_dir = code_dir; - dbg.log1('_load_func_code: loaded', func.config, code_dir); - return func; - } catch (err) { - console.error('_load_func_code: FAILED', err.stack || err); - throw err; - } - }); - } - - async _write_func_into_dir(code_dir, name, version, code_sha256, version_dir, func_json_path, req) { - const loading_dir = path.join(this.functions_loading_path, Date.now().toString(36)); - dbg.log0('_load_func_code: loading', loading_dir, code_dir); - const func = await this.rpc_client.func.read_func({ - name, - version, - read_code: true - }, req.params.rpc_options); - if (code_sha256 !== func.config.code_sha256 || - req.params.config.code_size !== func.config.code_size) { - throw new RpcError('FUNC_CODE_MISMATCH', - `Function code does not match for ${func.name} version ${func.version} code_size ${func.config.code_size} code_sha256 ${func.config.code_sha256} - requested code_size ${req.params.config.code_size} code_sha256 ${req.params.config.code_sha256}`); - } - const zipfile = await zip_utils.unzip_from_buffer(func[RPC_BUFFERS].zipfile); - await zip_utils.unzip_to_dir(zipfile, loading_dir); - await fs_utils.create_fresh_path(version_dir); - await fs_utils.folder_delete(code_dir); - await fs.promises.writeFile( - func_json_path, - JSON.stringify(func)); - await P.retry({ - attempts: 3, - delay_ms: 500, - func: async () => { - try { - await fs.promises.rename(loading_dir, code_dir); - } catch (e) { - dbg.error('Got error when trying to place new function, will retry', e); - throw e; - } - } - }); - return func; - } -} - -module.exports = FuncNode; diff --git a/src/agent/func_services/func_proc.js b/src/agent/func_services/func_proc.js deleted file mode 100644 index 4d7975efd3..0000000000 --- a/src/agent/func_services/func_proc.js +++ /dev/null @@ -1,105 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - - -try { - process.on('uncaughtException', fail); - process.on('unhandledRejection', fail); - - const path = require('path'); - const AWS = require('aws-sdk'); - const https = require('https'); - - process.once('message', msg => { - - // console.log('func_proc: received message', msg); - - if (msg.AWS_EXECUTION_ENV) process.env.AWS_EXECUTION_ENV = msg.AWS_EXECUTION_ENV; - if (msg.aws_config) { - if (msg.aws_config.endpoint.startsWith('https:')) { - msg.aws_config.httpOptions = { - agent: new https.Agent({ rejectUnauthorized: false }) - }; - } - AWS.config.update(msg.aws_config); - } - - const handler_arg = msg.config.handler; - const handler_split = handler_arg.split('.', 2); - const module_name = handler_split[0] + '.js'; - const export_name = handler_split[1]; - // eslint-disable-next-line global-require - const module_exports = require(path.resolve(module_name)); - const handler = export_name ? - module_exports[export_name] : - module_exports; - - if (typeof(handler) !== 'function') { - fail(new Error(`Func handler not a function ${handler_arg}`)); - } - - // http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-context.html - const context = { - callbackWaitsForEmptyEventLoop: false, - functionName: '', - functionVersion: '', - invokedFunctionArn: '', - memoryLimitInMB: 0, - awsRequestId: '', - logGroupName: '', - logStreamName: '', - identity: null, - clientContext: null, - getRemainingTimeInMillis: () => 60 * 1000, // TODO calculate timeout - }; - - if (msg.rpc_options) { - const api = require('../../api'); // eslint-disable-line global-require - const rpc = api.new_rpc_from_base_address(msg.rpc_options.address, 'EXTERNAL'); - const client = rpc.new_client(); - client.options = msg.rpc_options; - context.rpc_client = client; - } - - const callback = (err, reply) => { - if (err) { - console.log('func_proc: callback', err); - if (context.callbackWaitsForEmptyEventLoop) { - process.on('beforeExit', () => fail(err)); - } else { - fail(err); - } - return; - } - // console.log('func_proc: callback reply', reply); - if (context.callbackWaitsForEmptyEventLoop) { - process.on('beforeExit', () => success(reply)); - } else { - success(reply); - } - }; - - handler(msg.event, context, callback); - }); - -} catch (err) { - fail(err); -} - -function fail(err) { - console.error('func_proc: fail', err); - process.send({ - error: { - message: err.message, - code: err.code, - stack: err.stack, - } - }, () => process.exit(1)); -} - -function success(result) { - // console.log('func_proc: success', result); - process.send({ - result: result - }, () => process.exit(1)); -} diff --git a/src/api/api.js b/src/api/api.js index e859283788..d372c31567 100644 --- a/src/api/api.js +++ b/src/api/api.js @@ -30,8 +30,6 @@ api_schema.register_api(require('./cluster_server_api')); api_schema.register_api(require('./cluster_internal_api')); api_schema.register_api(require('./server_inter_process_api')); api_schema.register_api(require('./hosted_agents_api')); -api_schema.register_api(require('./func_api')); -api_schema.register_api(require('./func_node_api')); api_schema.register_api(require('./replication_api')); api_schema.compile(); diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js index 5634313fc4..069b4a8e91 100644 --- a/src/api/bucket_api.js +++ b/src/api/bucket_api.js @@ -686,43 +686,6 @@ module.exports = { } }, - add_bucket_lambda_trigger: { - method: 'PUT', - params: { - $ref: '#/definitions/new_lambda_trigger' - }, - auth: { - system: 'admin' - } - }, - - delete_bucket_lambda_trigger: { - method: 'DELETE', - required: ['id', 'bucket_name'], - params: { - type: 'object', - properties: { - id: { - objectid: true - }, - bucket_name: { $ref: 'common_api#/definitions/bucket_name' }, - } - }, - auth: { - system: 'admin' - } - }, - - update_bucket_lambda_trigger: { - method: 'PUT', - params: { - $ref: '#/definitions/update_lambda_trigger' - }, - auth: { - system: 'admin' - } - }, - update_all_buckets_default_pool: { method: 'PUT', params: { @@ -1196,12 +1159,6 @@ module.exports = { undeletable: { $ref: '#/definitions/undeletable_bucket_reason' }, - triggers: { - type: 'array', - items: { - $ref: '#/definitions/lambda_trigger_info' - } - }, tagging: { $ref: 'common_api#/definitions/tagging' }, @@ -1388,105 +1345,6 @@ module.exports = { enum: ['NOT_EMPTY'], type: 'string', }, - - - new_lambda_trigger: { - type: 'object', - required: ['bucket_name', 'event_name', 'func_name'], - properties: { - bucket_name: { $ref: 'common_api#/definitions/bucket_name' }, - event_name: { - $ref: 'common_api#/definitions/bucket_trigger_event' - }, - func_name: { - type: 'string' - }, - func_version: { - type: 'string' - }, - enabled: { - type: 'boolean', - }, - object_prefix: { - type: 'string' - }, - object_suffix: { - type: 'string' - }, - attempts: { - type: 'integer' - }, - } - }, - - update_lambda_trigger: { - type: 'object', - required: ['id'], - properties: { - id: { - objectid: true - }, - bucket_name: { $ref: 'common_api#/definitions/bucket_name' }, - event_name: { - $ref: 'common_api#/definitions/bucket_trigger_event' - }, - func_name: { - type: 'string' - }, - func_version: { - type: 'string' - }, - enabled: { - type: 'boolean', - }, - object_prefix: { - type: 'string' - }, - object_suffix: { - type: 'string' - }, - attempts: { - type: 'integer' - }, - } - }, - - lambda_trigger_info: { - type: 'object', - required: ['id', 'event_name', 'func_name'], - properties: { - id: { - objectid: true - }, - event_name: { - $ref: 'common_api#/definitions/bucket_trigger_event' - }, - func_name: { - type: 'string' - }, - func_version: { - type: 'string' - }, - enabled: { - type: 'boolean', - }, - permission_problem: { - type: 'boolean', - }, - last_run: { - idate: true - }, - object_prefix: { - type: 'string' - }, - object_suffix: { - type: 'string' - }, - attempts: { - type: 'integer' - }, - } - }, object_lock_configuration: { type: 'object', properties: { diff --git a/src/api/events_api.js b/src/api/events_api.js index 8a9eb1c050..ec7f216641 100644 --- a/src/api/events_api.js +++ b/src/api/events_api.js @@ -147,18 +147,6 @@ module.exports = { } } }, - func: { - type: 'object', - required: ['name'], - properties: { - name: { - type: 'string' - }, - linkable: { - type: 'boolean' - }, - } - }, actor: { type: 'object', required: ['email'], diff --git a/src/api/func_api.js b/src/api/func_api.js deleted file mode 100644 index e2e9f12a0b..0000000000 --- a/src/api/func_api.js +++ /dev/null @@ -1,472 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -/** - * - * FUNC API - * - */ -module.exports = { - - $id: 'func_api', - - methods: { - - create_func: { - method: 'POST', - params: { - type: 'object', - required: ['config', 'code'], - properties: { - config: { - $ref: '#/definitions/func_config' - }, - code: { - $ref: '#/definitions/func_code' - }, - publish: { - type: 'boolean' - }, - } - }, - reply: { - $ref: '#/definitions/func_info' - }, - auth: { - system: 'admin' - } - }, - - update_func: { - method: 'PUT', - params: { - type: 'object', - required: ['config'], - properties: { - config: { - $ref: '#/definitions/func_config' - }, - code: { - $ref: '#/definitions/func_code' - }, - } - }, - reply: { - $ref: '#/definitions/func_info' - }, - auth: { - system: 'admin' - } - }, - - delete_func: { - method: 'PUT', - params: { - type: 'object', - required: ['name', 'version'], - properties: { - name: { - type: 'string' - }, - version: { - type: 'string' - }, - }, - }, - auth: { - system: 'admin' - } - }, - - read_func: { - method: 'GET', - params: { - type: 'object', - required: ['name', 'version'], - properties: { - name: { - type: 'string' - }, - version: { - type: 'string' - }, - read_code: { - type: 'boolean' - } - }, - }, - reply: { - type: 'object', - required: ['config', 'code_location'], - properties: { - config: { - $ref: '#/definitions/func_config' - }, - code_location: { - $ref: '#/definitions/func_code_location' - }, - code: { - $ref: '#/definitions/func_code' - } - } - }, - auth: { - system: ['admin', 'agent'] - } - }, - - read_func_stats: { - method: 'GET', - params: { - type: 'object', - required: [ - 'name', - 'version', - 'since', - 'step' - ], - properties: { - name: { - type: 'string' - }, - version: { - type: 'string' - }, - since: { - idate: true - }, - till: { - idate: true - }, - step: { - type: 'integer' - }, - percentiles: { - type: 'array', - items: { - type: 'number', - minimum: 0, - maximum: 1 - } - }, - max_samples: { - type: 'integer' - }, - } - }, - reply: { - type: 'object', - required: [ - 'stats', - 'slices' - ], - properties: { - stats: { - $ref: '#/definitions/func_stats' - }, - slices: { - type: 'array', - items: { - $ref: '#/definitions/func_stats' - } - } - } - }, - auth: { - system: 'admin' - } - }, - - list_funcs: { - method: 'GET', - reply: { - type: 'object', - properties: { - functions: { - type: 'array', - items: { - $ref: '#/definitions/func_info' - } - }, - }, - }, - auth: { - system: 'admin' - } - }, - - list_func_versions: { - method: 'GET', - params: { - type: 'object', - required: ['name'], - properties: { - name: { - type: 'string' - } - }, - }, - reply: { - type: 'object', - properties: { - versions: { - type: 'array', - items: { - $ref: '#/definitions/func_info' - } - }, - }, - }, - auth: { - system: 'admin' - } - }, - - invoke_func: { - method: 'PUT', - params: { - type: 'object', - required: ['name', 'version'], - properties: { - name: { - type: 'string' - }, - version: { - type: 'string' - }, - event: { - $ref: '#/definitions/event_type' - } - }, - }, - reply: { - type: 'object', - properties: { - result: { - $ref: '#/definitions/event_type' - }, - error: { - $ref: '#/definitions/error_type' - } - } - }, - auth: { - system: ['admin', 'agent'] - } - }, - - /* - allocate_func_maps: { - method: 'PUT', - params: { - type: 'object', - properties: { - name: { - type: 'string' - } - }, - }, - reply: { - $ref: '#/definitions/func_info' - }, - auth: { - system: 'admin' - } - }, - */ - - }, - - definitions: { - - func_config: { - type: 'object', - required: ['name', 'version'], - properties: { - name: { - type: 'string' - }, - version: { - type: 'string' - }, - exec_account: { $ref: 'common_api#/definitions/email' }, - description: { - type: 'string' - }, - role: { - type: 'string' - }, - handler: { - type: 'string' - }, - runtime: { - type: 'string' - }, - memory_size: { - type: 'integer' - }, - timeout: { - type: 'integer' - }, - pools: { - type: 'array', - items: { - type: 'string' - } - }, - // the following fields are not configurable, - // and will be returned as info - code_size: { - type: 'integer' - }, - code_sha256: { - type: 'string' - }, - last_modified: { - idate: true - }, - last_modifier: { $ref: 'common_api#/definitions/email' }, - resource_name: { - type: 'string' - }, - } - }, - - func_code: { - type: 'object', - properties: { - zipfile_b64: { - type: 'string' - }, - s3_bucket: { - type: 'string' - }, - s3_key: { - type: 'string' - }, - s3_obj_version: { - type: 'string' - }, - s3_endpoint: { - type: 'string' - }, - url: { - type: 'string' - }, - } - }, - - func_code_location: { - type: 'object', - required: ['url', 'repository'], - properties: { - url: { - type: 'string' - }, - repository: { - type: 'string' - }, - } - }, - - func_info: { - type: 'object', - required: ['config', 'code_location'], - properties: { - config: { - $ref: '#/definitions/func_config' - }, - code_location: { - $ref: '#/definitions/func_code_location' - }, - } - }, - - event_type: { - oneOf: [{ - type: 'object', - additionalProperties: true, - properties: {}, - }, { - type: 'string' - }] - }, - - error_type: { - type: 'object', - required: ['message'], - properties: { - message: { - type: 'string' - }, - stack: { - type: 'string' - }, - code: { - type: 'string' - }, - }, - }, - - func_stats: { - type: 'object', - required: [ - 'since', - 'till', - 'invoked', - 'fulfilled', - 'rejected', - 'aggr_response_time', - 'max_response_time', - 'avg_response_time', - 'response_percentiles' - ], - properties: { - since: { - idate: true - }, - till: { - idate: true - }, - invoked: { - type: 'integer' - }, - fulfilled: { - type: 'integer' - }, - rejected: { - type: 'integer' - }, - aggr_response_time: { - type: 'integer' - }, - max_response_time: { - type: 'integer' - }, - avg_response_time: { - type: 'integer' - }, - response_percentiles: { - type: 'array', - items: { - type: 'object', - required: [ - 'percentile', - 'value' - ], - properties: { - percentile: { - type: 'number', - minimum: 0, - maximum: 1 - }, - value: { - type: 'integer' - } - } - } - } - } - } - } -}; diff --git a/src/api/func_node_api.js b/src/api/func_node_api.js deleted file mode 100644 index 5946748d52..0000000000 --- a/src/api/func_node_api.js +++ /dev/null @@ -1,99 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -/** - * - * FUNC NODE API - * - */ -module.exports = { - - $id: 'func_node_api', - - methods: { - - invoke_func: { - method: 'PUT', - params: { - type: 'object', - required: ['config'], - properties: { - config: { - $ref: 'func_api#/definitions/func_config' - }, - event: { - $ref: 'func_api#/definitions/event_type' - }, - aws_config: { - $ref: '#/definitions/aws_config' - }, - rpc_options: { - $ref: '#/definitions/rpc_options' - }, - }, - }, - reply: { - type: 'object', - properties: { - result: { - $ref: 'func_api#/definitions/event_type' - }, - error: { - $ref: 'func_api#/definitions/error_type' - } - } - }, - auth: { - system: 'admin' - } - }, - - }, - - definitions: { - - aws_config: { - type: 'object', - properties: { - endpoint: { - type: 'string' - }, - region: { - type: 'string' - }, - sslEnabled: { - type: 'boolean' - }, - s3ForcePathStyle: { - type: 'boolean' - }, - accessKeyId: { - type: 'string' - }, - secretAccessKey: { - type: 'string' - }, - } - }, - - rpc_options: { - type: 'object', - properties: { - address: { - type: 'string' - }, - auth_token: { - oneOf: [{ - type: 'string' - }, { - type: 'object', - additionalProperties: true, - properties: {}, - }] - }, - } - } - - } - -}; diff --git a/src/api/object_api.js b/src/api/object_api.js index b3cdf14f38..ecc33e4924 100644 --- a/src/api/object_api.js +++ b/src/api/object_api.js @@ -610,26 +610,6 @@ module.exports = { auth: { system: ['admin', 'user'] } }, - dispatch_triggers: { - method: 'PUT', - params: { - type: 'object', - required: [ - 'bucket', - 'obj', - 'event_name' - ], - properties: { - bucket: { $ref: 'common_api#/definitions/bucket_name' }, - obj: { - $ref: '#/definitions/object_info' - }, - event_name: { $ref: 'common_api#/definitions/bucket_trigger_event' } - } - }, - auth: { system: ['admin', 'user'] } - }, - delete_object: { method: 'DELETE', params: { diff --git a/src/api/system_api.js b/src/api/system_api.js index 5d5185f324..c8b627b4f7 100644 --- a/src/api/system_api.js +++ b/src/api/system_api.js @@ -553,12 +553,6 @@ module.exports = { $ref: 'account_api#/definitions/account_info' } }, - functions: { - type: 'array', - items: { - $ref: 'func_api#/definitions/func_info' - } - }, objects: { type: 'integer' }, diff --git a/src/endpoint/endpoint.js b/src/endpoint/endpoint.js index 03520ba7fa..86812eac8c 100755 --- a/src/endpoint/endpoint.js +++ b/src/endpoint/endpoint.js @@ -20,9 +20,7 @@ const s3_rest = require('./s3/s3_rest'); const blob_rest = require('./blob/blob_rest'); const sts_rest = require('./sts/sts_rest'); const iam_rest = require('./iam/iam_rest'); -const lambda_rest = require('./lambda/lambda_rest'); const endpoint_utils = require('./endpoint_utils'); -const FuncSDK = require('../sdk/func_sdk'); const StsSDK = require('../sdk/sts_sdk'); const ObjectIO = require('../sdk/object_io'); const ObjectSDK = require('../sdk/object_sdk'); @@ -70,7 +68,6 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma /** * @typedef {import('http').IncomingMessage & { * object_sdk?: ObjectSDK; - * func_sdk?: FuncSDK; * sts_sdk?: StsSDK; * virtual_hosts?: readonly string[]; * bucket_logger?: PersistentLogger; @@ -290,7 +287,6 @@ async function start_endpoint_server_and_cert(server_type, init_request_sdk, opt function create_endpoint_handler(server_type, init_request_sdk, { virtual_hosts, bucket_logger, notification_logger }) { if (server_type === SERVICES_TYPES_ENUM.S3) { const blob_rest_handler = process.env.ENDPOINT_BLOB_ENABLED === 'true' ? blob_rest : unavailable_handler; - const lambda_rest_handler = config.DB_TYPE === 'mongodb' ? lambda_rest : unavailable_handler; /** @type {EndpointHandler} */ const s3_endpoint_request_handler = (req, res) => { @@ -300,9 +296,7 @@ function create_endpoint_handler(server_type, init_request_sdk, { virtual_hosts, if (bucket_logger) req.bucket_logger = bucket_logger; if (notification_logger) req.notification_logger = notification_logger; init_request_sdk(req, res); - if (req.url.startsWith('/2015-03-31/functions')) { - return lambda_rest_handler(req, res); - } else if (req.headers['x-ms-version']) { + if (req.headers['x-ms-version']) { return blob_rest_handler(req, res); } else if (req.url.startsWith('/total_fork_count')) { return fork_count_handler(req, res); @@ -417,7 +411,6 @@ function fork_count_handler(req, res) { function create_init_request_sdk(rpc, internal_rpc_client, object_io) { const init_request_sdk = (req, res) => { const rpc_client = rpc.new_client(); - req.func_sdk = new FuncSDK(rpc_client); req.sts_sdk = new StsSDK(rpc_client, internal_rpc_client); req.object_sdk = new ObjectSDK({ rpc_client, diff --git a/src/endpoint/lambda/lambda_errors.js b/src/endpoint/lambda/lambda_errors.js deleted file mode 100644 index 73a3332606..0000000000 --- a/src/endpoint/lambda/lambda_errors.js +++ /dev/null @@ -1,132 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -/** - * @typedef {{ - * code?: string, - * message: string, - * http_code: number, - * detail?: string - * }} LambdaErrorSpec - */ - -class LambdaError extends Error { - - /** - * @param {LambdaErrorSpec} error_spec - */ - constructor({ code, message, http_code, detail }) { - super(message); // sets this.message - this.code = code; - this.http_code = http_code; - this.detail = detail; - } - - reply() { - return JSON.stringify({ - Message: this.message - }); - } - -} - -LambdaError.CodeStorageExceededException = Object.freeze({ - code: 'CodeStorageExceededException', - message: 'You have exceeded your maximum total code size per account.', - http_code: 400, -}); -LambdaError.InvalidParameterValueException = Object.freeze({ - code: 'InvalidParameterValueException', - message: 'One of the parameters in the request is invalid.', - http_code: 400, -}); -LambdaError.ResourceConflictException = Object.freeze({ - code: 'ResourceConflictException', - message: 'The resource already exists.', - http_code: 409, -}); -LambdaError.ResourceNotFoundException = Object.freeze({ - code: 'ResourceNotFoundException', - message: 'The resource specified in the request does not exist.', - http_code: 404, -}); -LambdaError.RequestTooLargeException = Object.freeze({ - code: 'RequestTooLargeException', - message: 'The request payload exceeded the Invoke request body JSON input limit.', - http_code: 413, -}); -LambdaError.ServiceException = Object.freeze({ - code: 'ServiceException', - message: 'The AWS Lambda service encountered an internal error.', - http_code: 500, -}); -LambdaError.TooManyRequestsException = Object.freeze({ - code: 'TooManyRequestsException', - message: 'TooManyRequestsException.', - http_code: 429, -}); -LambdaError.UnsupportedMediaTypeException = Object.freeze({ - code: 'UnsupportedMediaTypeException', - message: 'The content type of the Invoke request body is not JSON.', - http_code: 415, -}); -LambdaError.RequestTimeTooSkewed = Object.freeze({ - code: 'RequestTimeTooSkewed', - message: 'The difference between the request time and the server\'s time is too large.', - http_code: 403, -}); -LambdaError.AccessDenied = Object.freeze({ - code: 'AccessDenied', - message: 'Access Denied', - http_code: 403, -}); -LambdaError.SignatureDoesNotMatch = Object.freeze({ - code: 'SignatureDoesNotMatch', - message: 'The request signature we calculated does not match the signature you provided. Check your AWS secret access key and signing method. For more information, see REST Authentication and SOAP Authentication for details.', - http_code: 403, -}); -LambdaError.MissingRequestBodyError = Object.freeze({ - code: 'MissingRequestBodyError', - message: 'Request body is empty.', - http_code: 400, -}); -LambdaError.InvalidRequest = Object.freeze({ - code: 'InvalidRequest', - message: 'The content type of the request is not JSON.', - http_code: 400, -}); -LambdaError.InvalidDigest = Object.freeze({ - code: 'InvalidDigest', - message: 'The Content-MD5 you specified is not valid.', - http_code: 400, -}); -LambdaError.MaxMessageLengthExceeded = Object.freeze({ - code: 'MaxMessageLengthExceeded', - message: 'Your request was too big.', - http_code: 400, -}); - -//////////////////////////////////////////////////////////////// -// Errors actually returned by AWS S3 although not documented // -//////////////////////////////////////////////////////////////// - -LambdaError.NotImplemented = Object.freeze({ - code: 'NotImplemented', - message: 'A header you provided implies functionality that is not implemented.', - http_code: 501, -}); -LambdaError.BadRequestWithoutCode = Object.freeze({ - // same as BadRequest but without encoding the field - // was needed for one of the cases in ceph/s3tests - message: 'Bad Request', - http_code: 400, -}); -LambdaError.XAmzContentSHA256Mismatch = Object.freeze({ - code: 'XAmzContentSHA256Mismatch', - message: 'The provided \'x-amz-content-sha256\' header does not match what was computed.', - http_code: 400, - // ClientComputedContentSHA256: '...', - // S3ComputedContentSHA256: '...', -}); - -exports.LambdaError = LambdaError; diff --git a/src/endpoint/lambda/lambda_rest.js b/src/endpoint/lambda/lambda_rest.js deleted file mode 100644 index 44e4cbcbdf..0000000000 --- a/src/endpoint/lambda/lambda_rest.js +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; -/* eslint-disable no-control-regex */ - -const _ = require('lodash'); - -const dbg = require('../../util/debug_module')(__filename); -const config = require('../../../config'); -const js_utils = require('../../util/js_utils'); -const time_utils = require('../../util/time_utils'); -const http_utils = require('../../util/http_utils'); -const LambdaError = require('./lambda_errors').LambdaError; -const signature_utils = require('../../util/signature_utils'); - -const LAMBDA_OPS = js_utils.deep_freeze({ - get_service: require('./ops/lambda_list_funcs'), - get_func: require('./ops/lambda_get_func'), - delete_func: require('./ops/lambda_delete_func'), - post_service: require('./ops/lambda_create_func'), - post_func_invocations: require('./ops/lambda_invoke_func'), -}); - -const LAMBDA_MAX_BODY_LEN = 4 * 1024 * 1024; - -const RPC_ERRORS_TO_LAMBDA = Object.freeze({ - UNAUTHORIZED: LambdaError.AccessDenied, - FORBIDDEN: LambdaError.AccessDenied, - NO_SUCH_FUNC: LambdaError.ResourceNotFoundException, - CONFLICT: LambdaError.ResourceConflictException, -}); - -const non_printable_regexp = /[\x00-\x1F]/; - -async function lambda_rest(req, res) { - try { - await handle_request(req, res); - } catch (err) { - handle_error(req, res, err); - } -} - -async function handle_request(req, res) { - // fill up standard amz response headers - res.setHeader('x-amz-request-id', req.request_id); - res.setHeader('x-amz-id-2', req.request_id); - - // note that browsers will not allow origin=* with credentials - // but anyway we allow it by the agent server. - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Credentials', true); - res.setHeader('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,OPTIONS'); - res.setHeader('Access-Control-Allow-Headers', - 'Content-Type,Content-MD5,Authorization,X-Amz-User-Agent,X-Amz-Date,ETag,X-Amz-Content-Sha256'); - res.setHeader('Access-Control-Expose-Headers', 'ETag'); - - if (req.method === 'OPTIONS') { - dbg.log0('OPTIONS!'); - res.statusCode = 200; - res.end(); - return; - } - - // setting default headers which might get overriden by api's that - // return actual data in the reply instead of json - res.setHeader('Content-Type', 'application/json'); - res.setHeader('ETag', '"1"'); - check_headers(req); - authenticate_request(req); - - // resolve the op to call - const op_name = parse_op_name(req); - dbg.log0('LAMBDA REQUEST', req.method, req.originalUrl, 'op', op_name, 'request_id', req.request_id, req.headers); - const op = LAMBDA_OPS[op_name]; - if (!op || !op.handler) { - dbg.error('LAMBDA TODO (NotImplemented)', op_name, req.method, req.originalUrl); - throw new LambdaError(LambdaError.NotImplemented); - } - - const options = { - body: op.body, - reply: op.reply, - MAX_BODY_LEN: LAMBDA_MAX_BODY_LEN, - XML_ROOT_ATTRS: {}, - ErrorClass: LambdaError, - error_max_body_len_exceeded: LambdaError.MaxMessageLengthExceeded, - error_missing_body: LambdaError.MissingRequestBodyError, - error_invalid_body: LambdaError.InvalidRequest, - error_body_sha256_mismatch: LambdaError.XAmzContentSHA256Mismatch, - }; - - await http_utils.read_and_parse_body(req, options); - const reply = await op.handler(req, res); - http_utils.send_reply(req, res, reply, options); -} - -function check_headers(req) { - _.each(req.headers, (val, key) => { - // test for non printable characters - // 403 is required for unreadable headers - // eslint-disable-next-line no-control-regex - if (non_printable_regexp.test(val) || non_printable_regexp.test(key)) { - dbg.warn('Invalid header characters', key, val); - if (key !== 'expect') { - throw new LambdaError(LambdaError.AccessDenied); - } - } - }); - - if (req.headers['content-length'] === '') { - throw new LambdaError(LambdaError.BadRequestWithoutCode); - } - - const content_md5_b64 = req.headers['content-md5']; - if (typeof content_md5_b64 === 'string') { - req.content_md5 = Buffer.from(content_md5_b64, 'base64'); - if (req.content_md5.length !== 16) { - throw new LambdaError(LambdaError.InvalidDigest); - } - } - - req.content_sha256_sig = req.headers['x-amz-content-sha256']; - if (typeof req.content_sha256_sig === 'string') { - req.content_sha256_buf = Buffer.from(req.content_sha256_sig, 'hex'); - if (req.content_sha256_buf.length !== 32) { - throw new LambdaError(LambdaError.InvalidDigest); - } - } - - const req_time = - time_utils.parse_amz_date(req.headers['x-amz-date'] || req.query['X-Amz-Date']) || - time_utils.parse_http_header_date(req.headers.date); - if (Math.abs(Date.now() - req_time) > config.AMZ_DATE_MAX_TIME_SKEW_MILLIS) { - throw new LambdaError(LambdaError.RequestTimeTooSkewed); - } -} - -function authenticate_request(req) { - try { - const auth_token = signature_utils.make_auth_token_from_request(req); - auth_token.client_ip = http_utils.parse_client_ip(req); - req.func_sdk.set_auth_token(auth_token); - signature_utils.check_request_expiry(req); - } catch (err) { - dbg.error('authenticate_request: ERROR', err.stack || err); - throw new LambdaError(LambdaError.SignatureDoesNotMatch); - } -} - -function parse_op_name(req) { - const m = req.method.toLowerCase(); - const u = req.url.slice('/2015-03-31/functions'.length); - - // service url - if (u === '/' || u === '' || u[0] !== '/') { - req.params = {}; - return `${m}_service`; - } - - const index1 = u.indexOf('/', 1); - const pos1 = index1 < 0 ? u.length : index1; - const index2 = u.indexOf('/', pos1 + 1); - const pos2 = index2 < 0 ? u.length : index2; - - const func_name = decodeURIComponent(u.slice(1, pos1)); - const sub_resource = decodeURIComponent(u.slice(pos1 + 1, pos2)); - const sub_resource_id = decodeURIComponent(u.slice(pos2 + 1)); - - if (sub_resource && sub_resource_id) { - // func sub resource with identifier - req.params = { - func_name, - [sub_resource]: sub_resource_id, - }; - return `${m}_func_${sub_resource}`; - } else if (sub_resource) { - // func sub resource - req.params = { func_name }; - return `${m}_func_${sub_resource}`; - } else { - // func url - req.params = { func_name }; - return `${m}_func`; - } -} - -function handle_error(req, res, err) { - const lambda_err = - ((err instanceof LambdaError) && err) || - new LambdaError(RPC_ERRORS_TO_LAMBDA[err.rpc_code] || LambdaError.ServiceException); - - const reply = lambda_err.reply(); - dbg.error('LAMBDA ERROR', reply, - req.method, req.originalUrl, - JSON.stringify(req.headers), - err.stack || err); - res.statusCode = lambda_err.http_code; - res.setHeader('Content-Type', 'application/json'); - res.setHeader('Content-Length', Buffer.byteLength(reply)); - res.end(reply); -} - - -// EXPORTS -module.exports = lambda_rest; diff --git a/src/endpoint/lambda/lambda_utils.js b/src/endpoint/lambda/lambda_utils.js deleted file mode 100644 index ad6ff51afe..0000000000 --- a/src/endpoint/lambda/lambda_utils.js +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -function get_func_config(info) { - return { - FunctionName: info.config.name, - Version: info.config.version || '$LATEST', - Runtime: info.config.runtime, - Handler: info.config.handler, - Role: info.config.role, - MemorySize: info.config.memory_size, - Timeout: info.config.timeout, - Description: info.config.description, - CodeSize: info.config.code_size, - CodeSha256: info.config.code_sha256, - LastModified: new Date(info.config.last_modified).toISOString(), - FunctionArn: info.config.resource_name, - VpcConfig: { - SubnetIds: info.config.pools - } - }; -} - -exports.get_func_config = get_func_config; diff --git a/src/endpoint/lambda/ops/lambda_create_func.js b/src/endpoint/lambda/ops/lambda_create_func.js deleted file mode 100644 index 9142c3c6b4..0000000000 --- a/src/endpoint/lambda/ops/lambda_create_func.js +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const _ = require('lodash'); - -const lambda_utils = require('../lambda_utils'); - -async function create_func(req, res) { - const fn = req.body; - console.log('create_func', req.params, fn); - const func = await req.func_sdk.create_func({ - config: _.omitBy({ - name: fn.FunctionName, - version: '$LATEST', - description: fn.Description, - role: fn.Role, - runtime: fn.Runtime, - handler: fn.Handler, - memory_size: fn.MemorySize, - timeout: fn.Timeout, - pools: fn.VpcConfig && fn.VpcConfig.SubnetIds, - }, _.isUndefined), - code: _.omitBy({ - zipfile_b64: fn.Code.ZipFile, - s3_bucket: fn.Code.S3Bucket, - s3_key: fn.Code.S3Key, - s3_obj_version: fn.Code.S3ObjectVersion, - }, _.isUndefined), - publish: fn.Publish, - }); - return lambda_utils.get_func_config(func); -} - -module.exports = { - handler: create_func, - body: { - type: 'json', - }, - reply: { - type: 'json', - }, -}; diff --git a/src/endpoint/lambda/ops/lambda_delete_func.js b/src/endpoint/lambda/ops/lambda_delete_func.js deleted file mode 100644 index e8cff7107a..0000000000 --- a/src/endpoint/lambda/ops/lambda_delete_func.js +++ /dev/null @@ -1,19 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -function delete_func(req, res) { - return req.func_sdk.delete_func({ - name: req.params.func_name, - version: req.query.Qualifier || '$LATEST' - }); -} - -module.exports = { - handler: delete_func, - body: { - type: 'empty', - }, - reply: { - type: 'empty', - }, -}; diff --git a/src/endpoint/lambda/ops/lambda_get_func.js b/src/endpoint/lambda/ops/lambda_get_func.js deleted file mode 100644 index 743871650e..0000000000 --- a/src/endpoint/lambda/ops/lambda_get_func.js +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const lambda_utils = require('../lambda_utils'); - -async function get_func(req, res) { - console.log('read_func', req.params, req.query); - const func = await req.func_sdk.read_func({ - name: req.params.func_name, - version: req.query.Qualifier || '$LATEST' - }); - return { - Configuration: lambda_utils.get_func_config(func), - Code: { - Location: func.code_location.url, - RepositoryType: func.code_location.repository, - } - }; -} - -module.exports = { - handler: get_func, - body: { - type: 'empty', - }, - reply: { - type: 'json', - }, -}; diff --git a/src/endpoint/lambda/ops/lambda_invoke_func.js b/src/endpoint/lambda/ops/lambda_invoke_func.js deleted file mode 100644 index 2fe0f09a01..0000000000 --- a/src/endpoint/lambda/ops/lambda_invoke_func.js +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -async function invoke_func(req, res) { - const func_res = await req.func_sdk.invoke_func({ - name: req.params.func_name, - version: req.query.Qualifier || '$LATEST', - event: req.body, - }); - if (func_res.error) { - res.setHeader('x-amz-function-error', 'Unhandled'); - return func_res.error; - } - return func_res.result; -} - -module.exports = { - handler: invoke_func, - body: { - type: 'json', - }, - reply: { - type: 'json', - }, -}; diff --git a/src/endpoint/lambda/ops/lambda_list_funcs.js b/src/endpoint/lambda/ops/lambda_list_funcs.js deleted file mode 100644 index 91b8d17181..0000000000 --- a/src/endpoint/lambda/ops/lambda_list_funcs.js +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const _ = require('lodash'); -const lambda_utils = require('../lambda_utils'); - -async function list_funcs(req, res) { - const reply = await req.func_sdk.list_funcs(); - return { - Functions: _.map(reply.functions, func => lambda_utils.get_func_config(func)) - }; -} - -module.exports = { - handler: list_funcs, - body: { - type: 'empty', - }, - reply: { - type: 'json', - }, -}; diff --git a/src/sdk/func_sdk.js b/src/sdk/func_sdk.js deleted file mode 100644 index 5b7e7903a0..0000000000 --- a/src/sdk/func_sdk.js +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -class FuncSDK { - - constructor(rpc_client) { - this.rpc_client = rpc_client; - } - - set_auth_token(auth_token) { - this.rpc_client.options.auth_token = auth_token; - } - - invoke_func(params) { - console.log('invoke_func', params); - return this.rpc_client.func.invoke_func(params); - } - - list_funcs() { - return this.rpc_client.func.list_funcs(); - } - - read_func(params) { - return this.rpc_client.func.read_func(params); - } - - create_func(params) { - return this.rpc_client.func.create_func(params); - } - - delete_func(params) { - return this.rpc_client.func.delete_func(params); - } - -} - -module.exports = FuncSDK; diff --git a/src/sdk/namespace_cache.js b/src/sdk/namespace_cache.js index 6799d5e902..708a9064db 100644 --- a/src/sdk/namespace_cache.js +++ b/src/sdk/namespace_cache.js @@ -8,7 +8,6 @@ const dbg = require('../util/debug_module')(__filename); const cache_config = require('../../config.js').NAMESPACE_CACHING; const range_utils = require('../util/range_utils'); const RangeStream = require('../util/range_stream'); -const P = require('../util/promise'); const buffer_utils = require('../util/buffer_utils'); const stream_utils = require('../util/stream_utils'); const semaphore = require('../util/semaphore'); @@ -619,18 +618,6 @@ class NamespaceCache { read_count: 1, }); - const operation = 'ObjectRead'; - const load_for_trigger = !params.noobaa_trigger_agent && - object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - if (load_for_trigger) { - object_sdk.dispatch_triggers({ - active_triggers: this.active_triggers, - operation, - obj: params.object_md, - bucket: params.bucket - }); - } - return tap_stream; } @@ -640,12 +627,9 @@ class NamespaceCache { async upload_object(params, object_sdk) { dbg.log0("NamespaceCache.upload_object", _.omit(params, 'source_stream')); - const operation = 'ObjectCreated'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); const bucket_free_space_bytes = await this._get_bucket_free_space_bytes(params, object_sdk); let upload_response; - let etag; if (params.size > bucket_free_space_bytes) { dbg.log0("NamespaceCache.upload_object: object is too big, skip caching"); @@ -658,8 +642,6 @@ class NamespaceCache { hub_write_latency: Number(process.hrtime.bigint() - start_time) / 1e6, }); - etag = upload_response.etag; - } else { // UPLOAD SIMULTANEOUSLY TO BOTH @@ -739,18 +721,6 @@ class NamespaceCache { } upload_response = hub_res.value; - etag = upload_response.etag; - } - - if (load_for_trigger) { - const obj = { - bucket: params.bucket, - key: params.key, - size: params.size, - content_type: params.content_type, - etag - }; - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); } return upload_response; @@ -773,21 +743,7 @@ class NamespaceCache { } async complete_object_upload(params, object_sdk) { - const operation = 'ObjectCreated'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - const res = await this.namespace_hub.complete_object_upload(params, object_sdk); - if (load_for_trigger) { - const head_res = await this.read_object_md(params, object_sdk); - const obj = { - bucket: params.bucket, - key: params.key, - size: head_res.size, - content_type: head_res.content_type, - etag: head_res.etag - }; - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } await this._delete_object_from_cache(params, object_sdk); return res; } @@ -814,42 +770,15 @@ class NamespaceCache { throw cache_res.reason; } - const operation = 'ObjectRemoved'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - if (load_for_trigger) { - object_sdk.dispatch_triggers({ - active_triggers: this.active_triggers, - operation, - obj: params.object_md, - bucket: params.bucket - }); - } - return hub_res.value; } async delete_multiple_objects(params, object_sdk) { - const operation = 'ObjectRemoved'; const objects = params.objects.filter(obj => obj.version_id); if (objects.length > 0) { dbg.error('S3 Version request not (NotImplemented) for s3_post_bucket_delete', params); throw new S3Error(S3Error.NotImplemented); } - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - const head_res = load_for_trigger && await P.map(params.objects, async obj => { - const request = { - bucket: params.bucket, - key: obj.key, - version_id: obj.version_id - }; - let obj_md; - try { - obj_md = _.defaults({ key: obj.key }, await this.namespace_hub.read_object_md(request, object_sdk)); - } catch (err) { - if (err.rpc_code !== 'NO_SUCH_OBJECT') throw err; - } - return obj_md; - }); const [hub_res, cache_res] = await Promise.allSettled([ this.namespace_hub.delete_multiple_objects(params, object_sdk), @@ -862,21 +791,6 @@ class NamespaceCache { throw cache_res.reason; } - if (load_for_trigger) { - for (let i = 0; i < hub_res.value.length; ++i) { - const deleted_obj = hub_res.value[i]; - const head_obj = head_res[i]; - if (_.isUndefined(deleted_obj && deleted_obj.err_code) && head_obj) { - object_sdk.dispatch_triggers({ - active_triggers: this.active_triggers, - operation, - obj: head_obj, - bucket: params.bucket - }); - } - } - } - return hub_res.value; } diff --git a/src/sdk/namespace_merge.js b/src/sdk/namespace_merge.js index dc382adbdc..86795bd230 100644 --- a/src/sdk/namespace_merge.js +++ b/src/sdk/namespace_merge.js @@ -78,24 +78,14 @@ class NamespaceMerge { } async read_object_stream(params, object_sdk) { - const operation = 'ObjectRead'; - const load_for_trigger = !params.noobaa_trigger_agent && - object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); params = _.omit(params, 'noobaa_trigger_agent'); let reply; - let obj = { key: params.key }; // use the saved ns from read_object_md if (params.object_md && params.object_md.ns) { - obj = _.defaults(obj, params.object_md); reply = params.object_md.ns.read_object_stream(params, object_sdk); } else { - obj = load_for_trigger && _.defaults(obj, await this.read_object_md(params, object_sdk)); reply = this._ns_get(ns => ns.read_object_stream(params, object_sdk)); } - // Notice: We dispatch the trigger prior to the finish of the read - if (load_for_trigger) { - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -104,20 +94,7 @@ class NamespaceMerge { /////////////////// async upload_object(params, object_sdk) { - const operation = 'ObjectCreated'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - const reply = await this._ns_put(ns => ns.upload_object(params, object_sdk)); - if (load_for_trigger) { - const obj = { - bucket: params.bucket, - key: params.key, - size: params.size, - content_type: params.content_type, - etag: reply.etag - }; - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -152,21 +129,7 @@ class NamespaceMerge { } async complete_object_upload(params, object_sdk) { - const operation = 'ObjectCreated'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - const reply = await this._ns_put(ns => ns.complete_object_upload(params, object_sdk)); - if (load_for_trigger) { - const head_reply = await this.read_object_md(params, object_sdk); - const obj = { - bucket: params.bucket, - key: params.key, - size: head_reply.size, - content_type: head_reply.content_type, - etag: reply.etag - }; - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -181,69 +144,28 @@ class NamespaceMerge { // TODO should we: (1) delete from all ns ? (2) delete from writable ns ? (3) create a "delete marker" on writable ns async delete_object(params, object_sdk) { - const operation = 'ObjectRemoved'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - let obj; - try { - obj = load_for_trigger && _.defaults({ key: params.key }, await this.read_object_md(params, object_sdk)); - } catch (error) { - if (!_.includes(EXCEPT_REASONS, error.rpc_code || 'UNKNOWN_ERR')) throw error; - } const reply = await this._ns_map(ns => ns.delete_object(params, object_sdk), EXCEPT_REASONS); - // TODO: What should I send to the trigger on non existing objects delete? - if (load_for_trigger && obj) { - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } // TODO: Decide which one to return (currently we do not support versioning on our namespaces) return _.first(reply); } async delete_multiple_objects(params, object_sdk) { - const operation = 'ObjectRemoved'; - const load_for_trigger = object_sdk.should_run_triggers({ active_triggers: this.active_triggers, operation }); - const head_res = load_for_trigger && await this._ns_map(ns => P.map(params.objects, async obj => { - const request = { - bucket: params.bucket, - key: obj.key, - version_id: obj.version_id - }; - let obj_md; - try { - obj_md = _.defaults({ key: obj.key }, await ns.read_object_md(request, object_sdk)); - } catch (error) { - if (!_.includes(EXCEPT_REASONS, error.rpc_code || 'UNKNOWN_ERR')) throw error; - } - return obj_md; - })); const deleted_res = await this._ns_map(ns => ns.delete_multiple_objects(params, object_sdk)); const merged_res = this._merge_multiple_delete_responses({ - head_res, deleted_res, total_objects: params.objects.length }); - if (load_for_trigger) { - merged_res.forEach(object => { - const obj = object.obj; - if (object.success && obj) { - object_sdk.dispatch_triggers({ active_triggers: this.active_triggers, operation, obj, bucket: params.bucket }); - } - }); - } - return _.map(merged_res, obj => obj.res); } _merge_multiple_delete_responses(params) { - const { head_res, deleted_res } = params; + const { deleted_res } = params; let ns_conslusion; - if (head_res && (head_res.length !== deleted_res.length)) throw new S3Error(S3Error.InternalError); - for (let ns = 0; ns < deleted_res.length; ++ns) { const deleted_ns = deleted_res[ns]; - const head_ns = head_res && head_res[ns]; - const ns_merged = this._handle_single_namespace_deletes({ deleted_ns, head_ns }); + const ns_merged = this._handle_single_namespace_deletes({ deleted_ns }); if (ns_conslusion) { for (let obj_index = 0; obj_index < ns_conslusion.length; obj_index++) { ns_conslusion[obj_index] = @@ -260,12 +182,11 @@ class NamespaceMerge { _handle_single_namespace_deletes(params) { const response = []; - const { deleted_ns, head_ns } = params; + const { deleted_ns } = params; for (let i = 0; i < deleted_ns.length; ++i) { const res = deleted_ns[i]; - const obj = head_ns && head_ns[i]; if (_.isUndefined(res && res.err_code)) { - response.push({ success: true, obj, res }); + response.push({ success: true, res }); } else { response.push({ success: false, res }); } @@ -277,15 +198,11 @@ class NamespaceMerge { _pick_ns_obj_reply(params) { const { curr, cand } = params; const STATUSES = { - FAILED_WITH_INFO: 3, - FAILED_WITHOUT_INFO: 2, - SUCCEEDED_WITH_INFO: 1, + FAILED_WITHOUT_INFO: 1, SUCCEEDED_WITHOUT_INFO: 0 }; const get_object_status = object => { - if (object.success && object.obj) return STATUSES.SUCCEEDED_WITH_INFO; if (object.success) return STATUSES.SUCCEEDED_WITHOUT_INFO; - if (object.obj) return STATUSES.FAILED_WITH_INFO; return STATUSES.FAILED_WITHOUT_INFO; }; const curr_status = get_object_status(curr); @@ -293,8 +210,6 @@ class NamespaceMerge { if (curr_status > cand_status) return curr; if (cand_status > curr_status) return cand; - if ((cand_status === STATUSES.FAILED_WITH_INFO || cand_status === STATUSES.SUCCEEDED_WITH_INFO) && - (cand.obj.create_time > curr.obj.create_time)) return cand; return curr; } diff --git a/src/sdk/namespace_nb.js b/src/sdk/namespace_nb.js index ae5a36bd29..c11245662a 100644 --- a/src/sdk/namespace_nb.js +++ b/src/sdk/namespace_nb.js @@ -3,15 +3,10 @@ const _ = require('lodash'); -const P = require('../util/promise'); const blob_translator = require('./blob_translator'); const s3_utils = require('../endpoint/s3/s3_utils'); const S3Error = require('../endpoint/s3/s3_errors').S3Error; -const EXCEPT_REASONS = [ - 'NO_SUCH_OBJECT' -]; - const object_id_regex = /^[0-9a-fA-F]{24}$/; /** @@ -87,8 +82,6 @@ class NamespaceNB { } async read_object_stream(params, object_sdk) { - const operation = 'ObjectRead'; - let obj = { key: params.key }; params = _.defaults({ client: object_sdk.rpc_client, bucket: this.target_bucket, @@ -96,22 +89,8 @@ class NamespaceNB { // Noobaa bucket does not currrently support partNumber query parameter. Ignore it for now. // If set, part_number is positive integer from 1 to 10000 if (params.part_number) _.unset(params, 'part_number'); - const active_triggers = this.get_triggers_for_bucket(params.bucket); - const load_for_trigger = !params.noobaa_trigger_agent && object_sdk.should_run_triggers({ - active_triggers, - operation - }); params = _.omit(params, 'noobaa_trigger_agent'); - if (params.object_md) { - obj = _.defaults(obj, params.object_md); - } else { - obj = load_for_trigger && _.defaults(obj, await this.read_object_md(params, object_sdk)); - } const reply = object_sdk.object_io.read_object_stream(params); - // Notice: We dispatch the trigger prior to the finish of the read - if (load_for_trigger) { - object_sdk.dispatch_triggers({ active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -120,27 +99,11 @@ class NamespaceNB { /////////////////// async upload_object(params, object_sdk) { - const operation = 'ObjectCreated'; params = _.defaults({ client: object_sdk.rpc_client, bucket: this.target_bucket, }, params); - const active_triggers = this.get_triggers_for_bucket(params.bucket); - const load_for_trigger = object_sdk.should_run_triggers({ - active_triggers, - operation - }); const reply = await object_sdk.object_io.upload_object(params); - if (load_for_trigger) { - const obj = { - bucket: params.bucket, - key: params.key, - size: params.size, - content_type: params.content_type, - etag: reply.etag - }; - object_sdk.dispatch_triggers({ active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -183,24 +146,8 @@ class NamespaceNB { } async complete_object_upload(params, object_sdk) { - const operation = 'ObjectCreated'; if (this.target_bucket) params = _.defaults({ bucket: this.target_bucket }, params); - const active_triggers = this.get_triggers_for_bucket(params.bucket); - const load_for_trigger = object_sdk.should_run_triggers({ - active_triggers, - operation - }); const reply = await object_sdk.rpc_client.object.complete_object_upload(params); - if (load_for_trigger) { - const obj = { - bucket: params.bucket, - key: params.key, - size: reply.size, - content_type: reply.content_type, - etag: reply.etag - }; - object_sdk.dispatch_triggers({ active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } @@ -216,79 +163,20 @@ class NamespaceNB { /////////////////// async delete_object(params, object_sdk) { - const operation = 'ObjectRemoved'; if (this.target_bucket) params = _.defaults({ bucket: this.target_bucket }, params); - const active_triggers = this.get_triggers_for_bucket(params.bucket); - const load_for_trigger = object_sdk.should_run_triggers({ - active_triggers, - operation - }); - let obj; - try { - obj = load_for_trigger && _.defaults({ key: params.key }, await this.read_object_md(params, object_sdk)); - } catch (error) { - if (!_.includes(EXCEPT_REASONS, error.rpc_code || 'UNKNOWN_ERR')) throw error; - } const reply = await object_sdk.rpc_client.object.delete_object(params); - // TODO: What should I send to the trigger on non existing objects delete? - if (load_for_trigger && obj) { - object_sdk.dispatch_triggers({ active_triggers, operation, obj, bucket: params.bucket }); - } return reply; } async delete_multiple_objects(params, object_sdk) { - const operation = 'ObjectRemoved'; if (this.target_bucket) params = _.defaults({ bucket: this.target_bucket }, params); - const active_triggers = this.get_triggers_for_bucket(params.bucket); - const load_for_trigger = object_sdk.should_run_triggers({ - active_triggers, - operation - }); // TODO: What should I do instead of failing on one failed head request? // I cannot exclude the files that failed from delete since it will be considered altering the request of the client // TODO: Notice that we do not handle the md_conditions for the heads - const head_res = load_for_trigger && await P.map(params.objects, async obj => { - const request = { - bucket: params.bucket, - key: obj.key, - version_id: obj.version_id - }; - let obj_md; - try { - obj_md = _.defaults({ key: obj.key }, await this.read_object_md(request, object_sdk)); - } catch (error) { - if (!_.includes(EXCEPT_REASONS, error.rpc_code || 'UNKNOWN_ERR')) throw error; - } - return obj_md; - }); const deleted_res = await object_sdk.rpc_client.object.delete_multiple_objects(params); - if (load_for_trigger) { - this._dispatch_multiple_delete_triggers({ - head_res, - deleted_res, - object_sdk, - operation, - bucket: params.bucket, - active_triggers - }); - } return deleted_res; } - // Both responses should be in the same order and the same length - _dispatch_multiple_delete_triggers(params) { - const { head_res, deleted_res, object_sdk, operation, bucket, active_triggers } = params; - if (head_res.length !== deleted_res.length) throw new S3Error(S3Error.InternalError); - for (let i = 0; i < deleted_res.length; ++i) { - const deleted_obj = deleted_res[i]; - const head_obj = head_res[i]; - if (_.isUndefined(deleted_obj && deleted_obj.err_code) && head_obj) { - object_sdk.dispatch_triggers({ active_triggers, operation, obj: head_obj, bucket }); - } - } - } - //////////////////// // OBJECT TAGGING // //////////////////// diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 63ef93c244..183d135db8 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -196,7 +196,6 @@ interface Bucket extends Base { last_update: number; }; lifecycle_configuration_rules?: object; - lambda_triggers?: object; master_key_id: ID; } diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index 787a2a0572..8d65186a39 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -79,7 +79,6 @@ const dn_cache = new LRUCache({ const MULTIPART_NAMESPACES = [ 'NET_STORAGE' ]; -const required_obj_properties = ['obj_id', 'bucket', 'key', 'size', 'content_type', 'etag']; /** _validate_account is an additional layer (to expiry_ms) * and in NC deployment it checks the stat of the config file @@ -1092,20 +1091,6 @@ class ObjectSDK { }); } - async dispatch_triggers({ active_triggers, obj, operation, bucket }) { - const dispatch = this.should_run_triggers({ active_triggers, obj, operation }); - if (dispatch) { - const dispatch_obj = _.pick(obj, required_obj_properties); - // Dummy obj_id (not all flows return with obj_id and we need it for the API schema) - dispatch_obj.obj_id = '10101010aaaabbbbccccdddd'; - await this.internal_rpc_client.object.dispatch_triggers({ - bucket, - event_name: operation, - obj: dispatch_obj - }); - } - } - ///////////////////////// // BUCKET NOTIFICATION // ///////////////////////// diff --git a/src/server/func_services/func_indexes.js b/src/server/func_services/func_indexes.js deleted file mode 100644 index 0ff75e303d..0000000000 --- a/src/server/func_services/func_indexes.js +++ /dev/null @@ -1,14 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -module.exports = [{ - fields: { - system: 1, - name: 1, - version: 1, - deleted: 1, // allow to filter deleted - }, - options: { - unique: true, - } -}, ]; diff --git a/src/server/func_services/func_schema.js b/src/server/func_services/func_schema.js deleted file mode 100644 index 13362ca711..0000000000 --- a/src/server/func_services/func_schema.js +++ /dev/null @@ -1,84 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -module.exports = { - $id: 'func_schema', - type: 'object', - required: [ - '_id', - 'system', - 'pools', - 'name', - 'exec_account', - 'version', - 'runtime', - 'handler', - 'last_modified', - 'last_modifier', - 'code_size', - 'code_sha256', - 'code', - ], - properties: { - _id: { - objectid: true - }, - deleted: { - date: true - }, - system: { - objectid: true - }, - pools: { - type: 'array', - items: { - objectid: true - } - }, - name: { - type: 'string' - }, - exec_account: { - objectid: true - }, - version: { - type: 'string' - }, - description: { - type: 'string' - }, - role: { - type: 'string' - }, - runtime: { - type: 'string' - }, - handler: { - type: 'string' - }, - memory_size: { - type: 'integer' - }, - timeout: { - type: 'integer' - }, - last_modified: { - date: true - }, - last_modifier: { - objectid: true - }, - resource_name: { - type: 'string' - }, - code_size: { - type: 'integer' - }, - code_sha256: { - type: 'string' - }, - code: { - type: 'string' - } - } -}; diff --git a/src/server/func_services/func_server.js b/src/server/func_services/func_server.js deleted file mode 100644 index 3888250e89..0000000000 --- a/src/server/func_services/func_server.js +++ /dev/null @@ -1,446 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const _ = require('lodash'); - -const crypto = require('crypto'); -const P = require('../../util/promise'); -const stream = require('stream'); -const buffer_utils = require('../../util/buffer_utils'); -const { RpcError, RPC_BUFFERS } = require('../../rpc'); -const dbg = require('../../util/debug_module')(__filename); -const FuncNode = require('../../agent/func_services/func_node'); -const addr_utils = require('../../util/addr_utils'); -const http_utils = require('../../util/http_utils'); -const Dispatcher = require('../notifications/dispatcher'); -const server_rpc = require('../server_rpc'); -const func_store = require('./func_store'); -const func_stats_store = require('./func_stats_store'); -const system_store = require('../system_services/system_store').get_instance(); -const auth_server = require('../common_services/auth_server'); -const node_allocator = require('../node_services/node_allocator'); - -const AWS = require('aws-sdk'); - -const FUNC_CONFIG_FIELDS_MUTABLE = [ - 'description', - 'role', - 'handler', - 'runtime', - 'memory_size', - 'timeout', -]; -const FUNC_CONFIG_DEFAULTS = { - handler: 'index', - runtime: 'nodejs6', - memory_size: 128, - timeout: 60, -}; -const FUNC_CONFIG_FIELDS_IMMUTABLE = [ - 'code_size', - 'code_sha256', - 'resource_name', -]; - -const FUNC_STATS_DEFAULTS = { - invoked: 0, - fulfilled: 0, - rejected: 0, - aggr_response_time: 0, - max_response_time: 0, - avg_response_time: 0, -}; - -async function create_func(req) { - dbg.log0('create_func::', req.params.config); - const { config: func_config, code: func_code } = req.params; - const { _id: system, pools_by_name } = req.system; - const { _id: exec_account } = req.account; - const { name, pools: pool_names = [] } = func_config; - const version = '$LATEST'; - const pools = pool_names.map(pool_name => pools_by_name[pool_name]); - for (const pool of pools) { - if (pool.cloud_pool_info || pool.mongo_pool_info) { - throw new RpcError('FORBIDDEN', 'invalid_pool'); - } - } - const resource_name = `arn:noobaa:lambda:region:${system}:function:${name}:${version}`; - const { code, code_sha256, code_size } = await _get_func_code_b64(req, func_code); - - const func_id = func_store.instance().make_func_id(); - await func_store.instance().create_func({ - ...FUNC_CONFIG_DEFAULTS, - ..._.pick(func_config, FUNC_CONFIG_FIELDS_MUTABLE), - _id: func_id, - system, - exec_account, - name, - version, - last_modified: new Date(), - last_modifier: exec_account, - resource_name, - pools: pools.map(pool => pool._id), - code, - code_sha256, - code_size - }); - - Dispatcher.instance().activity({ - event: 'functions.func_created', - level: 'info', - system: req.system._id, - actor: req.account && req.account._id, - func: func_id, - desc: '' - }); - - await _load_func(req); - return _get_func_info(req.func); -} - -async function update_func(req) { - await _load_func(req); - - const { func, params, system } = req; - const func_config = params.config; - const config_updates = _.pick(func_config, FUNC_CONFIG_FIELDS_MUTABLE); - let code_updated = false; - - if (func_config.pools) { - config_updates.pools = _.map( - func_config.pools, - pool_name => system.pools_by_name[pool_name]._id - ); - } - - const func_code = params.code; - if (func_code) { - const { code, code_sha256, code_size } = await _get_func_code_b64(req, func_code); - - config_updates.code = code; - config_updates.code_sha256 = code_sha256; - config_updates.code_size = code_size; - - code_updated = true; - } - - config_updates.last_modified = new Date(); - config_updates.last_modifier = req.account._id; - - await func_store.instance().update_func(func._id, config_updates); - await _load_func(req); - - const act = { - level: 'info', - system: req.system._id, - actor: req.account && req.account._id, - func: req.func._id, - desc: '' - }; - if (code_updated) { - act.event = 'functions.func_code_edit'; - } else { - act.event = 'functions.func_config_edit'; - } - - Dispatcher.instance().activity(act); - return _get_func_info(func); -} - -async function delete_func(req) { - dbg.log0('delete_func::', req.params.name); - await _load_func(req); - //TODO: We might not deleting the record, need to check if we have bg for that. - // If not we might want to change the code field to an empty one. (or maybe not). - await func_store.instance().delete_func(req.func._id); -} - -async function read_func(req) { - await _load_func(req); - const reply = _get_func_info(req.func); - if (req.params.read_code) { - const system = req.system._id; - const { name, version } = req.params; - const func = await func_store.instance().read_func(system, name, version); - const zipfile_b64 = func.code; - //Converting the base64 string into a zipfile again (stream then buffer). - const zipfile_stream = new stream.Readable({ - read(size) { - this.push(Buffer.from(zipfile_b64, 'base64')); - this.push(null); - } - }); - const zipfile = await buffer_utils.read_stream_join(zipfile_stream); - reply[RPC_BUFFERS] = { zipfile }; - } - return reply; -} - -async function read_func_stats(req) { - // Load the function into the request. - await _load_func(req); - - const { - since, - till = Date.now(), - step, - percentiles = [0.5, 0.9, 0.99], - max_samples = 10000 - } = req.params; - - if (till <= since) { - throw new Error('read_func_stats: Invalid time range'); - } - - if (!step || step <= 0) { - throw new Error('read_func_stats: Invalid step value'); - } - - const normalized_since = Math.floor(since / step) * step; - const normalized_till = Math.ceil(till / step) * step; - const by_key = _.fromPairs(await func_stats_store.instance() - .query_func_stats({ - system: req.system._id, - func: req.func._id, - since: new Date(normalized_since), - till: new Date(normalized_till), - step, - percentiles, - max_samples - })); - - const emptyPercentilesArray = percentiles.map(percentile => { - const value = 0; - return { percentile, value }; - }); - - return { - //stats is counting the `-1` key that is used for "Request and Error Count Over Time" - stats: { - ...FUNC_STATS_DEFAULTS, - since: normalized_since, - till: normalized_till, - response_percentiles: emptyPercentilesArray, - ...by_key[-1] - }, - //slices is per since time key for the candles charts - slices: _.range( - normalized_since, - normalized_till, - step - ).map(since_time => ({ - ...FUNC_STATS_DEFAULTS, - since: since_time, - till: since_time + step, - response_percentiles: emptyPercentilesArray, - ...by_key[since_time] - })) - }; -} - -async function list_funcs(req) { - const funcs = await func_store.instance().list_funcs(req.system._id); - return { - functions: _.map(funcs, _get_func_info) - }; -} - -async function list_func_versions(req) { - const funcs = await func_store.instance().list_func_versions( - req.system._id, - req.params.name - ); - return { - versions: _.map(funcs, _get_func_info) - }; -} - -const server_func_node = new FuncNode({ - rpc_client: server_rpc.client, - storage_path: '/tmp/', -}); - -async function invoke_func(req) { - let res; - dbg.log0('invoke_func::', req.params.name); - const time = new Date(); - await _load_func(req); - check_event_permission(req); - await P.map(req.func.pools, pool => node_allocator.refresh_pool_alloc(pool)); - - const func = req.func; - const node = node_allocator.allocate_node({ pools: func.pools }); - const params = { - config: _get_func_info(func).config, - event: req.params.event, - aws_config: _make_aws_config(req), - rpc_options: _make_rpc_options(req), - }; - try { - if (node) { - dbg.log0('invoking on node', - func.name, req.params.event, - node.name, node.pool); - res = await server_rpc.client.func_node.invoke_func(params, { - address: node.rpc_address - }); - } else { - dbg.log0('invoking on server', func.name, req.params.event); - res = await server_func_node.invoke_func({ params }); - } - } catch (e) { - dbg.error('invoke returned with error:', e); - throw e; - } - await func_stats_store.instance().create_func_stat({ - system: req.func.system, - func: req.func._id, - time: time, - took: Date.now() - time.getTime(), - error: res.error ? true : undefined, - error_msg: res.error ? res.error.message : undefined, - }); - if (!_.isUndefined(res.error)) { - throw res.error; - } - return res; -} - -function check_event_permission(req) { - const event = req.params.event; - if (event && event.Records) { - _.forEach(event.Records, record => { - const bucket_name = record && record.s3 && record.s3.bucket && record.s3.bucket.name; - if (typeof(bucket_name) === 'string') { - const bucket = req.system.buckets_by_name && req.system.buckets_by_name[bucket_name]; - if (!bucket || bucket.deleting) throw new RpcError('UNAUTHORIZED', 'No such bucket'); - } - }); - } -} - -// _get_func_code will return the code in base64, the code size and it's sha256 -async function _get_func_code_b64(req, func_code) { - let code; - let code_size; - const sha256 = crypto.createHash('sha256'); - if (func_code.zipfile_b64) { - // zipfile is given as base64 string - code = func_code.zipfile_b64; - code_size = Buffer.from(func_code.zipfile_b64, 'base64').length; - } else if (req.rpc_params[RPC_BUFFERS] && req.rpc_params[RPC_BUFFERS].zipfile) { - // zipfile is given as buffer - code = req.rpc_params[RPC_BUFFERS].zipfile.toString('base64'); - code_size = req.rpc_params[RPC_BUFFERS].zipfile.length; - } else if (func_code.s3_bucket && func_code.s3_key) { - console.log(`reading function code from bucket ${func_code.s3_bucket} and key ${func_code.s3_key}`); - const account_keys = req.account.access_keys[0]; - const s3_endpoint = new AWS.S3({ - endpoint: 'http://localhost', - accessKeyId: account_keys.access_key, - secretAccessKey: account_keys.secret_key, - }); - - const get_object_req = await s3_endpoint.getObject({ - Bucket: func_code.s3_bucket, - Key: func_code.s3_key, - }).promise(); - code = get_object_req.Body.toString('base64'); - code_size = get_object_req.ContentLength; - } else if (func_code.url) { - const get_res = await http_utils.http_get(func_code.url); - if (get_res.statusCode < 200 || get_res.statusCode > 299) { - throw new Error(`failed GET request from ${func_code.url}. got status ${get_res.statusCode}`); - } - const code_buffer = await buffer_utils.read_stream_join(get_res); - code = code_buffer.toString('base64'); - code_size = code_buffer.length; - } else { - throw new Error('Unsupported code'); - } - return { - code, - code_sha256: sha256.update(code).digest('base64'), - code_size, - }; -} - -async function _load_func(req) { - const system = req.system._id; - const name = req.params.name || _.get(req, 'params.config.name'); - const version = req.params.version || _.get(req, 'params.config.version'); - const func = await func_store.instance().read_func(system, name, version); - req.func = func; - func.pools = _.map(func.pools, pool_id => system_store.data.get_by_id(pool_id)); - return func; -} - -function _get_func_info(func) { - const config = _.pick(func, - 'name', - 'version', - FUNC_CONFIG_FIELDS_MUTABLE, - FUNC_CONFIG_FIELDS_IMMUTABLE); - config.last_modified = func.last_modified.getTime(); - const account = system_store.data.get_by_id(func.last_modifier); - config.last_modifier = account ? account.email : ''; - config.pools = _.map(func.pools, pool => { - if (pool.name) return pool.name; - return system_store.data.get_by_id(pool).name; - }); - config.exec_account = system_store.data.get_by_id(func.exec_account).email; - const code_location = { - url: '', - repository: '' - }; - return { - config, - code_location - }; -} - -function _make_rpc_options(req) { - // TODO: need to change when we start to run funcs outside the - // cluster (should use a proper hint based on agent location) - const base_address = addr_utils.get_base_address(req.system.system_address); - const account = system_store.data.get_by_id(req.func.exec_account); - const auth_token = auth_server.make_auth_token({ - system_id: req.system._id, - account_id: account._id, - role: 'admin', - }); - return { - address: base_address.toString(), - auth_token: auth_token, - }; -} - -function _make_aws_config(req) { - // TODO: need to change when we start to run funcs outside the - // cluster (should use a proper hint based on agent location) - const ep_addr = addr_utils.get_base_address(req.system.system_address, { - hint: 'INTERNAL', - service: 's3', - api: 's3', - protocol: 'https', - }); - const account = system_store.data.get_by_id(req.func.exec_account); - const account_keys = account.access_keys[0]; - return { - region: 'us-east-1', - endpoint: ep_addr.href, - sslEnabled: false, - s3ForcePathStyle: true, - accessKeyId: account_keys.access_key.unwrap(), - secretAccessKey: account_keys.secret_key.unwrap(), - }; -} - -exports.create_func = create_func; -exports.update_func = update_func; -exports.delete_func = delete_func; -exports.read_func = read_func; -exports.read_func_stats = read_func_stats; -exports.list_funcs = list_funcs; -exports.list_func_versions = list_func_versions; -exports.invoke_func = invoke_func; diff --git a/src/server/func_services/func_stats_indexes.js b/src/server/func_services/func_stats_indexes.js deleted file mode 100644 index 19933a11e1..0000000000 --- a/src/server/func_services/func_stats_indexes.js +++ /dev/null @@ -1,13 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -module.exports = [{ - fields: { - system: 1, - id: 1, - latency_ms: 1, - }, - options: { - unique: false, - } -}, ]; diff --git a/src/server/func_services/func_stats_schema.js b/src/server/func_services/func_stats_schema.js deleted file mode 100644 index 3ebb889877..0000000000 --- a/src/server/func_services/func_stats_schema.js +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -module.exports = { - $id: 'func_stats_schema', - type: 'object', - required: [ - '_id', - 'system', - 'func', - 'time', - 'took', - ], - properties: { - _id: { - objectid: true - }, - system: { - objectid: true - }, - func: { - objectid: true - }, - time: { - date: true - }, - took: { - type: 'number' - }, - error: { - type: 'boolean' - }, - error_msg: { - type: 'string' - }, - } -}; diff --git a/src/server/func_services/func_stats_store.js b/src/server/func_services/func_stats_store.js deleted file mode 100644 index 927ad0d972..0000000000 --- a/src/server/func_services/func_stats_store.js +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -// const _ = require('lodash'); -const mongodb = require('mongodb'); - -// const dbg = require('../../util/debug_module')(__filename); -const db_client = require('../../util/db_client'); - -const func_stats_schema = require('./func_stats_schema'); -const func_stats_indexes = require('./func_stats_indexes'); -const mongo_functions = require('../../util/mongo_functions'); -class FuncStatsStore { - - constructor() { - this._func_stats = db_client.instance().define_collection({ - name: 'func_stats', - schema: func_stats_schema, - db_indexes: func_stats_indexes, - }); - } - - static instance() { - if (!FuncStatsStore._instance) FuncStatsStore._instance = new FuncStatsStore(); - return FuncStatsStore._instance; - } - - make_func_stat_id(id_str) { - return new mongodb.ObjectId(id_str); - } - - async create_func_stat(stat) { - if (!stat._id) { - stat._id = this.make_func_stat_id(); - } - try { - this._func_stats.validate(stat); - await this._func_stats.insertOne(stat); - } catch (err) { - db_client.instance().check_duplicate_key_conflict(err, 'func stat'); - } - return stat; - } - - async query_func_stats(params) { - - const records = await this._func_stats - .mapReduce( - mongo_functions.map_func_stats, - mongo_functions.reduce_func_stats, { - finalize: mongo_functions.finalize_func_stats, - query: { - system: params.system, - func: params.func, - time: { - $gte: params.since, - $lt: params.till - } - }, - scope: { - step: params.step, - percentiles: params.percentiles, - max_samples: params.max_samples - }, - out: { - inline: 1 - } - } - ); - - return records.map(record => [ - record._id, - record.value - ]); - } -} - -/** @type {FuncStatsStore} */ -FuncStatsStore._instance = undefined; - -// EXPORTS -exports.FuncStatsStore = FuncStatsStore; -exports.instance = FuncStatsStore.instance; diff --git a/src/server/func_services/func_store.js b/src/server/func_services/func_store.js deleted file mode 100644 index 1313a8eb92..0000000000 --- a/src/server/func_services/func_store.js +++ /dev/null @@ -1,108 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const mongodb = require('mongodb'); - -const db_client = require('../../util/db_client'); - -const func_schema = require('./func_schema'); -const func_indexes = require('./func_indexes'); - -class FuncStore { - - constructor() { - this._funcs = db_client.instance().define_collection({ - name: 'funcs', - schema: func_schema, - db_indexes: func_indexes, - }); - } - - static instance() { - if (!FuncStore._instance) FuncStore._instance = new FuncStore(); - return FuncStore._instance; - } - - make_func_id(id_str) { - return new mongodb.ObjectId(id_str); - } - - async create_func(func) { - try { - this._funcs.validate(func); - await this._funcs.insertOne(func); - } catch (err) { - db_client.instance().check_duplicate_key_conflict(err, 'func'); - } - return func; - } - - async delete_func(func_id) { - await this._funcs.updateOne({ - _id: func_id, - }, { - $set: { - deleted: new Date() - } - }); - } - - async update_func(func_id, set_updates) { - await this._funcs.updateOne({ - _id: func_id, - }, { - $set: set_updates - }); - } - - async read_func(system, name, version) { - const res = await this._funcs.findOne({ - system: system, - name: name, - version: version, - deleted: null, - }); - return db_client.instance().check_entity_not_deleted(res, 'func'); - } - - async get_by_id_include_deleted(func_id) { - return this._funcs.findOne({ - _id: func_id, - }); - } - - async list_funcs(system) { - return this._funcs.find({ - system: system, - version: '$LATEST', - deleted: null, - }); - } - - async list_funcs_by_pool(system, pool) { - return this._funcs.find({ - system: system, - pools: pool, - deleted: null, - }); - } - - async list_func_versions(system, name) { - return this._funcs.find({ - system: system, - name: name, - deleted: null, - }); - } - - code_filename(system, name, version) { - return system + '/' + name + '/' + version; - } - -} - -FuncStore._instance = undefined; - -// EXPORTS -exports.FuncStore = FuncStore; -exports.instance = FuncStore.instance; diff --git a/src/server/md_server.js b/src/server/md_server.js index b8f200a8a0..b099255bce 100644 --- a/src/server/md_server.js +++ b/src/server/md_server.js @@ -10,7 +10,6 @@ async function register_rpc() { db_client.instance().connect(); server_rpc.register_object_services(); - server_rpc.register_func_services(); server_rpc.register_common_services(); const u = url.parse(server_rpc.rpc.router.md); diff --git a/src/server/notifications/dispatcher.js b/src/server/notifications/dispatcher.js index a9f62dc64f..377322420b 100644 --- a/src/server/notifications/dispatcher.js +++ b/src/server/notifications/dispatcher.js @@ -12,7 +12,6 @@ const alerts_rules = require('./alerts_rules'); const ActivityLogStore = require('../analytic_services/activity_log_store').ActivityLogStore; const system_store = require('../system_services/system_store').get_instance(); const nodes_store = require('../node_services/nodes_store').NodesStore.instance(); -const func_store = require('../func_services/func_store').FuncStore.instance(); const nodes_client = require('../node_services/nodes_client'); const SensitiveString = require('../../util/sensitive_string'); @@ -50,27 +49,26 @@ class Dispatcher { } //Activity Log - activity(item) { - const self = this; + async activity(item) { item.desc = new SensitiveString(item.desc); dbg.log0('Adding ActivityLog entry', item); item.time = item.time || new Date(); - return ActivityLogStore.instance().create(item) - .then(() => { - if (!config.SEND_EVENTS_REMOTESYS) { - return P.resolve(); - } - const l = { - id: String(item._id), - level: item.level, - event: item.event, - }; - return self._resolve_activity_item(item, l); - }) - .then(logitem => self.send_syslog(JSON.stringify(logitem))); + const created = await ActivityLogStore.instance().create(item); + const activity_doc = created || item; // fall-back for unit tests + if (!config.SEND_EVENTS_REMOTESYS) { + return; + } + const l = { + id: String(activity_doc._id), + level: activity_doc.level, + event: activity_doc.event, + }; + // Resolve & enrich log data + const logitem = await this._resolve_activity_item(activity_doc, l); + this.send_syslog(JSON.stringify(logitem)); } - read_activity_log(req) { + async read_activity_log(req) { const self = this; const query = _.pick(req.rpc_params, ['till', 'since', 'skip', 'limit']); @@ -79,22 +77,22 @@ class Dispatcher { } query.system = req.system._id; - return ActivityLogStore.instance().read_activity_log(query) - .then(logs => P.map(logs, function(log_item) { - const l = { - id: String(log_item._id), - level: log_item.level, - event: log_item.event, - time: log_item.time.getTime(), - }; - - if (log_item.desc) { - l.desc = log_item.desc.split('\n'); - } - return P.resolve(self._resolve_activity_item(log_item, l)) - .then(() => l); - })) - .then(logs => ({ logs })); + const activity_log = await ActivityLogStore.instance().read_activity_log(query); + const logs = await P.map(activity_log, async function(log_item) { + const l = { + id: String(log_item._id), + level: log_item.level, + event: log_item.event, + time: log_item.time.getTime(), + }; + + if (log_item.desc) { + l.desc = log_item.desc.split('\n'); + } + await self._resolve_activity_item(log_item, l); + return l; + }); + return { logs }; } //Remote Syslog @@ -154,109 +152,86 @@ class Dispatcher { })); } - _resolve_activity_item(log_item, l) { - return P.resolve() - .then(() => nodes_client.instance().populate_nodes( - log_item.system, log_item, 'node', 'node', NODE_POPULATE_FIELDS)) - .then(() => MDStore.instance().populate_objects( - log_item, 'obj', OBJECT_POPULATE_FIELDS)) - .then(async () => { - if (log_item.node) { - const { name, pool, os_info, host_seq } = log_item.node; - l.node = {}; - if (name) { - l.node.linkable = true; - l.node.name = `${os_info.hostname}#${host_seq}`; - l.node.pool = pool; - - } else { - l.node.linkable = false; - } - } + async _resolve_activity_item(log_item, l) { + await nodes_client.instance().populate_nodes(log_item.system, log_item, 'node', 'node', NODE_POPULATE_FIELDS); + await MDStore.instance().populate_objects(log_item, 'obj', OBJECT_POPULATE_FIELDS); + if (log_item.node) { + const { name, pool, os_info, host_seq } = log_item.node; + l.node = {}; + if (name) { + l.node.linkable = true; + l.node.name = `${os_info.hostname}#${host_seq}`; + l.node.pool = pool; + + } else { + l.node.linkable = false; + } + } - if (log_item.obj) { - const { key, version = 'null', bucket: bucket_id, deleted } = log_item.obj; - l.obj = { key, version }; - l.obj.linkable = !deleted; + if (log_item.obj) { + const { key, version = 'null', bucket: bucket_id, deleted } = log_item.obj; + l.obj = { key, version }; + l.obj.linkable = !deleted; + + const bucket = await system_store.data.get_by_id(bucket_id); + if (bucket) { + l.obj.bucket = bucket.name; + } else { + const { record } = await system_store.data.get_by_id_include_deleted(bucket_id, 'buckets'); + l.obj.bucket = record.name; + l.obj.linkable = false; + } + } - const bucket = system_store.data.get_by_id(bucket_id); - if (bucket) { - l.obj.bucket = bucket.name; - } else { - const { record } = await system_store.data.get_by_id_include_deleted(bucket_id, 'buckets'); - l.obj.bucket = record.name; - l.obj.linkable = false; - } - } + if (log_item.server) { + if (!log_item.server.hostname) { + log_item.server.hostname = ''; + } + l.server = log_item.server; + } - if (log_item.server) { - if (!log_item.server.hostname) { - log_item.server.hostname = ''; - } - l.server = log_item.server; - } + const node = log_item.node && !l.node.linkable && await nodes_store.get_hidden_by_id(log_item.node); + if (node) { + const { host_name, host_sequence, pool: pool_id } = node; + l.node.name = `${host_name}#${host_sequence}`; - return P.resolve(log_item.node && !l.node.linkable && nodes_store.get_hidden_by_id(log_item.node)); - }) - .then(async node => { - if (node) { - const { host_name, host_sequence, pool: pool_id } = node; - l.node.name = `${host_name}#${host_sequence}`; + const pool = await system_store.data.get_by_id(pool_id); + if (pool) { + l.node.pool = pool.name; - const pool = system_store.data.get_by_id(pool_id); - if (pool) { - l.node.pool = pool.name; + } else { + const { record } = await system_store.data.get_by_id_include_deleted(pool_id, 'pools'); + l.node.pool = record && record.name; + } + } + const tier = log_item.tier && await system_store.data.get_by_id_include_deleted(log_item.tier, 'tiers'); - } else { - const { record } = await system_store.data.get_by_id_include_deleted(pool_id, 'pools'); - l.node.pool = record && record.name; - } - } - return P.resolve(log_item.tier && system_store.data.get_by_id_include_deleted(log_item.tier, 'tiers')); - }) - .then(tier => { - if (tier) { - l.tier = _.pick(tier.record, 'name'); - l.tier.linkable = tier.linkable; - } - return P.resolve(log_item.bucket && system_store.data.get_by_id_include_deleted(log_item.bucket, 'buckets')); - }) - .then(bucket => { - if (bucket) { - l.bucket = _.pick(bucket.record, 'name'); - l.bucket.linkable = bucket.linkable; - } - return P.resolve(log_item.pool && system_store.data.get_by_id_include_deleted(log_item.pool, 'pools')); - }) - .then(pool => { - if (pool) { - l.pool = _.pick(pool.record, 'name', 'resource_type'); - l.pool.name = l.pool.name.split('#')[0]; - l.pool.linkable = pool.linkable; - } - return P.resolve(log_item.account && system_store.data.get_by_id_include_deleted(log_item.account, 'accounts')); - }) - .then(account => { - if (account) { - l.account = _.pick(account.record, 'email'); - l.account.linkable = account.linkable; - } - return P.resolve(log_item.func && func_store.get_by_id_include_deleted(log_item.func, 'funcs')); - }) - .then(func => { - if (func) { - l.func = _.pick(func, 'name'); - l.func.linkable = func.linkable; - } - return P.resolve(log_item.actor && system_store.data.get_by_id_include_deleted(log_item.actor, 'accounts')); - }) - .then(actor => { - if (actor) { - l.actor = _.pick(actor.record, 'email'); - l.actor.linkable = actor.linkable; - } - return log_item; - }); + if (tier) { + l.tier = _.pick(tier.record, 'name'); + l.tier.linkable = tier.linkable; + } + const bucket = log_item.bucket && await system_store.data.get_by_id_include_deleted(log_item.bucket, 'buckets'); + if (bucket) { + l.bucket = _.pick(bucket.record, 'name'); + l.bucket.linkable = bucket.linkable; + } + const pool = log_item.pool && await system_store.data.get_by_id_include_deleted(log_item.pool, 'pools'); + if (pool) { + l.pool = _.pick(pool.record, 'name', 'resource_type'); + l.pool.name = l.pool.name.split('#')[0]; + l.pool.linkable = pool.linkable; + } + const account = log_item.account && await system_store.data.get_by_id_include_deleted(log_item.account, 'accounts'); + if (account) { + l.account = _.pick(account.record, 'email'); + l.account.linkable = account.linkable; + } + const actor = log_item.actor && await system_store.data.get_by_id_include_deleted(log_item.actor, 'accounts'); + if (actor) { + l.actor = _.pick(actor.record, 'email'); + l.actor.linkable = actor.linkable; + } + return l; } } diff --git a/src/server/notifications/event_server.js b/src/server/notifications/event_server.js index 64d034f7cc..b20a395cde 100644 --- a/src/server/notifications/event_server.js +++ b/src/server/notifications/event_server.js @@ -15,8 +15,8 @@ const Dispatcher = require('../notifications/dispatcher'); * READ_ACTIVITY_LOG * */ -function read_activity_log(req) { - return Dispatcher.instance().read_activity_log(req); +async function read_activity_log(req) { + return await Dispatcher.instance().read_activity_log(req); } /** diff --git a/src/server/object_services/events_dispatcher.js b/src/server/object_services/events_dispatcher.js deleted file mode 100644 index 086e384749..0000000000 --- a/src/server/object_services/events_dispatcher.js +++ /dev/null @@ -1,99 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const _ = require('lodash'); -const P = require('../../util/promise'); -const dbg = require('../../util/debug_module')(__filename); -const server_rpc = require('../server_rpc'); -const system_store = require('../system_services/system_store').get_instance(); - -const TRIGGER_ATTEMPTS = 2; -const DELAY_BETWEEN_TRIGGER_ATTEMPTS = 5000; // ? - -function get_triggers_for_event(bucket, obj, event_name) { - return _.filter(bucket.lambda_triggers, trigger => - trigger.enabled && (trigger.event_name === event_name || trigger.event_name === event_name.split(':')[0]) && - (!trigger.object_prefix || obj.key.startsWith(trigger.object_prefix)) && - (!trigger.object_suffix || obj.key.endsWith(trigger.object_suffix)) - ); -} - -async function run_bucket_triggers(triggers_to_run, bucket, obj, actor, token) { - if (!triggers_to_run || !triggers_to_run.length) return; - const now = Date.now(); - const updates = []; - for (const { _id: trigger_id } of triggers_to_run) { - updates.push({ - $find: { _id: bucket._id, 'lambda_triggers._id': trigger_id }, - $set: { 'lambda_triggers.$.last_run': now }, - }); - } - system_store.make_changes_in_background({ - update: { - buckets: updates - } - }); - await P.map(triggers_to_run, trigger => { - const event = create_object_event({ - bucket: bucket.name, - time: obj.create_time, - object: obj, - actor: actor, - event_name: trigger.event_name, - id: trigger._id - }); - return P.retry({ - attempts: trigger.attempts ? trigger.attempts : TRIGGER_ATTEMPTS, - delay_ms: DELAY_BETWEEN_TRIGGER_ATTEMPTS, - func: () => run_trigger(trigger, event, bucket.system, token), - }); - }); -} - -// run_trigger invoke a lambda function on an event of a trigger (put/get/list object) -async function run_trigger(trigger, event, system, token) { - const result = await server_rpc.client.func.invoke_func({ - name: trigger.func_name, - version: trigger.func_version, - event: event, - }, { - auth_token: token - }); - if (result.error) { - dbg.error('Got following error in run_trigger:', result.error, ';trigger:', trigger, ';event:', event); - } -} - -function create_object_event({ bucket, object, time, actor, event_name, id }) { - const event = { - Records: [{ - eventVersion: 2.0, - eventSource: 'aws:s3', - eventTime: time, - eventName: event_name, - userIdentity: { - principalId: String(actor.id) - }, - requestParameters: { - sourceIPAddress: 'localhost' - }, - s3: { - s3SchemaVersion: 1.0, - configurationId: id, - bucket: { - name: bucket, - }, - object: { - key: object.key, - size: object.size, - eTag: object.etag, - // sequencer: TBD - } - } - }] - }; - return event; -} - -exports.run_bucket_triggers = run_bucket_triggers; -exports.get_triggers_for_event = get_triggers_for_event; diff --git a/src/server/object_services/object_server.js b/src/server/object_services/object_server.js index 03a25392c4..f9c3245afc 100644 --- a/src/server/object_services/object_server.js +++ b/src/server/object_services/object_server.js @@ -31,7 +31,6 @@ const nodes_client = require('../node_services/nodes_client'); const system_store = require('../system_services/system_store').get_instance(); const { BucketStatsStore } = require('../analytic_services/bucket_stats_store'); const { EndpointStatsStore } = require('../analytic_services/endpoint_stats_store'); -const events_dispatcher = require('./events_dispatcher'); const { IoStatsStore } = require('../analytic_services/io_stats_store'); const { ChunkAPI } = require('../../sdk/map_api_types'); const config = require('../../../config'); @@ -1744,14 +1743,6 @@ function check_quota(bucket) { } } -async function dispatch_triggers(req) { - load_bucket(req); - const triggers_to_run = events_dispatcher.get_triggers_for_event(req.bucket, req.rpc_params.obj, req.rpc_params.event_name); - if (triggers_to_run.length === 0) return; - setTimeout(() => events_dispatcher.run_bucket_triggers( - triggers_to_run, req.bucket, req.rpc_params.obj, req.account._id, req.auth_token), 1000); -} - function _parse_version_seq_from_version_id(version_str) { return (version_str.startsWith('nbver-') && Number(version_str.slice(6))) || undefined; @@ -2199,7 +2190,6 @@ exports.update_endpoint_stats = update_endpoint_stats; exports.put_object_tagging = put_object_tagging; exports.get_object_tagging = get_object_tagging; exports.delete_object_tagging = delete_object_tagging; -exports.dispatch_triggers = dispatch_triggers; // object lock exports.put_object_legal_hold = put_object_legal_hold; exports.get_object_legal_hold = get_object_legal_hold; diff --git a/src/server/server_rpc.js b/src/server/server_rpc.js index 780ce52652..1639912e58 100644 --- a/src/server/server_rpc.js +++ b/src/server/server_rpc.js @@ -120,14 +120,6 @@ class ServerRpc { require('./object_services/object_server'), options); } - register_func_services() { - const rpc = this.rpc; - const schema = rpc.schema; - const options = this.get_server_options(); - rpc.register_service(schema.func_api, - require('./func_services/func_server'), options); - } - register_bg_services() { const rpc = this.rpc; const schema = rpc.schema; diff --git a/src/server/system_services/account_server.js b/src/server/system_services/account_server.js index 98a5d6be4d..abc33d6fee 100644 --- a/src/server/system_services/account_server.js +++ b/src/server/system_services/account_server.js @@ -17,7 +17,6 @@ const SensitiveString = require('../../util/sensitive_string'); const cloud_utils = require('../../util/cloud_utils'); const auth_server = require('../common_services/auth_server'); const system_store = require('../system_services/system_store').get_instance(); -const bucket_server = require('../system_services/bucket_server'); const pool_server = require('../system_services/pool_server'); const azure_storage = require('../../util/azure_storage_wrap'); const NetStorage = require('../../util/NetStorageKit-Node-master/lib/netstorage'); @@ -463,12 +462,6 @@ function update_account_s3_access(req) { account: account._id, desc: desc_string.join('\n'), }); - if (removed_buckets.length) { - _.forEach(removed_buckets, bucket_name => { - const bucket = req.system.buckets_by_name[bucket_name.unwrap()]; - bucket_server.check_for_lambda_permission_issue(req, bucket, [account]); - }); - } }); } diff --git a/src/server/system_services/bucket_server.js b/src/server/system_services/bucket_server.js index dd11df7d83..0424204e2a 100644 --- a/src/server/system_services/bucket_server.js +++ b/src/server/system_services/bucket_server.js @@ -26,7 +26,6 @@ const cloud_utils = require('../../util/cloud_utils'); const nodes_client = require('../node_services/nodes_client'); const pool_server = require('../system_services/pool_server'); const system_store = require('../system_services/system_store').get_instance(); -const func_store = require('../func_services/func_store'); const replication_store = require('../system_services/replication_store'); const node_allocator = require('../node_services/node_allocator'); const azure_storage = require('../../util/azure_storage_wrap'); @@ -47,8 +46,6 @@ const VALID_BUCKET_NAME_REGEXP = const EXTERNAL_BUCKET_LIST_TO = 30 * 1000; //30s const EXTERNAL_BUCKET_ENCRYPTION = 30 * 1000; //30s -const trigger_properties = ['event_name', 'object_prefix', 'object_suffix']; - function new_bucket_defaults(name, system_id, tiering_policy_id, owner_account_id, tag, lock_enabled) { const now = Date.now(); return { @@ -69,7 +66,6 @@ function new_bucket_defaults(name, system_id, tiering_policy_id, owner_account_i last_update: (Math.floor(now / config.MD_AGGREGATOR_INTERVAL) * config.MD_AGGREGATOR_INTERVAL) - (2 * config.MD_GRACE_IN_MILLISECONDS), }, - lambda_triggers: [], versioning: config.WORM_ENABLED && lock_enabled ? 'ENABLED' : 'DISABLED', object_lock_configuration: config.WORM_ENABLED ? { object_lock_enabled: lock_enabled ? 'Enabled' : 'Disabled', @@ -699,10 +695,6 @@ async function read_bucket_sdk_info(req) { name: bucket.name, website: bucket.website, s3_policy: bucket.s3_policy, - active_triggers: _.map( - _.filter(bucket.lambda_triggers, 'enabled'), - trigger => _.pick(trigger, trigger_properties) - ), system_owner: bucket.system.owner.email, bucket_owner: bucket.owner_account.email, bucket_info: await P.map_props({ @@ -710,7 +702,6 @@ async function read_bucket_sdk_info(req) { nodes_aggregate_pool: bucket.tiering && nodes_client.instance().aggregate_nodes_by_pool(pool_names, system._id), hosts_aggregate_pool: bucket.tiering && nodes_client.instance().aggregate_hosts_by_pool(null, system._id), // num_of_objects: MDStore.instance().count_objects_of_bucket(bucket._id), - func_configs: get_bucket_func_configs(req, bucket), unused_refresh_tiering_alloc: bucket.tiering && node_allocator.refresh_tiering_alloc(bucket.tiering), }) .then(get_bucket_info), @@ -925,24 +916,6 @@ async function update_buckets(req) { P.map(update_alerts, alert => Dispatcher.instance().alert(alert.sev, alert.sysid, alert.alert, alert.rule)); } - -function check_for_lambda_permission_issue(req, bucket, removed_accounts) { - if (!removed_accounts.length) return; - if (!bucket.lambda_triggers || !bucket.lambda_triggers.length) return; - _.forEach(bucket.lambda_triggers, trigger => - func_store.instance().read_func(req.system._id, trigger.func_name, trigger.func_version) - .then(func => { - const account = _.find(removed_accounts, acc => acc._id.toString() === func.exec_account.toString()); - if (account) { - Dispatcher.instance().alert('MAJOR', req.system._id, - `Account’s ${account.email.unwrap()} ${bucket.name.unwrap()} bucket access was removed. - The configured lambda trigger for function ${func.name} will no longer be invoked`); - } - }) - ); -} - - async function delete_bucket_and_objects(req) { const bucket = find_bucket(req); @@ -1348,86 +1321,6 @@ async function get_cloud_buckets(req) { } } -/** - * - * ADD_BUCKET_LAMBDA_TRIGGER - * - */ -async function add_bucket_lambda_trigger(req) { - dbg.log0('add new bucket lambda trigger', req.rpc_params); - const new_trigger = req.rpc_params; - new_trigger.func_version = new_trigger.func_version || '$LATEST'; - const bucket = find_bucket(req, req.rpc_params.bucket_name); - await validate_trigger_update(bucket, new_trigger); - const trigger = _.omitBy({ - _id: system_store.new_system_store_id(), - event_name: new_trigger.event_name, - func_name: new_trigger.func_name, - func_version: new_trigger.func_version, - enabled: new_trigger.enabled !== false, - object_prefix: new_trigger.object_prefix || undefined, - object_suffix: new_trigger.object_suffix || undefined, - attempts: new_trigger.attempts || undefined, - }, _.isUndefined); - await system_store.make_changes({ - update: { - buckets: [{ - _id: bucket._id, - $push: { lambda_triggers: trigger }, - }] - } - }); -} - -/** - * - * DELETE_BUCKET_LAMBDA_TRIGGER - * - */ -async function delete_bucket_lambda_trigger(req) { - dbg.log0('delete bucket lambda trigger', req.rpc_params); - const trigger_id = req.rpc_params.id; - const bucket = find_bucket(req, req.rpc_params.bucket_name); - const trigger = bucket.lambda_triggers.find(trig => trig._id.toString() === trigger_id); - if (!trigger) { - throw new RpcError('NO_SUCH_TRIGGER', 'This trigger does not exists: ' + trigger_id); - } - await system_store.make_changes({ - update: { - buckets: [{ - _id: bucket._id, - $pull: { - lambda_triggers: { - _id: trigger._id - } - } - }] - } - }); -} - -async function update_bucket_lambda_trigger(req) { - dbg.log0('update bucket lambda trigger', req.rpc_params); - const updates = _.pick(req.rpc_params, 'event_name', 'func_name', 'func_version', 'enabled', 'object_prefix', 'object_suffix', 'attempts'); - if (_.isEmpty(updates)) return; - updates.func_version = updates.func_version || '$LATEST'; - const bucket = find_bucket(req, req.rpc_params.bucket_name); - const trigger = _.find(bucket.lambda_triggers, trig => trig._id.toString() === req.rpc_params.id); - if (!trigger) { - throw new RpcError('NO_SUCH_TRIGGER', 'This trigger does not exists: ' + req.rpc_params._id); - } - const validate_trigger = { ...trigger, ...updates }; - await validate_trigger_update(bucket, validate_trigger); - await system_store.make_changes({ - update: { - buckets: [{ - $find: { _id: bucket._id, 'lambda_triggers._id': trigger._id }, - $set: _.mapKeys(updates, (value, key) => `lambda_triggers.$.${key}`) - }] - } - }); -} - async function update_all_buckets_default_pool(req) { const pool_name = req.rpc_params.pool_name; @@ -1542,28 +1435,6 @@ function validate_nsfs_bucket(req) { } } -function validate_trigger_update(bucket, validated_trigger) { - dbg.log0('validate_trigger_update: Checking new trigger is legal:', validated_trigger); - let validate_function = true; - _.forEach(bucket.lambda_triggers, trigger => { - const is_same_trigger = String(trigger._id) === String(validated_trigger._id); - const is_same_func = - validated_trigger.func_name === trigger.func_name && - validated_trigger.func_version === trigger.func_version; - if (is_same_trigger && is_same_func) validate_function = false; // if this is update and function didn't change - don't validate - if (!is_same_trigger && - is_same_func && - trigger.event_name === validated_trigger.event_name && - trigger.object_prefix === validated_trigger.object_prefix && - trigger.object_suffix === validated_trigger.object_suffix && - trigger.attempts === validated_trigger.attempts) { - throw new RpcError('TRIGGER_DUPLICATE', 'This trigger is the same as an existing one'); - } - }); - if (!validate_function) return P.resolve(); // if update doesn't change function - no need to validate access - return func_store.instance().read_func(bucket.system._id, validated_trigger.func_name, validated_trigger.func_version); -} - function _inject_usage_to_cloud_bucket(target_name, endpoint, usage_list) { const res = { name: target_name @@ -1593,7 +1464,6 @@ function get_bucket_info({ bucket, nodes_aggregate_pool, hosts_aggregate_pool, - func_configs, bucket_stats = undefined, unused_refresh_tiering_alloc = undefined, }) { @@ -1682,13 +1552,6 @@ function get_bucket_info({ info.tiering.mode = calc_bucket_mode(tiering.tiers, metrics, ignore_quota); } - info.triggers = _.map(bucket.lambda_triggers, trigger => { - const ret_trigger = _.omit(trigger, '_id'); - ret_trigger.id = trigger._id.toString(); - ret_trigger.permission_problem = true; - return ret_trigger; - }); - if (bucket_stats) { info.stats = { reads: bucket_stats.total_reads || 0, @@ -1852,18 +1715,6 @@ function _calc_metrics({ }; } -function get_bucket_func_configs(req, bucket) { - if (!bucket.lambda_triggers || !bucket.lambda_triggers.length) return; - return P.map(bucket.lambda_triggers, trigger => - server_rpc.client.func.read_func({ - name: trigger.func_name, - version: trigger.func_version - }, { - auth_token: req.auth_token - }) - .then(func_info => func_info.config)); -} - function calc_namespace_bucket_mode(namespace_dict) { const rrs = namespace_dict.read_resources; @@ -2234,11 +2085,7 @@ exports.export_bucket_bandwidth_usage = export_bucket_bandwidth_usage; exports.get_bucket_throughput_usage = get_bucket_throughput_usage; exports.get_objects_size_histogram = get_objects_size_histogram; exports.get_buckets_stats_by_content_type = get_buckets_stats_by_content_type; -//Triggers -exports.add_bucket_lambda_trigger = add_bucket_lambda_trigger; -exports.update_bucket_lambda_trigger = update_bucket_lambda_trigger; -exports.delete_bucket_lambda_trigger = delete_bucket_lambda_trigger; -exports.check_for_lambda_permission_issue = check_for_lambda_permission_issue; +//Tagging exports.delete_bucket_tagging = delete_bucket_tagging; exports.put_bucket_tagging = put_bucket_tagging; exports.get_bucket_tagging = get_bucket_tagging; diff --git a/src/server/system_services/pool_server.js b/src/server/system_services/pool_server.js index 8795406613..15e43f67b8 100644 --- a/src/server/system_services/pool_server.js +++ b/src/server/system_services/pool_server.js @@ -22,7 +22,6 @@ const auth_server = require('../common_services/auth_server'); const HistoryDataStore = require('../analytic_services/history_data_store').HistoryDataStore; const IoStatsStore = require('../analytic_services/io_stats_store').IoStatsStore; const pool_ctrls = require('./pool_controllers'); -const func_store = require('../func_services/func_store'); const { KubeStore } = require('../kube-store.js'); @@ -739,13 +738,6 @@ async function delete_hosts_pool(req, pool) { pool: pool._id, desc: `${pool.name} was emptyed and deleted`, }); - const related_funcs = await func_store.instance().list_funcs_by_pool(req.system._id, pool._id); - for (const func of related_funcs) { - const new_pools_arr = func.pools.filter(function(obj) { - return obj.toString() !== (pool._id).toString(); - }); - await func_store.instance().update_func(func._id, { 'pools': new_pools_arr }); - } } } diff --git a/src/server/system_services/schemas/bucket_schema.js b/src/server/system_services/schemas/bucket_schema.js index 5d96276b45..4d4f3fae39 100644 --- a/src/server/system_services/schemas/bucket_schema.js +++ b/src/server/system_services/schemas/bucket_schema.js @@ -232,43 +232,6 @@ module.exports = { s3_policy: { $ref: 'common_api#/definitions/bucket_policy', }, - lambda_triggers: { - type: 'array', - items: { - type: 'object', - required: ['_id', 'event_name', 'func_name', 'func_version', 'enabled'], - properties: { - _id: { - objectid: true - }, - event_name: { - enum: ['ObjectCreated', 'ObjectRemoved', 'ObjectRead' /* 'ObjectCreated:Put', 'ObjectCreated:CompleteMultipartUpload', ... */ ], - type: 'string' - }, - func_name: { - type: 'string' - }, - func_version: { - type: 'string' - }, - enabled: { - type: 'boolean', - }, - last_run: { - idate: true - }, - object_prefix: { - type: 'string' - }, - object_suffix: { - type: 'string' - }, - attempts: { - type: 'integer' - }, - } - } - }, stats: { type: 'object', properties: { diff --git a/src/server/system_services/system_server.js b/src/server/system_services/system_server.js index 26fb9b3ae0..3542710430 100644 --- a/src/server/system_services/system_server.js +++ b/src/server/system_services/system_server.js @@ -491,7 +491,6 @@ async function read_system(req) { nodes_aggregate_pool_with_cloud_no_mongo, hosts_aggregate_pool, accounts, - funcs, buckets_stats, endpoint_groups } = await P.map_props({ @@ -517,15 +516,6 @@ async function read_system(req) { refresh_system_alloc_unused: node_allocator.refresh_system_alloc(system), - funcs: P.resolve() - // using default domain - will serve the list_funcs from web_server so if - // endpoint is down it will not fail the read_system - .then(() => server_rpc.client.func.list_funcs({}, { - auth_token: req.auth_token, - domain: 'default' - })) - .then(res => res.functions), - buckets_stats: BucketStatsStore.instance().get_all_buckets_stats({ system: system._id }), endpoint_groups: _get_endpoint_groups() @@ -605,12 +595,10 @@ async function read_system(req) { bucket => { const tiering_pools_status = node_allocator.get_tiering_status(bucket.tiering); Object.assign(tiering_status_by_tier, tiering_pools_status); - const func_configs = funcs.map(func => func.config); const b = bucket_server.get_bucket_info({ bucket, nodes_aggregate_pool: nodes_aggregate_pool_with_cloud_and_mongo, hosts_aggregate_pool, - func_configs, bucket_stats: stats_by_bucket[bucket.name], }); @@ -626,7 +614,6 @@ async function read_system(req) { nodes_aggregate_pool_with_cloud_and_mongo, tiering_status_by_tier[String(tier._id)])), accounts: accounts, - functions: funcs, storage: size_utils.to_bigint_storage(_.defaults({ used: objects_sys.size, }, nodes_aggregate_pool_with_cloud_no_mongo.storage, SYS_STORAGE_DEFAULTS)), diff --git a/src/server/web_server.js b/src/server/web_server.js index 8c26b66236..c0efc683a7 100755 --- a/src/server/web_server.js +++ b/src/server/web_server.js @@ -58,7 +58,6 @@ async function main() { server_rpc.register_system_services(); server_rpc.register_node_services(); server_rpc.register_object_services(); - server_rpc.register_func_services(); server_rpc.register_common_services(); server_rpc.rpc.router.default = 'fcall://fcall'; server_rpc.rpc.register_http_app(app); diff --git a/src/test/framework/system_tests_list.js b/src/test/framework/system_tests_list.js index 47d060d570..1da89fefaa 100644 --- a/src/test/framework/system_tests_list.js +++ b/src/test/framework/system_tests_list.js @@ -49,14 +49,6 @@ const tests = [ agent_cpu: '250m', agent_mem: '150Mi' }, - { - name: 'test-bucket-lambda-triggers', - test: './src/test/system_tests/test_bucket_lambda_triggers.js', - server_cpu: '400m', - server_mem: '400Mi', - agent_cpu: '250m', - agent_mem: '150Mi' - } ]; module.exports = tests; diff --git a/src/test/lambda/create_backup_file_func.js b/src/test/lambda/create_backup_file_func.js deleted file mode 100644 index 3094291d87..0000000000 --- a/src/test/lambda/create_backup_file_func.js +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -/* sample event - trigger structure: -{ - "Records": [{ - "s3": { - "bucket": { "name": "first.bucket" }, - "object": { "key": "Apple.jpg" } - } - }] -} -*/ - -const AWS = require('aws-sdk'); - -exports.handler = async function(event, context, callback) { - const srcBucket = event.Records[0].s3.bucket.name; - const key = event.Records[0].s3.object.key; - const s3 = new AWS.S3(); - - try { - const head = await s3.headObject({ - Bucket: srcBucket, - Key: key - }) - .promise(); - await s3.putObject({ - Bucket: srcBucket, - Key: key + '.json', - ContentType: 'text/json', - Body: JSON.stringify({ event, head }, null, 4), - }) - .promise(); - return callback(null); - } catch (err) { - return callback(err); - } -}; diff --git a/src/test/lambda/delete_backup_file_func.js b/src/test/lambda/delete_backup_file_func.js deleted file mode 100644 index 38779f86ea..0000000000 --- a/src/test/lambda/delete_backup_file_func.js +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const AWS = require('aws-sdk'); - -exports.handler = function(event, context, callback) { - const srcBucket = event.Records[0].s3.bucket.name; - const key = event.Records[0].s3.object.key; - const s3 = new AWS.S3(); - - s3.deleteObject({ - Bucket: srcBucket, - Key: key + '.json', - }) - .promise() - .then(() => callback(null)) - .catch(err => callback(err)); -}; diff --git a/src/test/lambda/denial_of_service_func.js b/src/test/lambda/denial_of_service_func.js deleted file mode 100644 index 1fba8a9a07..0000000000 --- a/src/test/lambda/denial_of_service_func.js +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const AWS = require('aws-sdk'); - -exports.handler = function(event, context, callback) { - const start = Date.now(); - const end = start + event.time; - let num_calls = 0; - let num_errors = 0; - let took = 0; - const lambda = new AWS.Lambda(event.lambda_conf); - - for (let i = 0; i < event.concur; ++i) { - worker(); - } - - function worker() { - const now = Date.now(); - if (now >= end) { - return callback(null, { - num_calls: num_calls, - num_errors: num_errors, - took: took, - }); - } - num_calls += 1; - lambda.invoke({ - FunctionName: event.func_name, - Payload: JSON.stringify(event.func_event), - }, function(err, res) { - if (err) { - num_errors += 1; - } else { - took += Date.now() - now; - } - setImmediate(worker); - }); - } -}; diff --git a/src/test/lambda/dos_event_sample.json b/src/test/lambda/dos_event_sample.json deleted file mode 100644 index 7cfd167ae2..0000000000 --- a/src/test/lambda/dos_event_sample.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1000, - "concur": 10, - "func_name": "word_count_func", - "func_event": { - "text": "The enemy is anybody who's going to get you killed, no matter which side he is on." - } -} diff --git a/src/test/lambda/test_lambda.sh b/src/test/lambda/test_lambda.sh deleted file mode 100755 index ada5f5561f..0000000000 --- a/src/test/lambda/test_lambda.sh +++ /dev/null @@ -1,37 +0,0 @@ -NAME="test_lambda" -ENDPOINT="http://localhost:6001" -PROFILE="noobaa" -AWS="aws --endpoint-url ${ENDPOINT} --profile ${PROFILE} --debug" - -echo "-----> Preparing ..." -rm ${NAME}_out -rm ${NAME}.zip -rm ${NAME}.js -cat >${NAME}.js < create-function ..." -${AWS} lambda create-function \ - --function-name ${NAME} \ - --runtime nodejs4.3 \ - --handler ${NAME}.handler \ - --role arn:aws:iam::112233445566:role/lambda-test \ - --zip-file fileb://${NAME}.zip - -echo -echo "-----> list-functions ..." -${AWS} lambda list-functions - -echo -echo "-----> invoke ..." -${AWS} lambda invoke \ - --function-name ${NAME} ${NAME}_out - -echo -echo "-----> print ..." -cat ${NAME}_out diff --git a/src/test/lambda/word_count_func.js b/src/test/lambda/word_count_func.js deleted file mode 100644 index ea006539db..0000000000 --- a/src/test/lambda/word_count_func.js +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const http = require('http'); -const https = require('https'); -const crypto = require('crypto'); - -exports.handler = function(event, context, callback) { - let text = ''; - - if (event.random) { - text = random_text(event.random); - return callback(null, count_text(text, event.return_text)); - } - - if (event.text) { - text = String(event.text); - return callback(null, count_text(text, event.return_text)); - } - - if (event.url) { - return (event.url.startsWith('https:') ? https : http) - .get(event.url, res => res - .setEncoding('utf8') - .on('data', data => { - text += data; - }) - .once('end', () => { - const reply = count_text(text, event.return_text); - reply.status_code = res.statusCode; - reply.headers = res.headers; - callback(null, reply); - }) - .once('error', err => callback(err)) - ) - .once('error', err => callback(err)); - } - - return callback(new Error('WordCount: Bad Event')); -}; - -function count_text(text, return_text) { - const words = text.match(/\S+/g); - const lines = text.match(/\n/g); - return { - bytes: Buffer.byteLength(text), - chars: text.length, - words: words ? words.length : 0, - lines: lines ? lines.length : 0, - text: return_text ? text : undefined, - }; -} - -function random_text(length) { - let str = ''; - const WORDSET = 'abcdefghijklmnopqrstuvwxyz'; - const CHARSET = WORDSET + ' '.repeat(0.2 * WORDSET.length) + '\n'.repeat(0.1 * WORDSET.length); - const cipher = crypto.createCipheriv('aes-128-gcm', crypto.randomBytes(16), crypto.randomBytes(12)); - const zero_buf = Buffer.alloc(Math.min(1024, length)); - while (length > 0) { - const rand_buf = cipher.update(zero_buf); - for (let i = 0; i < rand_buf.length; ++i) { - const b = rand_buf[i]; - str += CHARSET[b % CHARSET.length]; - } - length -= zero_buf.length; - } - return str; -} diff --git a/src/test/system_tests/mongodb_defaults.js b/src/test/system_tests/mongodb_defaults.js index a1948c286d..0fc78b28de 100644 --- a/src/test/system_tests/mongodb_defaults.js +++ b/src/test/system_tests/mongodb_defaults.js @@ -102,7 +102,6 @@ db.buckets.updateMany({}, { reads: 0, writes: 0 }, - lambda_triggers: [], versioning: 'DISABLED' } }); diff --git a/src/test/system_tests/sanity_build_test.js b/src/test/system_tests/sanity_build_test.js index 73a872a5c4..d6fc5d9388 100644 --- a/src/test/system_tests/sanity_build_test.js +++ b/src/test/system_tests/sanity_build_test.js @@ -125,9 +125,6 @@ async function create_configuration() { //Create various accounts await _create_accounts(); - //Create lambda funcs - await _create_lambda(); - await TEST_CTX.client.system.read_system(); } @@ -255,40 +252,6 @@ async function _create_accounts() { await TEST_CTX.client.account.update_account(ac5_update); } -async function _create_lambda() { - console.info('Creating Functions and Triggers'); - //Create func - const code_buffer = Buffer.from([80, 75, 3, 4, 10, 0, 0, 0, 8, 0, 217, 98, 219, 76, 166, 14, 198, 22, 88, 0, 0, 0, 119, 0, 0, 0, 7, 0, - 0, 0, 109, 97, 105, 110, 46, 106, 115, 75, 43, 205, 75, 46, 201, 204, 207, 83, 200, 77, 204, 204, 211, 208, 84, 168, 230, 82, 80, - 200, 73, 45, 81, 72, 84, 176, 85, 48, 180, 6, 114, 146, 243, 243, 138, 243, 115, 82, 245, 114, 242, 211, 53, 18, 60, 18, 21, 84, - 170, 19, 107, 19, 52, 65, 50, 137, 218, 218, 32, 42, 45, 63, 95, 3, 200, 175, 229, 226, 74, 131, 153, 5, 22, 2, 25, 117, 104, 1, - 178, 118, 117, 160, 184, 58, 68, 41, 0, 80, 75, 1, 2, 20, 0, 10, 0, 0, 0, 8, 0, 217, 98, 219, 76, 166, 14, 198, 22, 88, 0, 0, 0, - 119, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 97, 105, 110, 46, 106, 115, 80, 75, 5, 6, 0, 0, 0, 0, 1, - 0, 1, 0, 53, 0, 0, 0, 125, 0, 0, 0, 0, 0 - ]); - - await TEST_CTX.client.func.create_func({ - config: { - name: 'testfunc', - version: '$LATEST', - 'description': 'testytestytesyfunc', - 'runtime': 'nodejs6', - 'handler': 'main.main', - 'memory_size': 128, - 'timeout': 450 - }, - code: { zipfile_b64: code_buffer.toString('base64') } - }); - - //Create triggers - await TEST_CTX.client.bucket.add_bucket_lambda_trigger({ - bucket_name: 'first.bucket', - object_suffix: '.dat', - func_name: 'testfunc', - event_name: 'ObjectRemoved' - }); -} - async function test_data() { //test against mirror & ec -> TEST_CTX.bucket_mirror diff --git a/src/test/system_tests/test_bucket_lambda_triggers.js b/src/test/system_tests/test_bucket_lambda_triggers.js deleted file mode 100644 index c38c150e10..0000000000 --- a/src/test/system_tests/test_bucket_lambda_triggers.js +++ /dev/null @@ -1,509 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -const argv = require('minimist')(process.argv); -const dbg = require('../../util/debug_module')(__filename); -if (argv.log_file) { - dbg.set_log_to_file(argv.log_file); -} -dbg.set_process_name('test_bucket_lambda_triggers'); - -const _ = require('lodash'); -const api = require('../../api'); -const P = require('../../util/promise'); -const dotenv = require('../../util/dotenv'); -const ops = require('../utils/basic_server_ops'); -const path = require('path'); -const rpc = api.new_rpc(); -const zip_utils = require('../../util/zip_utils'); -const assert = require('assert'); -const AWS = require('aws-sdk'); -const fs = require('fs'); -const test_utils = require('./test_utils'); - -dotenv.load(); - -const { - mgmt_ip = 'localhost', - mgmt_port = '8080', - s3_ip = 'localhost', - s3_port = '80', -} = argv; - - -/*** Config ***/ -const TIME_FOR_FUNC_TO_RUN = 15000; -const TIME_FOR_SDK_TO_UPDATE = 60000; -const NUM_OF_RETRIES = 10; -const POOL_NAME = 'test-pool'; - -const client = rpc.new_client({ - address: `ws://${mgmt_ip}:${mgmt_port}` -}); - -const full_access_user = { - name: 'full_access', - email: 'full_access@noobaa.com', - password: 'master', - has_login: true, - s3_access: true, - default_resource: POOL_NAME, -}; - -const bucket1_user = { - name: 'bucket1_access', - email: 'bucket1_access@noobaa.com', - password: 'onlyb1', - has_login: true, - s3_access: true, - default_resource: POOL_NAME, -}; - -const external_connection = { - auth_method: 'AWS_V2', - endpoint: `http://${s3_ip}:${s3_port}`, - endpoint_type: 'S3_COMPATIBLE', - identity: '123', - secret: 'abc', - name: 'conn1' -}; - -const ROLE_ARN = 'arn:aws:iam::112233445566:role/lambda-test'; - -const trigger_based_func_create = { - FunctionName: 'create_backup_file', - Description: 'will create second copy of the same file', - Runtime: 'nodejs6', - Handler: 'create_backup_file_func.handler', - Role: ROLE_ARN, - MemorySize: 128, - VpcConfig: { - SubnetIds: [POOL_NAME] - }, - Files: [{ - path: 'create_backup_file_func.js', - fs_path: path.join(__dirname, '../lambda/create_backup_file_func.js'), - }] -}; - -const trigger_based_func_delete = { - FunctionName: 'delete_backup_file', - Description: 'will create second copy of the same file', - Runtime: 'nodejs6', - Handler: 'delete_backup_file_func.handler', - Role: ROLE_ARN, - MemorySize: 128, - VpcConfig: { - SubnetIds: [POOL_NAME] - }, - Files: [{ - path: 'delete_backup_file_func.js', - fs_path: path.join(__dirname, '../lambda/delete_backup_file_func.js'), - }] -}; - -const trigger_based_func_read = { - FunctionName: 'read_backup_file', - Description: 'will create second copy of the same file', - Runtime: 'nodejs6', - Handler: 'create_backup_file_func.handler', - Role: ROLE_ARN, - MemorySize: 128, - VpcConfig: { - SubnetIds: [POOL_NAME] - }, - Files: [{ - path: 'create_backup_file_func.js', - fs_path: path.join(__dirname, '../lambda/create_backup_file_func.js'), - }] -}; - - -/*** Utils ***/ - -async function authenticate() { - const auth_params = { - email: 'demo@noobaa.com', - password: 'DeMo1', - system: 'demo' - }; - await client.create_auth_token(auth_params); -} - -async function prepare_func(fn) { - const zipfile = await zip_utils.zip_from_files(fn.Files); - const zip_buffer = await zip_utils.zip_to_buffer(zipfile); - delete fn.Files; - fn.Code = { - ZipFile: zip_buffer - }; -} - -function get_new_server(user) { - const access_key = user.access_keys.access_key.unwrap(); - const secret_key = user.access_keys.secret_key.unwrap(); - return new AWS.S3({ - endpoint: `http://${s3_ip}:${s3_port}`, - s3ForcePathStyle: true, - accessKeyId: access_key, - secretAccessKey: secret_key, - maxRedirects: 10, - }); -} - -function get_new_lambda(user) { - const access_key = user.access_keys.access_key.unwrap(); - const secret_key = user.access_keys.secret_key.unwrap(); - return new AWS.Lambda({ - region: 'us-east-1', - endpoint: `http://${s3_ip}:${s3_port}`, - accessKeyId: access_key, - secretAccessKey: secret_key, - sslEnabled: false, - }); -} - -function account_by_name(accounts, email) { - return accounts.find(account => account.email.unwrap() === email); -} - -function bucket_by_name(buckets, name) { - return buckets.find(bucket => bucket.name.unwrap() === name); -} - -/*** Tests ***/ - -async function setup() { - const triggered_buckets = ['bucket1', 'ns.external.bucket1']; - let account; - let system_info = await client.system.read_system(); - // const internal_resource = system_info.pools.find(item => item.name.indexOf('system-internal') > -1).name; - if (!argv.no_setup) { - await test_utils.create_hosts_pool(client, POOL_NAME, 3); - - // Create test buckets. - await client.bucket.create_bucket({ name: 'bucket1' }); - await client.bucket.create_bucket({ name: 'bucket2' }); - - // //find the internal pool and set spillover on thge buckets - - // await client.bucket.update_bucket({ name: 'bucket1', spillover: internal_resource }); - // await client.bucket.update_bucket({ name: 'bucket2', spillover: internal_resource }); - - // Create NS buckets and resources - await client.bucket.create_bucket({ name: 'ns.internal.bucket1' }); - await client.bucket.create_bucket({ name: 'ns.internal.bucket2' }); - - // await client.bucket.update_bucket({ name: 'ns.internal.bucket1', spillover: internal_resource }); - // await client.bucket.update_bucket({ name: 'ns.internal.bucket2', spillover: internal_resource }); - - await client.account.add_external_connection(external_connection); - - await client.pool.create_namespace_resource({ - connection: external_connection.name, - name: 'ns.rsc.on.buck1', - target_bucket: 'ns.internal.bucket1' - }); - - await client.pool.create_namespace_resource({ - connection: external_connection.name, - name: 'ns.rsc.on.buck2', - target_bucket: 'ns.internal.bucket2' - }); - const nsr1 = { resource: 'ns.rsc.on.buck1' }; - const nsr2 = { resource: 'ns.rsc.on.buck2' }; - await client.bucket.create_bucket({ - name: 'ns.external.bucket1', - namespace: { - read_resources: [nsr1], - write_resource: nsr2 - } - }); - - // add new accounts: - await client.account.create_account(_.clone(full_access_user)); - await client.account.create_account(_.clone(bucket1_user)); - } - - system_info = await client.system.read_system(); - - account = account_by_name(system_info.accounts, full_access_user.email); - full_access_user.access_keys = account.access_keys[0]; - - account = account_by_name(system_info.accounts, bucket1_user.email); - bucket1_user.access_keys = account.access_keys[0]; - - return triggered_buckets; -} - -async function run_test() { - let trigger_id; - let last_run; - let last_run_new; - let system_info; - let bucket; - await authenticate(); - const target_buckets = await setup(); - - console.log(`Adding functions trigger_based_func_create, trigger_based_func_delete`); - await test_add_function(bucket1_user, trigger_based_func_create); - await test_add_function(full_access_user, trigger_based_func_delete); - await test_add_function(bucket1_user, trigger_based_func_read); - - for (const b of target_buckets) { - console.log(`Running test on ${b}`); - //Test create trigger - console.log(`Adding trigger for ${b}`); - await test_add_bucket_trigger('ObjectCreated', trigger_based_func_create, b); - system_info = await client.system.read_system(); - bucket = bucket_by_name(system_info.buckets, b); - trigger_id = bucket.triggers[0].id; - console.log(`testing trigger invoked for ${b}`); - await test_trigger_run_when_should(bucket1_user, 'file0.dat', b); - system_info = await client.system.read_system(); - bucket = bucket_by_name(system_info.buckets, b); - last_run = bucket.triggers[0].last_run; - - console.log(`testing trigger not invoked for ${b}`); - await test_trigger_dont_run_when_shouldnt(bucket1_user, 'file1.notdat', b); - console.log(`disabling bucket lambda trigger on ${b}`); - await client.bucket.update_bucket_lambda_trigger({ - bucket_name: b, - id: trigger_id, - enabled: false - }); - await P.delay(TIME_FOR_SDK_TO_UPDATE); - - console.log(`testing trigger not invoked after removed for ${b}`); - await test_trigger_dont_run_when_shouldnt(bucket1_user, 'file2.dat', b); - console.log(`enabling bucket lambda trigger on ${b}`); - await client.bucket.update_bucket_lambda_trigger({ - bucket_name: b, - id: trigger_id, - enabled: true - }); - await P.delay(TIME_FOR_SDK_TO_UPDATE); - await test_trigger_run_when_should(bucket1_user, 'file3.dat', b); - // Used for DeleteObjects later on - await test_trigger_run_when_should(bucket1_user, 'sloth_multiple.dat', b); - console.log(`changing bucket lambda trigger prefix to /bla. on ${b}`); - await client.bucket.update_bucket_lambda_trigger({ - bucket_name: b, - id: trigger_id, - object_prefix: '/bla' - }); - await P.delay(TIME_FOR_SDK_TO_UPDATE); - - await test_trigger_dont_run_when_shouldnt(bucket1_user, '/tmp/file4.dat', b); - console.log(`changing bucket lambda trigger prefix to /tmp on ${b}`); - await client.bucket.update_bucket_lambda_trigger({ - bucket_name: b, - id: trigger_id, - object_prefix: '/tmp' - }); - await P.delay(TIME_FOR_SDK_TO_UPDATE); - - await test_trigger_run_when_should(bucket1_user, '/tmp/file5.dat', b); - system_info = await client.system.read_system(); - console.log(`Checking that last_run of bucket_trigger has advanced for ${b}`); - bucket = bucket_by_name(system_info.buckets, b); - last_run_new = bucket.triggers[0].last_run; - assert(last_run_new > last_run, `expecting last run to advance but didn't. previous last_run: ${new Date(last_run)} new last_run: ${new Date(last_run_new)}`); - console.log(`last run has advanced as should for ${b}. previous last_run: ${new Date(last_run)} new last_run: ${new Date(last_run_new)}`); - - console.log(`Checking that multi delete triggers as should for ${b}`); - await test_trigger_run_when_should_multi(bucket1_user, b, '/tmp/multi-file', '.dat', 10); - // should fail as bucket1_user was removed from bucket access and is the runner of the function - await test_trigger_dont_run_when_shouldnt(full_access_user, '/tmp/file6.dat', b); - - console.log(`Checking object removed trigger for ${b}`); - await test_add_bucket_trigger('ObjectRemoved', trigger_based_func_delete, b); - await test_delete_trigger_run(full_access_user, 'file3.dat', b); - await test_delete_trigger_run(full_access_user, 'sloth_multiple.dat', b, /* multiple */ true); - } - - console.log('test_bucket_lambda_triggers PASSED'); -} - -async function test_add_function(user, func) { - const lambda = get_new_lambda(user); - await prepare_func(func); - - try { - await lambda.deleteFunction({ FunctionName: func.FunctionName }).promise(); - } catch (err) { - console.log('Delete function if exist:', func.FunctionName, err.message); - } - - await lambda.createFunction(func).promise(); - console.log('function created.'); -} - -async function test_add_bucket_trigger(type, func, bucketname) { - console.log(`adding trigger ${type} for ${bucketname}`); - await client.bucket.add_bucket_lambda_trigger({ - bucket_name: bucketname, - object_suffix: '.dat', - func_name: func.FunctionName, - event_name: type - }); - await P.delay(TIME_FOR_SDK_TO_UPDATE); - console.log('bucket lambda trigger created.'); -} - -async function test_trigger_run_when_should(user, file_param, bucketname) { - console.log(`test trigger run for ${bucketname}`); - const s3 = get_new_server(user); - let file_not_created = true; - let retries = 0; - const fname = await ops.generate_random_file(1); - const params1 = { - Bucket: bucketname, - Key: file_param, - Body: fs.createReadStream(fname) - }; - await s3.upload(params1).promise(); - while (retries < NUM_OF_RETRIES && file_not_created) { - const params2 = { - Bucket: bucketname, - Key: file_param + '.json' - }; - try { - await s3.headObject(params2).promise(); - file_not_created = false; - } catch (err) { - if (err.statusCode === 404) { - retries += 1; - console.log('file wasn\'t created yet...'); - await P.delay(TIME_FOR_FUNC_TO_RUN); - } else { - throw new Error(`expecting head to fail with statusCode 404 - File not found but got different error ${err}`); - } - } - } - - assert(file_not_created === false, `expecting file to be created but didn't uploaded ${bucketname}/${file_param} , retries ${retries}`); - console.log(`bucket lambda trigger worked. file created for: ${bucketname}/${file_param}`); -} - -async function test_trigger_dont_run_when_shouldnt(user, file_param, bucketname) { - console.log(`test trigger should not run for ${bucketname}`); - const s3 = get_new_server(user); - const fname = await ops.generate_random_file(1); - - const params1 = { - Bucket: bucketname, - Key: file_param, - Body: fs.createReadStream(fname) - }; - await s3.upload(params1).promise(); - await P.delay(TIME_FOR_FUNC_TO_RUN); - - const params2 = { - Bucket: bucketname, - Key: file_param + '.json' - }; - - try { - await s3.headObject(params2).promise(); - throw new Error(`expecting head to fail with statusCode 404 - File not found ${bucketname}/${file_param}`); - } catch (err) { - assert(err.statusCode === 404, 'expecting upload to fail with statusCode 404 - File not found' + err.statusCode); - console.log(`bucket lambda trigger worked. file not created for: ${bucketname}/${file_param}`); - } -} - -async function test_delete_trigger_run(user, file_param, bucketname, multiple) { - console.log(`test delete trigger run for ${bucketname}`); - const s3 = get_new_server(user); - const params = { - Bucket: bucketname, - Key: file_param - }; - const params2 = { - Bucket: bucketname, - Key: file_param + '.json' - }; - let file_not_deleted = true; - let retries = 0; - try { - await s3.headObject(params2).promise(); - } catch (err) { - console.log('json file not exist - test can\'t succeed:', file_param + '.json', err); - throw new Error(`expecting head to fail with statusCode 404 - File not found ${bucketname}/${file_param}`); - } - - if (multiple) { - await s3.deleteObjects({ - Bucket: bucketname, - Delete: { Objects: [_.pick(params, 'Key')] }, - }).promise(); - } else { - await s3.deleteObject(params).promise(); - } - - while (retries < NUM_OF_RETRIES && file_not_deleted) { - try { - await s3.headObject(params2).promise(); - retries += 1; - console.log('file wasn\'t deleted yet...'); - await P.delay(TIME_FOR_FUNC_TO_RUN); - } catch (err) { - if (err.statusCode === 404) { - file_not_deleted = false; - } else { - throw new Error(`expecting head to fail with statusCode 404 - File not found but got different error ${err}`); - } - } - } - - assert(file_not_deleted === false, `expecting file to be deleted but didn't ${bucketname}/${file_param} retries ${retries}`); - console.log(`bucket lambda trigger worked. file deleted for: ${bucketname}/${file_param}`); -} - -async function test_trigger_run_when_should_multi(user, bucketname, files_prefix, suffix, num_of_files) { - console.log(`test multi delete trigger should run for ${bucketname}`); - const s3 = get_new_server(user); - const names = []; - for (let i = 0; i < num_of_files; ++i) { - names.push(files_prefix + '_no_' + i + suffix); - } - const fname = await ops.generate_random_file(1); - - await P.map(names, name => { - const params1 = { - Bucket: bucketname, - Key: name, - Body: fs.createReadStream(fname) - }; - return s3.upload(params1).promise(); - }); - await P.delay(TIME_FOR_FUNC_TO_RUN * 2); // wait for the functions to run... - - const params2 = { - Bucket: bucketname, - Prefix: files_prefix - }; - const data = await s3.listObjects(params2).promise(); - - assert(data.Contents.length === (num_of_files * 2), `bucket ${bucketname} lambda trigger failed. files created: ${data.Contents}`); - console.log(`bucket lambda trigger worked. files ${data.Contents.length} created on ${bucketname}`); -} - -async function main() { - try { - await run_test(); - process.exit(0); - } catch (err) { - console.error('run_test failed with error:', err, err.stack); - process.exit(1); - } -} - -module.exports = { - run_test: run_test -}; - -if (require.main === module) { - main(); -} diff --git a/src/test/unit_tests/coretest.js b/src/test/unit_tests/coretest.js index c34b47184c..d47d32828c 100644 --- a/src/test/unit_tests/coretest.js +++ b/src/test/unit_tests/coretest.js @@ -127,7 +127,6 @@ function setup(options = {}) { server_rpc.register_bg_services(); server_rpc.register_hosted_agents_services(); server_rpc.register_object_services(); - server_rpc.register_func_services(); server_rpc.register_common_services(); server_rpc.rpc.set_request_logger(() => dbg.log1.apply(dbg, arguments)); _.each(server_rpc.rpc.router, (val, key) => { diff --git a/src/test/unit_tests/test_namespace_cache.js b/src/test/unit_tests/test_namespace_cache.js index d9fda92015..e03f41adee 100644 --- a/src/test/unit_tests/test_namespace_cache.js +++ b/src/test/unit_tests/test_namespace_cache.js @@ -354,7 +354,6 @@ class MockNamespace { async get_blob_block_lists(params, object_sdk) { return this.not_implemented(); } async set_triggers_for_bucket(params, object_sdk) { return this.not_implemented(); } async get_triggers_for_bucket(params, object_sdk) { return this.not_implemented(); } - async _dispatch_multiple_delete_triggers(params, object_sdk) { return this.not_implemented(); } async create_uls(params, object_sdk) { return this.not_implemented(); } async delete_uls(params, object_sdk) { return this.not_implemented(); } } diff --git a/src/test/unit_tests/test_system_servers.js b/src/test/unit_tests/test_system_servers.js index 4da2f80e6e..39df6a7c2f 100644 --- a/src/test/unit_tests/test_system_servers.js +++ b/src/test/unit_tests/test_system_servers.js @@ -12,7 +12,6 @@ const assert = require('assert'); const S3Auth = require('aws-sdk/lib/signers/s3'); const P = require('../../util/promise'); -const zip_utils = require('../../util/zip_utils'); const config = require('../../../config'); const check_deletion_ownership = require('../../server/system_services/pool_server').check_deletion_ownership; @@ -504,51 +503,6 @@ mocha.describe('system_servers', function() { } }); - mocha.it('lambda triggers works', async function() { - if (config.DB_TYPE !== 'mongodb') this.skip(); // eslint-disable-line no-invalid-this - this.timeout(90000); // eslint-disable-line no-invalid-this - const zipfile = await zip_utils.zip_from_files([{ - path: 'main.js', - data: ` - /* Copyright (C) 2016 NooBaa */ - 'use strict'; - exports.handler = function(event, context, callback) { - coretest.log('func event', event); - callback(); - }; - ` - }]); - const zipbuffer = await zip_utils.zip_to_buffer(zipfile); - await rpc_client.func.create_func({ - config: { - name: 'func1', - version: '$LATEST', - handler: 'main.handler' - }, - code: { zipfile_b64: zipbuffer.toString('base64') } - }); - await rpc_client.bucket.add_bucket_lambda_trigger({ - bucket_name: BUCKET, - event_name: 'ObjectCreated', - func_name: 'func1', - object_prefix: '/bla/' - }); - const bucket = await rpc_client.bucket.read_bucket({ name: BUCKET }); - await rpc_client.bucket.update_bucket_lambda_trigger({ - bucket_name: BUCKET, - id: bucket.triggers[0].id, - enabled: false - }); - await rpc_client.bucket.delete_bucket_lambda_trigger({ - bucket_name: BUCKET, - id: bucket.triggers[0].id, - }); - await rpc_client.func.delete_func({ - name: 'func1', - version: '$LATEST' - }); - }); - mocha.it('Calculate md5_etag for bucket works', async function() { this.timeout(90000); // eslint-disable-line no-invalid-this diff --git a/src/tools/mongodb_bucket_blow.js b/src/tools/mongodb_bucket_blow.js index 93aa0c464f..ba3a62bc65 100644 --- a/src/tools/mongodb_bucket_blow.js +++ b/src/tools/mongodb_bucket_blow.js @@ -66,7 +66,6 @@ for (let j = 0; j < 5; ++j) { objects_hist: [], last_update: Date.now() - (2 * 90000) }, - lambda_triggers: [], versioning: "DISABLED", last_update: now, }); diff --git a/src/upgrade/migration_to_postgres.js b/src/upgrade/migration_to_postgres.js index a7a3bd3ba8..6a22ea572b 100644 --- a/src/upgrade/migration_to_postgres.js +++ b/src/upgrade/migration_to_postgres.js @@ -24,8 +24,6 @@ const object_part_indexes = require('../server/object_services/schemas/object_pa const data_chunk_indexes = require('../server/object_services/schemas/data_chunk_indexes'); const data_block_indexes = require('../server/object_services/schemas/data_block_indexes'); const node_schema = require('../server/node_services/node_schema'); -const func_schema = require('../server/func_services/func_schema'); -const func_stats_schema = require('../server/func_services/func_stats_schema'); const bucket_stats_schema = require('../server/analytic_services/bucket_stats_schema'); const endpoint_group_report_schema = require('../server/analytic_services/endpoint_group_report_schema'); const s3_usage_schema = require('../server/analytic_services/s3_usage_schema'); @@ -162,20 +160,6 @@ const COLLECTIONS = [{ }, { // nodes_store name: 'nodes', schema: node_schema, -}, { // func_store - name: 'funcs', - schema: func_schema, - db_indexes: [{ - fields: { - system: 1, - name: 1, - version: 1, - deleted: 1, - }, - options: { - unique: true, - } - }] }, { // bucket_stats_store name: 'bucketstats', schema: bucket_stats_schema, @@ -227,16 +211,6 @@ const COLLECTIONS = [{ end_time: 1 } }], -}, { // func_stats_store - name: 'func_stats', - schema: func_stats_schema, - db_indexes: [{ - fields: { - system: 1, - id: 1, - latency_ms: 1, - } - }] }, { // activity_log_store name: 'activitylogs', schema: activity_log_schema,