From f257ba5285e78360714442f818825493dab1ef5e Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 04:34:39 +0200 Subject: [PATCH 01/58] added idempotency router and middleware --- resources/buildConfigDefinitions.js | 10 +++++ .../Storage/Mongo/MongoStorageAdapter.js | 5 ++- .../Postgres/PostgresStorageAdapter.js | 1 + src/Adapters/Storage/StorageAdapter.js | 3 +- src/Controllers/DatabaseController.js | 42 +++++++++++++++++++ src/Controllers/SchemaController.js | 14 +++++++ src/Options/Definitions.js | 31 ++++++++++++++ src/Options/docs.js | 9 ++++ src/Options/index.js | 15 +++++++ src/ParseServer.js | 2 + src/Routers/IdempotencyRouter.js | 21 ++++++++++ src/middlewares.js | 38 +++++++++++++++++ 12 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 src/Routers/IdempotencyRouter.js diff --git a/resources/buildConfigDefinitions.js b/resources/buildConfigDefinitions.js index 8215792a82..a640e1c3c7 100644 --- a/resources/buildConfigDefinitions.js +++ b/resources/buildConfigDefinitions.js @@ -52,6 +52,9 @@ function getENVPrefix(iface) { if (iface.id.name === 'LiveQueryOptions') { return 'PARSE_SERVER_LIVEQUERY_'; } + if (iface.id.name === 'IdempotencyOptions') { + return 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_'; + } } function processProperty(property, iface) { @@ -170,6 +173,13 @@ function parseDefaultValue(elt, value, t) { }); literalValue = t.objectExpression(props); } + if (type == 'IdempotencyOptions') { + const object = parsers.objectParser(value); + const props = Object.keys(object).map((key) => { + return t.objectProperty(key, object[value]); + }); + literalValue = t.objectExpression(props); + } if (type == 'ProtectedFields') { const prop = t.objectProperty( t.stringLiteral('_User'), t.objectPattern([ diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 7b8d2a3ef3..95d5956b4a 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -692,7 +692,8 @@ export class MongoStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - indexType: any = 1 + indexType: any = 1, + ttl: ?Number, ): Promise { schema = convertParseSchemaToMongoSchema(schema); const indexCreationRequest = {}; @@ -705,6 +706,7 @@ export class MongoStorageAdapter implements StorageAdapter { const defaultOptions: Object = { background: true, sparse: true }; const indexNameOptions: Object = indexName ? { name: indexName } : {}; + const ttlOptions: Object = ttl !== undefined ? { expireAfterSeconds: ttl } : {}; const caseInsensitiveOptions: Object = caseInsensitive ? { collation: MongoCollection.caseInsensitiveCollation() } : {}; @@ -712,6 +714,7 @@ export class MongoStorageAdapter implements StorageAdapter { ...defaultOptions, ...caseInsensitiveOptions, ...indexNameOptions, + ...ttlOptions, }; return this._adaptiveCollection(className) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 6b58120778..5727d02c4b 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1209,6 +1209,7 @@ export class PostgresStorageAdapter implements StorageAdapter { '_GlobalConfig', '_GraphQLConfig', '_Audience', + '_Idempotency', ...results.map((result) => result.className), ...joins, ]; diff --git a/src/Adapters/Storage/StorageAdapter.js b/src/Adapters/Storage/StorageAdapter.js index 0256841b23..e58deff991 100644 --- a/src/Adapters/Storage/StorageAdapter.js +++ b/src/Adapters/Storage/StorageAdapter.js @@ -93,7 +93,8 @@ export interface StorageAdapter { fieldNames: string[], indexName?: string, caseSensitive?: boolean, - indexType?: any + indexType?: any, + ttl?: number, ): Promise; ensureUniqueness( className: string, diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index b56a8d3efc..2a7d0a3bda 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -17,6 +17,7 @@ import type { QueryOptions, FullQueryOptions, } from '../Adapters/Storage/StorageAdapter'; +import Config from '../Config'; function addWriteACL(query, acl) { const newQuery = _.cloneDeep(query); @@ -1747,6 +1748,9 @@ class DatabaseController { const roleClassPromise = this.loadSchema().then((schema) => schema.enforceClassExists('_Role') ); + const idempotencyClassPromise = this.loadSchema().then((schema) => + schema.enforceClassExists('_Idempotency') + ); const usernameUniqueness = userClassPromise .then(() => @@ -1802,6 +1806,42 @@ class DatabaseController { throw error; }); + const idempotencyRequestIdUniqueness = idempotencyClassPromise + .then(() => { + return this.adapter.ensureUniqueness( + '_Idempotency', + requiredUserFields, + ['reqId'] + ); + }) + .catch((error) => { + logger.warn( + 'Unable to ensure uniqueness for idempotency request ID: ', + error + ); + throw error; + }); + + const idempotencyExpireIndex = idempotencyClassPromise + .then(() => { + return this.adapter.ensureIndex( + '_Idempotency', + requiredUserFields, + ['expire'], + 'ttl', + false, + undefined, + 0 + ) + }) + .catch((error) => { + logger.warn( + 'Unable to create TTL index for idempotency expire date: ', + error + ); + throw error; + }); + const roleUniqueness = roleClassPromise .then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']) @@ -1823,6 +1863,8 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, + idempotencyRequestIdUniqueness, + idempotencyExpireIndex, adapterInit, indexPromise, ]); diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 435a4b5570..d26e9bc2b8 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -144,6 +144,10 @@ const defaultColumns: { [string]: SchemaFields } = Object.freeze({ lastUsed: { type: 'Date' }, timesUsed: { type: 'Number' }, }, + _Idempotency: { + reqId: { type: 'String' }, + expire: { type: 'Date' }, + } }); const requiredColumns = Object.freeze({ @@ -161,6 +165,7 @@ const systemClasses = Object.freeze([ '_JobStatus', '_JobSchedule', '_Audience', + '_Idempotency' ]); const volatileClasses = Object.freeze([ @@ -171,6 +176,7 @@ const volatileClasses = Object.freeze([ '_GraphQLConfig', '_JobSchedule', '_Audience', + '_Idempotency' ]); // Anything that start with role @@ -660,6 +666,13 @@ const _AudienceSchema = convertSchemaToAdapterSchema( classLevelPermissions: {}, }) ); +const _IdempotencySchema = convertSchemaToAdapterSchema( + injectDefaultSchema({ + className: '_Idempotency', + fields: defaultColumns._Idempotency, + classLevelPermissions: {}, + }) +); const VolatileClassesSchemas = [ _HooksSchema, _JobStatusSchema, @@ -668,6 +681,7 @@ const VolatileClassesSchemas = [ _GlobalConfigSchema, _GraphQLConfigSchema, _AudienceSchema, + _IdempotencySchema ]; const dbTypeMatchesObjectType = ( diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 725002034c..ecf8bb1e77 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -115,6 +115,13 @@ module.exports.ParseServerOptions = { action: parsers.booleanParser, default: false, }, + idempotencyOptions: { + env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS', + help: + 'Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production.', + action: parsers.objectParser, + default: {}, + }, dotNetKey: { env: 'PARSE_SERVER_DOT_NET_KEY', help: 'Key for Unity and .Net SDK', @@ -523,3 +530,27 @@ module.exports.LiveQueryServerOptions = { action: parsers.moduleOrObjectParser, }, }; + +module.exports.IdempotencyOptions = { + functions: { + env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS', + help: 'Array of function names, use `*` for all, default is none.', + action: parsers.arrayParser, + }, + jobs: { + env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS', + help: 'Array of job names, use `*` for all, default is none.', + action: parsers.arrayParser, + }, + classes: { + env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES', + help: 'Array of class names, use `*` for all, default is none.', + action: parsers.arrayParser, + }, + ttl: { + env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL', + help: 'The duration in seconds after which a request record is discarded from the database, default is 300s.', + action: parsers.numberParser, + default: 300, + }, +}; diff --git a/src/Options/docs.js b/src/Options/docs.js index 9e5b6ac8f8..1fd3fb939b 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -36,6 +36,7 @@ * @property {Boolean} jsonLogs Log as structured JSON objects * @property {LiveQueryOptions} liveQuery parse-server's LiveQuery configuration object * @property {LiveQueryServerOptions} liveQueryServerOptions Live query server configuration options (will start the liveQuery server) + * @property {IdempotencyOptions} idempotencyOptions Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production. * @property {Adapter} loggerAdapter Adapter module for the logging sub-system * @property {String} logLevel Sets the level for logs * @property {String} logsFolder Folder for the logs (defaults to './logs'); set to null to disable file based logging @@ -110,3 +111,11 @@ * @property {Number} websocketTimeout Number of milliseconds between ping/pong frames. The WebSocket server sends ping/pong frames to the clients to keep the WebSocket alive. This value defines the interval of the ping/pong frame from the server to clients, defaults to 10 * 1000 ms (10 s). * @property {Adapter} wssAdapter Adapter module for the WebSocketServer */ + +/** + * @interface IdempotencyOptions + * @property {String[]} functions Array of function names, use `*` for all, default is none. + * @property {String[]} jobs Array of job names, use `*` for all, default is none. + * @property {String[]} classes Array of class names, use `*` for all, default is none. + * @property {Number} ttl The duration in seconds after which a request record is discarded from the database, default is 300s. + */ diff --git a/src/Options/index.js b/src/Options/index.js index 3aa5b7fc6b..4fc9aa102e 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -186,6 +186,10 @@ export interface ParseServerOptions { startLiveQueryServer: ?boolean; /* Live query server configuration options (will start the liveQuery server) */ liveQueryServerOptions: ?LiveQueryServerOptions; + /* Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production. + :ENV: PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS + :DEFAULT: false */ + idempotencyOptions: ?IdempotencyOptions; /* Full path to your GraphQL custom schema.graphql file */ graphQLSchema: ?string; /* Mounts the GraphQL endpoint @@ -270,3 +274,14 @@ export interface LiveQueryServerOptions { /* Adapter module for the WebSocketServer */ wssAdapter: ?Adapter; } + +export interface IdempotencyOptions { + /* Array of function names, use `*` for all, default is none. */ + functions: ?(string[]); + /* Array of job names, use `*` for all, default is none. */ + jobs: ?(string[]); + /* Array of class names, use `*` for all, default is none. */ + classes: ?(string[]); + /* The duration in seconds after which a request record is discarded from the database, default is 300s. */ + ttl: ?Number; +} diff --git a/src/ParseServer.js b/src/ParseServer.js index 081a8c15a4..9077ed6502 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -36,6 +36,7 @@ import { SessionsRouter } from './Routers/SessionsRouter'; import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; import { AudiencesRouter } from './Routers/AudiencesRouter'; +import { IdempotencyRouter } from './Routers/IdempotencyRouter'; import { AggregateRouter } from './Routers/AggregateRouter'; import { ParseServerRESTController } from './ParseServerRESTController'; import * as controllers from './Controllers'; @@ -231,6 +232,7 @@ class ParseServer { new HooksRouter(), new CloudCodeRouter(), new AudiencesRouter(), + new IdempotencyRouter(), new AggregateRouter(), ]; diff --git a/src/Routers/IdempotencyRouter.js b/src/Routers/IdempotencyRouter.js new file mode 100644 index 0000000000..427d08e370 --- /dev/null +++ b/src/Routers/IdempotencyRouter.js @@ -0,0 +1,21 @@ +import ClassesRouter from './ClassesRouter'; +import * as middleware from '../middlewares'; + +export class IdempotencyRouter extends ClassesRouter { + className() { + return '_Idempotency'; + } + + mountRoutes() { + this.route( + 'POST', + '/idempotency', + middleware.promiseEnforceMasterKeyAccess, + req => { + return this.handleCreate(req); + } + ); + } +} + +export default IdempotencyRouter; diff --git a/src/middlewares.js b/src/middlewares.js index 681053e8e9..737d95f0a9 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -4,6 +4,7 @@ import auth from './Auth'; import Config from './Config'; import ClientSDK from './ClientSDK'; import defaultLogger from './logger'; +import rest from './rest'; export const DEFAULT_ALLOWED_HEADERS = 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, Content-Type, Pragma, Cache-Control'; @@ -400,6 +401,43 @@ export function promiseEnforceMasterKeyAccess(request) { return Promise.resolve(); } +/** + * Deduplicates a request to ensure idempotency. Duplicates are determined by the request ID + * in the request header. If a request has no request ID, it is executed anyway. + * @param {*} req The request to evaluate. + * @returns Promise<{}> + */ +export function promiseEnsureIdempotency(req) { + // Get request ID + const requestId = req.headers["x-parse-request-id"]; + if (!requestId) { return Promise.resolve(); } + const { functions, jobs, classes, ttl } = req.config.idempotencyOptions; + // Determine whether idempotency is enabled for current request path + const split = req.path.match(/^\/([^\/]*)\/([^\/]*)/i); + const route = split[1]; + const item = split[2]; + const functionMatch = functions && route == "functions" && (functions.includes("*") || functions.includes(item)); + const jobMatch = jobs && route == "jobs" && (jobs.includes("*") || jobs.includes(item)); + const classMatch = classes && route == "classes" && (classes.includes("*") || classes.includes(item)); + if (!functionMatch && !jobMatch && !classMatch) { return Promise.resolve(); } + // Try to track request + const expiryDate = new Date(new Date().setSeconds(new Date().getSeconds() + ttl)); + return rest.create( + req.config, + auth.nobody(req.config), + '_Idempotency', + { reqId: requestId, expire: Parse._encode(expiryDate) } + ).catch (e => { + if (e.code == Parse.Error.DUPLICATE_VALUE) { + throw new Parse.Error( + Parse.Error.DUPLICATE_REQUEST, + 'Duplicate request' + ); + } + throw e; + }); +} + function invalidRequest(req, res) { res.status(403); res.end('{"error":"unauthorized"}'); From 2785a9152c1a01ca7f7890e68b79c45ddcfba271 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 04:41:39 +0200 Subject: [PATCH 02/58] added idempotency rules for routes classes, functions, jobs, installaions, users --- src/Routers/ClassesRouter.js | 5 +++-- src/Routers/FunctionsRouter.js | 4 +++- src/Routers/InstallationsRouter.js | 5 +++-- src/Routers/UsersRouter.js | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Routers/ClassesRouter.js b/src/Routers/ClassesRouter.js index d85101af6e..f91665af8e 100644 --- a/src/Routers/ClassesRouter.js +++ b/src/Routers/ClassesRouter.js @@ -2,6 +2,7 @@ import PromiseRouter from '../PromiseRouter'; import rest from '../rest'; import _ from 'lodash'; import Parse from 'parse/node'; +import { promiseEnsureIdempotency } from '../middlewares'; const ALLOWED_GET_QUERY_KEYS = [ 'keys', @@ -244,10 +245,10 @@ export class ClassesRouter extends PromiseRouter { this.route('GET', '/classes/:className/:objectId', req => { return this.handleGet(req); }); - this.route('POST', '/classes/:className', req => { + this.route('POST', '/classes/:className', promiseEnsureIdempotency, req => { return this.handleCreate(req); }); - this.route('PUT', '/classes/:className/:objectId', req => { + this.route('PUT', '/classes/:className/:objectId', promiseEnsureIdempotency, req => { return this.handleUpdate(req); }); this.route('DELETE', '/classes/:className/:objectId', req => { diff --git a/src/Routers/FunctionsRouter.js b/src/Routers/FunctionsRouter.js index 5e6f422718..6e590fb807 100644 --- a/src/Routers/FunctionsRouter.js +++ b/src/Routers/FunctionsRouter.js @@ -4,7 +4,7 @@ var Parse = require('parse/node').Parse, triggers = require('../triggers'); import PromiseRouter from '../PromiseRouter'; -import { promiseEnforceMasterKeyAccess } from '../middlewares'; +import { promiseEnforceMasterKeyAccess, promiseEnsureIdempotency } from '../middlewares'; import { jobStatusHandler } from '../StatusHandler'; import _ from 'lodash'; import { logger } from '../logger'; @@ -34,11 +34,13 @@ export class FunctionsRouter extends PromiseRouter { this.route( 'POST', '/functions/:functionName', + promiseEnsureIdempotency, FunctionsRouter.handleCloudFunction ); this.route( 'POST', '/jobs/:jobName', + promiseEnsureIdempotency, promiseEnforceMasterKeyAccess, function(req) { return FunctionsRouter.handleCloudJob(req); diff --git a/src/Routers/InstallationsRouter.js b/src/Routers/InstallationsRouter.js index a35afd9bb1..20d8d58eb3 100644 --- a/src/Routers/InstallationsRouter.js +++ b/src/Routers/InstallationsRouter.js @@ -2,6 +2,7 @@ import ClassesRouter from './ClassesRouter'; import rest from '../rest'; +import { promiseEnsureIdempotency } from '../middlewares'; export class InstallationsRouter extends ClassesRouter { className() { @@ -35,10 +36,10 @@ export class InstallationsRouter extends ClassesRouter { this.route('GET', '/installations/:objectId', req => { return this.handleGet(req); }); - this.route('POST', '/installations', req => { + this.route('POST', '/installations', promiseEnsureIdempotency, req => { return this.handleCreate(req); }); - this.route('PUT', '/installations/:objectId', req => { + this.route('PUT', '/installations/:objectId', promiseEnsureIdempotency, req => { return this.handleUpdate(req); }); this.route('DELETE', '/installations/:objectId', req => { diff --git a/src/Routers/UsersRouter.js b/src/Routers/UsersRouter.js index c2587bed6e..65825d8843 100644 --- a/src/Routers/UsersRouter.js +++ b/src/Routers/UsersRouter.js @@ -8,6 +8,7 @@ import rest from '../rest'; import Auth from '../Auth'; import passwordCrypto from '../password'; import { maybeRunTrigger, Types as TriggerTypes } from '../triggers'; +import { promiseEnsureIdempotency } from '../middlewares'; export class UsersRouter extends ClassesRouter { className() { @@ -442,7 +443,7 @@ export class UsersRouter extends ClassesRouter { this.route('GET', '/users', req => { return this.handleFind(req); }); - this.route('POST', '/users', req => { + this.route('POST', '/users', promiseEnsureIdempotency, req => { return this.handleCreate(req); }); this.route('GET', '/users/me', req => { @@ -451,7 +452,7 @@ export class UsersRouter extends ClassesRouter { this.route('GET', '/users/:objectId', req => { return this.handleGet(req); }); - this.route('PUT', '/users/:objectId', req => { + this.route('PUT', '/users/:objectId', promiseEnsureIdempotency, req => { return this.handleUpdate(req); }); this.route('DELETE', '/users/:objectId', req => { From 579abe5b247c71221eea595c3fc2ea02104634e5 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 05:17:43 +0200 Subject: [PATCH 03/58] fixed typo --- src/Adapters/Storage/Mongo/MongoStorageAdapter.js | 2 +- src/Options/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 95d5956b4a..70f3f0b325 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -693,7 +693,7 @@ export class MongoStorageAdapter implements StorageAdapter { indexName: ?string, caseInsensitive: boolean = false, indexType: any = 1, - ttl: ?Number, + ttl: ?number, ): Promise { schema = convertParseSchemaToMongoSchema(schema); const indexCreationRequest = {}; diff --git a/src/Options/index.js b/src/Options/index.js index 4fc9aa102e..82de1d9f0b 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -283,5 +283,5 @@ export interface IdempotencyOptions { /* Array of class names, use `*` for all, default is none. */ classes: ?(string[]); /* The duration in seconds after which a request record is discarded from the database, default is 300s. */ - ttl: ?Number; + ttl: ?number; } From 3f51df7368428664e465b07059d6a121f38a6b79 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 05:40:23 +0200 Subject: [PATCH 04/58] ignore requests without header --- src/middlewares.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares.js b/src/middlewares.js index 737d95f0a9..efbde79d6f 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -409,7 +409,7 @@ export function promiseEnforceMasterKeyAccess(request) { */ export function promiseEnsureIdempotency(req) { // Get request ID - const requestId = req.headers["x-parse-request-id"]; + const requestId = ((req || {}).headers || {})["x-parse-request-id"]; if (!requestId) { return Promise.resolve(); } const { functions, jobs, classes, ttl } = req.config.idempotencyOptions; // Determine whether idempotency is enabled for current request path From cdd4eb0d96e8fc8bfc4e864ce61569b2e4828687 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 06:00:05 +0200 Subject: [PATCH 05/58] removed unused var --- src/Controllers/DatabaseController.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 2a7d0a3bda..197d95e8b1 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -17,7 +17,6 @@ import type { QueryOptions, FullQueryOptions, } from '../Adapters/Storage/StorageAdapter'; -import Config from '../Config'; function addWriteACL(query, acl) { const newQuery = _.cloneDeep(query); From c6a7c45e5c9990f876a8c2192215973800f70ad5 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 06:22:08 +0200 Subject: [PATCH 06/58] enabled feature only for MongoDB --- src/middlewares.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/middlewares.js b/src/middlewares.js index efbde79d6f..78af1787be 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -5,6 +5,7 @@ import Config from './Config'; import ClientSDK from './ClientSDK'; import defaultLogger from './logger'; import rest from './rest'; +import MongoStorageAdapter from '../lib/Adapters/Storage/Mongo/MongoStorageAdapter'; export const DEFAULT_ALLOWED_HEADERS = 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, Content-Type, Pragma, Cache-Control'; @@ -408,6 +409,8 @@ export function promiseEnforceMasterKeyAccess(request) { * @returns Promise<{}> */ export function promiseEnsureIdempotency(req) { + // Disable for Postgres + if (!(req.config.database.adapter instanceof MongoStorageAdapter)) { return Promise.resolve(); } // Get request ID const requestId = ((req || {}).headers || {})["x-parse-request-id"]; if (!requestId) { return Promise.resolve(); } From 4557fe90ed76bdbb0be4a81b72f46e3c22728476 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 06:23:34 +0200 Subject: [PATCH 07/58] changed code comment --- src/middlewares.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares.js b/src/middlewares.js index 78af1787be..12ce78aac6 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -409,7 +409,7 @@ export function promiseEnforceMasterKeyAccess(request) { * @returns Promise<{}> */ export function promiseEnsureIdempotency(req) { - // Disable for Postgres + // Enable feature only for MongoDB if (!(req.config.database.adapter instanceof MongoStorageAdapter)) { return Promise.resolve(); } // Get request ID const requestId = ((req || {}).headers || {})["x-parse-request-id"]; From e3f40d18d3e6cb3ff2e837401720ccfad2faab21 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 18:01:37 +0200 Subject: [PATCH 08/58] fixed inconsistend storage adapter specification --- spec/ParseQuery.Aggregate.spec.js | 2 +- src/Adapters/Storage/Mongo/MongoStorageAdapter.js | 7 +++---- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 4 ++-- src/Adapters/Storage/StorageAdapter.js | 3 +-- src/Controllers/DatabaseController.js | 3 +-- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index b7dd291d1c..d9a684d0a9 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -1440,7 +1440,7 @@ describe('Parse.Query Aggregate testing', () => { ['location'], 'geoIndex', false, - '2dsphere' + { indexType: '2dsphere' }, ); // Create objects const GeoObject = Parse.Object.extend('GeoObject'); diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 70f3f0b325..9c08d3a073 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -692,8 +692,7 @@ export class MongoStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - indexType: any = 1, - ttl: ?number, + options?: Object = {}, ): Promise { schema = convertParseSchemaToMongoSchema(schema); const indexCreationRequest = {}; @@ -701,12 +700,12 @@ export class MongoStorageAdapter implements StorageAdapter { transformKey(className, fieldName, schema) ); mongoFieldNames.forEach((fieldName) => { - indexCreationRequest[fieldName] = indexType; + indexCreationRequest[fieldName] = options.indexType !== undefined ? options.indexType : 1; }); const defaultOptions: Object = { background: true, sparse: true }; const indexNameOptions: Object = indexName ? { name: indexName } : {}; - const ttlOptions: Object = ttl !== undefined ? { expireAfterSeconds: ttl } : {}; + const ttlOptions: Object = options.ttl !== undefined ? { expireAfterSeconds: options.ttl } : {}; const caseInsensitiveOptions: Object = caseInsensitive ? { collation: MongoCollection.caseInsensitiveCollation() } : {}; diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 5727d02c4b..ce7b746f4f 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2577,9 +2577,9 @@ export class PostgresStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - conn: ?any = null + options?: Object = {}, ): Promise { - conn = conn != null ? conn : this._client; + const conn = options.conn !== undefined ? options.conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName != null ? { name: indexName } : { name: defaultIndexName }; diff --git a/src/Adapters/Storage/StorageAdapter.js b/src/Adapters/Storage/StorageAdapter.js index e58deff991..7031c21d7f 100644 --- a/src/Adapters/Storage/StorageAdapter.js +++ b/src/Adapters/Storage/StorageAdapter.js @@ -93,8 +93,7 @@ export interface StorageAdapter { fieldNames: string[], indexName?: string, caseSensitive?: boolean, - indexType?: any, - ttl?: number, + options?: Object, ): Promise; ensureUniqueness( className: string, diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 197d95e8b1..2f359e8708 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1829,8 +1829,7 @@ class DatabaseController { ['expire'], 'ttl', false, - undefined, - 0 + { ttl: 0 }, ) }) .catch((error) => { From 827e0a8c3bd3a2d742c77c65aaeded3593ed1f6f Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 18:21:25 +0200 Subject: [PATCH 09/58] Trigger notification From d6c15a15ec2b174e7bae2ef4d27f7f15d53c7089 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 18:34:45 +0200 Subject: [PATCH 10/58] Travis CI trigger From 472a84c3d0359b2e554b543a3e8cda104490da3d Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 18:52:02 +0200 Subject: [PATCH 11/58] Travis CI trigger From bd8333f155af7742c59fc0270a4c99aeb569b32a Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 22:21:52 +0200 Subject: [PATCH 12/58] Travis CI trigger From 7b0e93de4ac98df036edc70d1a09e29e66e6c13d Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 23:15:44 +0200 Subject: [PATCH 13/58] rebuilt option definitions --- src/Options/Definitions.js | 1052 ++++++++++++++++++------------------ src/Options/docs.js | 5 +- 2 files changed, 515 insertions(+), 542 deletions(-) diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index ecf8bb1e77..24a7044435 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -3,554 +3,526 @@ This code has been generated by resources/buildConfigDefinitions.js Do not edit manually, but update Options/index.js */ -var parsers = require('./parsers'); +var parsers = require("./parsers"); module.exports.ParseServerOptions = { - accountLockout: { - env: 'PARSE_SERVER_ACCOUNT_LOCKOUT', - help: 'account lockout policy for failed login attempts', - action: parsers.objectParser, - }, - allowClientClassCreation: { - env: 'PARSE_SERVER_ALLOW_CLIENT_CLASS_CREATION', - help: 'Enable (or disable) client class creation, defaults to true', - action: parsers.booleanParser, - default: true, - }, - allowCustomObjectId: { - env: 'PARSE_SERVER_ALLOW_CUSTOM_OBJECT_ID', - help: 'Enable (or disable) custom objectId', - action: parsers.booleanParser, - default: false, - }, - allowHeaders: { - env: 'PARSE_SERVER_ALLOW_HEADERS', - help: 'Add headers to Access-Control-Allow-Headers', - action: parsers.arrayParser, - }, - analyticsAdapter: { - env: 'PARSE_SERVER_ANALYTICS_ADAPTER', - help: 'Adapter module for the analytics', - action: parsers.moduleOrObjectParser, - }, - appId: { - env: 'PARSE_SERVER_APPLICATION_ID', - help: 'Your Parse Application ID', - required: true, - }, - appName: { - env: 'PARSE_SERVER_APP_NAME', - help: 'Sets the app name', - }, - auth: { - env: 'PARSE_SERVER_AUTH_PROVIDERS', - help: - 'Configuration for your authentication providers, as stringified JSON. See http://docs.parseplatform.org/parse-server/guide/#oauth-and-3rd-party-authentication', - action: parsers.objectParser, - }, - cacheAdapter: { - env: 'PARSE_SERVER_CACHE_ADAPTER', - help: 'Adapter module for the cache', - action: parsers.moduleOrObjectParser, - }, - cacheMaxSize: { - env: 'PARSE_SERVER_CACHE_MAX_SIZE', - help: 'Sets the maximum size for the in memory cache, defaults to 10000', - action: parsers.numberParser('cacheMaxSize'), - default: 10000, - }, - cacheTTL: { - env: 'PARSE_SERVER_CACHE_TTL', - help: - 'Sets the TTL for the in memory cache (in ms), defaults to 5000 (5 seconds)', - action: parsers.numberParser('cacheTTL'), - default: 5000, - }, - clientKey: { - env: 'PARSE_SERVER_CLIENT_KEY', - help: 'Key for iOS, MacOS, tvOS clients', - }, - cloud: { - env: 'PARSE_SERVER_CLOUD', - help: 'Full path to your cloud code main.js', - }, - cluster: { - env: 'PARSE_SERVER_CLUSTER', - help: - 'Run with cluster, optionally set the number of processes default to os.cpus().length', - action: parsers.numberOrBooleanParser, - }, - collectionPrefix: { - env: 'PARSE_SERVER_COLLECTION_PREFIX', - help: 'A collection prefix for the classes', - default: '', - }, - customPages: { - env: 'PARSE_SERVER_CUSTOM_PAGES', - help: 'custom pages for password validation and reset', - action: parsers.objectParser, - default: {}, - }, - databaseAdapter: { - env: 'PARSE_SERVER_DATABASE_ADAPTER', - help: 'Adapter module for the database', - action: parsers.moduleOrObjectParser, - }, - databaseOptions: { - env: 'PARSE_SERVER_DATABASE_OPTIONS', - help: 'Options to pass to the mongodb client', - action: parsers.objectParser, - }, - databaseURI: { - env: 'PARSE_SERVER_DATABASE_URI', - help: - 'The full URI to your database. Supported databases are mongodb or postgres.', - required: true, - default: 'mongodb://localhost:27017/parse', - }, - directAccess: { - env: 'PARSE_SERVER_ENABLE_EXPERIMENTAL_DIRECT_ACCESS', - help: - 'Replace HTTP Interface when using JS SDK in current node runtime, defaults to false. Caution, this is an experimental feature that may not be appropriate for production.', - action: parsers.booleanParser, - default: false, - }, - idempotencyOptions: { - env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS', - help: - 'Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production.', - action: parsers.objectParser, - default: {}, - }, - dotNetKey: { - env: 'PARSE_SERVER_DOT_NET_KEY', - help: 'Key for Unity and .Net SDK', - }, - emailAdapter: { - env: 'PARSE_SERVER_EMAIL_ADAPTER', - help: 'Adapter module for email sending', - action: parsers.moduleOrObjectParser, - }, - emailVerifyTokenValidityDuration: { - env: 'PARSE_SERVER_EMAIL_VERIFY_TOKEN_VALIDITY_DURATION', - help: 'Email verification token validity duration, in seconds', - action: parsers.numberParser('emailVerifyTokenValidityDuration'), - }, - enableAnonymousUsers: { - env: 'PARSE_SERVER_ENABLE_ANON_USERS', - help: 'Enable (or disable) anon users, defaults to true', - action: parsers.booleanParser, - default: true, - }, - enableExpressErrorHandler: { - env: 'PARSE_SERVER_ENABLE_EXPRESS_ERROR_HANDLER', - help: 'Enables the default express error handler for all errors', - action: parsers.booleanParser, - default: false, - }, - enableSingleSchemaCache: { - env: 'PARSE_SERVER_ENABLE_SINGLE_SCHEMA_CACHE', - help: - 'Use a single schema cache shared across requests. Reduces number of queries made to _SCHEMA, defaults to false, i.e. unique schema cache per request.', - action: parsers.booleanParser, - default: false, - }, - expireInactiveSessions: { - env: 'PARSE_SERVER_EXPIRE_INACTIVE_SESSIONS', - help: - 'Sets wether we should expire the inactive sessions, defaults to true', - action: parsers.booleanParser, - default: true, - }, - fileKey: { - env: 'PARSE_SERVER_FILE_KEY', - help: 'Key for your files', - }, - filesAdapter: { - env: 'PARSE_SERVER_FILES_ADAPTER', - help: 'Adapter module for the files sub-system', - action: parsers.moduleOrObjectParser, - }, - graphQLPath: { - env: 'PARSE_SERVER_GRAPHQL_PATH', - help: 'Mount path for the GraphQL endpoint, defaults to /graphql', - default: '/graphql', - }, - graphQLSchema: { - env: 'PARSE_SERVER_GRAPH_QLSCHEMA', - help: 'Full path to your GraphQL custom schema.graphql file', - }, - host: { - env: 'PARSE_SERVER_HOST', - help: 'The host to serve ParseServer on, defaults to 0.0.0.0', - default: '0.0.0.0', - }, - javascriptKey: { - env: 'PARSE_SERVER_JAVASCRIPT_KEY', - help: 'Key for the Javascript SDK', - }, - jsonLogs: { - env: 'JSON_LOGS', - help: 'Log as structured JSON objects', - action: parsers.booleanParser, - }, - liveQuery: { - env: 'PARSE_SERVER_LIVE_QUERY', - help: "parse-server's LiveQuery configuration object", - action: parsers.objectParser, - }, - liveQueryServerOptions: { - env: 'PARSE_SERVER_LIVE_QUERY_SERVER_OPTIONS', - help: - 'Live query server configuration options (will start the liveQuery server)', - action: parsers.objectParser, - }, - loggerAdapter: { - env: 'PARSE_SERVER_LOGGER_ADAPTER', - help: 'Adapter module for the logging sub-system', - action: parsers.moduleOrObjectParser, - }, - logLevel: { - env: 'PARSE_SERVER_LOG_LEVEL', - help: 'Sets the level for logs', - }, - logsFolder: { - env: 'PARSE_SERVER_LOGS_FOLDER', - help: - "Folder for the logs (defaults to './logs'); set to null to disable file based logging", - default: './logs', - }, - masterKey: { - env: 'PARSE_SERVER_MASTER_KEY', - help: 'Your Parse Master Key', - required: true, - }, - masterKeyIps: { - env: 'PARSE_SERVER_MASTER_KEY_IPS', - help: - 'Restrict masterKey to be used by only these ips, defaults to [] (allow all ips)', - action: parsers.arrayParser, - default: [], - }, - maxLimit: { - env: 'PARSE_SERVER_MAX_LIMIT', - help: 'Max value for limit option on queries, defaults to unlimited', - action: parsers.numberParser('maxLimit'), - }, - maxLogFiles: { - env: 'PARSE_SERVER_MAX_LOG_FILES', - help: - "Maximum number of logs to keep. If not set, no logs will be removed. This can be a number of files or number of days. If using days, add 'd' as the suffix. (default: null)", - action: parsers.objectParser, - }, - maxUploadSize: { - env: 'PARSE_SERVER_MAX_UPLOAD_SIZE', - help: 'Max file size for uploads, defaults to 20mb', - default: '20mb', - }, - middleware: { - env: 'PARSE_SERVER_MIDDLEWARE', - help: 'middleware for express server, can be string or function', - }, - mountGraphQL: { - env: 'PARSE_SERVER_MOUNT_GRAPHQL', - help: 'Mounts the GraphQL endpoint', - action: parsers.booleanParser, - default: false, - }, - mountPath: { - env: 'PARSE_SERVER_MOUNT_PATH', - help: 'Mount path for the server, defaults to /parse', - default: '/parse', - }, - mountPlayground: { - env: 'PARSE_SERVER_MOUNT_PLAYGROUND', - help: 'Mounts the GraphQL Playground - never use this option in production', - action: parsers.booleanParser, - default: false, - }, - objectIdSize: { - env: 'PARSE_SERVER_OBJECT_ID_SIZE', - help: "Sets the number of characters in generated object id's, default 10", - action: parsers.numberParser('objectIdSize'), - default: 10, - }, - passwordPolicy: { - env: 'PARSE_SERVER_PASSWORD_POLICY', - help: 'Password policy for enforcing password related rules', - action: parsers.objectParser, - }, - playgroundPath: { - env: 'PARSE_SERVER_PLAYGROUND_PATH', - help: 'Mount path for the GraphQL Playground, defaults to /playground', - default: '/playground', - }, - port: { - env: 'PORT', - help: 'The port to run the ParseServer, defaults to 1337.', - action: parsers.numberParser('port'), - default: 1337, - }, - preserveFileName: { - env: 'PARSE_SERVER_PRESERVE_FILE_NAME', - help: 'Enable (or disable) the addition of a unique hash to the file names', - action: parsers.booleanParser, - default: false, - }, - preventLoginWithUnverifiedEmail: { - env: 'PARSE_SERVER_PREVENT_LOGIN_WITH_UNVERIFIED_EMAIL', - help: - 'Prevent user from login if email is not verified and PARSE_SERVER_VERIFY_USER_EMAILS is true, defaults to false', - action: parsers.booleanParser, - default: false, - }, - protectedFields: { - env: 'PARSE_SERVER_PROTECTED_FIELDS', - help: - 'Protected fields that should be treated with extra security when fetching details.', - action: parsers.objectParser, - default: { - _User: { - '*': ['email'], - }, - }, - }, - publicServerURL: { - env: 'PARSE_PUBLIC_SERVER_URL', - help: 'Public URL to your parse server with http:// or https://.', - }, - push: { - env: 'PARSE_SERVER_PUSH', - help: - 'Configuration for push, as stringified JSON. See http://docs.parseplatform.org/parse-server/guide/#push-notifications', - action: parsers.objectParser, - }, - readOnlyMasterKey: { - env: 'PARSE_SERVER_READ_ONLY_MASTER_KEY', - help: - 'Read-only key, which has the same capabilities as MasterKey without writes', - }, - restAPIKey: { - env: 'PARSE_SERVER_REST_API_KEY', - help: 'Key for REST calls', - }, - revokeSessionOnPasswordReset: { - env: 'PARSE_SERVER_REVOKE_SESSION_ON_PASSWORD_RESET', - help: - "When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.", - action: parsers.booleanParser, - default: true, - }, - scheduledPush: { - env: 'PARSE_SERVER_SCHEDULED_PUSH', - help: 'Configuration for push scheduling, defaults to false.', - action: parsers.booleanParser, - default: false, - }, - schemaCacheTTL: { - env: 'PARSE_SERVER_SCHEMA_CACHE_TTL', - help: - 'The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 5000; set 0 to disable.', - action: parsers.numberParser('schemaCacheTTL'), - default: 5000, - }, - serverCloseComplete: { - env: 'PARSE_SERVER_SERVER_CLOSE_COMPLETE', - help: 'Callback when server has closed', - }, - serverStartComplete: { - env: 'PARSE_SERVER_SERVER_START_COMPLETE', - help: 'Callback when server has started', - }, - serverURL: { - env: 'PARSE_SERVER_URL', - help: 'URL to your parse server with http:// or https://.', - required: true, - }, - sessionLength: { - env: 'PARSE_SERVER_SESSION_LENGTH', - help: 'Session duration, in seconds, defaults to 1 year', - action: parsers.numberParser('sessionLength'), - default: 31536000, - }, - silent: { - env: 'SILENT', - help: 'Disables console output', - action: parsers.booleanParser, - }, - startLiveQueryServer: { - env: 'PARSE_SERVER_START_LIVE_QUERY_SERVER', - help: 'Starts the liveQuery server', - action: parsers.booleanParser, - }, - userSensitiveFields: { - env: 'PARSE_SERVER_USER_SENSITIVE_FIELDS', - help: - 'Personally identifiable information fields in the user table the should be removed for non-authorized users. Deprecated @see protectedFields', - action: parsers.arrayParser, - }, - verbose: { - env: 'VERBOSE', - help: 'Set the logging to verbose', - action: parsers.booleanParser, - }, - verifyUserEmails: { - env: 'PARSE_SERVER_VERIFY_USER_EMAILS', - help: 'Enable (or disable) user email validation, defaults to false', - action: parsers.booleanParser, - default: false, - }, - webhookKey: { - env: 'PARSE_SERVER_WEBHOOK_KEY', - help: 'Key sent with outgoing webhook calls', - }, + "accountLockout": { + "env": "PARSE_SERVER_ACCOUNT_LOCKOUT", + "help": "account lockout policy for failed login attempts", + "action": parsers.objectParser + }, + "allowClientClassCreation": { + "env": "PARSE_SERVER_ALLOW_CLIENT_CLASS_CREATION", + "help": "Enable (or disable) client class creation, defaults to true", + "action": parsers.booleanParser, + "default": true + }, + "allowCustomObjectId": { + "env": "PARSE_SERVER_ALLOW_CUSTOM_OBJECT_ID", + "help": "Enable (or disable) custom objectId", + "action": parsers.booleanParser, + "default": false + }, + "allowHeaders": { + "env": "PARSE_SERVER_ALLOW_HEADERS", + "help": "Add headers to Access-Control-Allow-Headers", + "action": parsers.arrayParser + }, + "analyticsAdapter": { + "env": "PARSE_SERVER_ANALYTICS_ADAPTER", + "help": "Adapter module for the analytics", + "action": parsers.moduleOrObjectParser + }, + "appId": { + "env": "PARSE_SERVER_APPLICATION_ID", + "help": "Your Parse Application ID", + "required": true + }, + "appName": { + "env": "PARSE_SERVER_APP_NAME", + "help": "Sets the app name" + }, + "auth": { + "env": "PARSE_SERVER_AUTH_PROVIDERS", + "help": "Configuration for your authentication providers, as stringified JSON. See http://docs.parseplatform.org/parse-server/guide/#oauth-and-3rd-party-authentication", + "action": parsers.objectParser + }, + "cacheAdapter": { + "env": "PARSE_SERVER_CACHE_ADAPTER", + "help": "Adapter module for the cache", + "action": parsers.moduleOrObjectParser + }, + "cacheMaxSize": { + "env": "PARSE_SERVER_CACHE_MAX_SIZE", + "help": "Sets the maximum size for the in memory cache, defaults to 10000", + "action": parsers.numberParser("cacheMaxSize"), + "default": 10000 + }, + "cacheTTL": { + "env": "PARSE_SERVER_CACHE_TTL", + "help": "Sets the TTL for the in memory cache (in ms), defaults to 5000 (5 seconds)", + "action": parsers.numberParser("cacheTTL"), + "default": 5000 + }, + "clientKey": { + "env": "PARSE_SERVER_CLIENT_KEY", + "help": "Key for iOS, MacOS, tvOS clients" + }, + "cloud": { + "env": "PARSE_SERVER_CLOUD", + "help": "Full path to your cloud code main.js" + }, + "cluster": { + "env": "PARSE_SERVER_CLUSTER", + "help": "Run with cluster, optionally set the number of processes default to os.cpus().length", + "action": parsers.numberOrBooleanParser + }, + "collectionPrefix": { + "env": "PARSE_SERVER_COLLECTION_PREFIX", + "help": "A collection prefix for the classes", + "default": "" + }, + "customPages": { + "env": "PARSE_SERVER_CUSTOM_PAGES", + "help": "custom pages for password validation and reset", + "action": parsers.objectParser, + "default": {} + }, + "databaseAdapter": { + "env": "PARSE_SERVER_DATABASE_ADAPTER", + "help": "Adapter module for the database", + "action": parsers.moduleOrObjectParser + }, + "databaseOptions": { + "env": "PARSE_SERVER_DATABASE_OPTIONS", + "help": "Options to pass to the mongodb client", + "action": parsers.objectParser + }, + "databaseURI": { + "env": "PARSE_SERVER_DATABASE_URI", + "help": "The full URI to your database. Supported databases are mongodb or postgres.", + "required": true, + "default": "mongodb://localhost:27017/parse" + }, + "directAccess": { + "env": "PARSE_SERVER_ENABLE_EXPERIMENTAL_DIRECT_ACCESS", + "help": "Replace HTTP Interface when using JS SDK in current node runtime, defaults to false. Caution, this is an experimental feature that may not be appropriate for production.", + "action": parsers.booleanParser, + "default": false + }, + "dotNetKey": { + "env": "PARSE_SERVER_DOT_NET_KEY", + "help": "Key for Unity and .Net SDK" + }, + "emailAdapter": { + "env": "PARSE_SERVER_EMAIL_ADAPTER", + "help": "Adapter module for email sending", + "action": parsers.moduleOrObjectParser + }, + "emailVerifyTokenValidityDuration": { + "env": "PARSE_SERVER_EMAIL_VERIFY_TOKEN_VALIDITY_DURATION", + "help": "Email verification token validity duration, in seconds", + "action": parsers.numberParser("emailVerifyTokenValidityDuration") + }, + "enableAnonymousUsers": { + "env": "PARSE_SERVER_ENABLE_ANON_USERS", + "help": "Enable (or disable) anon users, defaults to true", + "action": parsers.booleanParser, + "default": true + }, + "enableExpressErrorHandler": { + "env": "PARSE_SERVER_ENABLE_EXPRESS_ERROR_HANDLER", + "help": "Enables the default express error handler for all errors", + "action": parsers.booleanParser, + "default": false + }, + "enableSingleSchemaCache": { + "env": "PARSE_SERVER_ENABLE_SINGLE_SCHEMA_CACHE", + "help": "Use a single schema cache shared across requests. Reduces number of queries made to _SCHEMA, defaults to false, i.e. unique schema cache per request.", + "action": parsers.booleanParser, + "default": false + }, + "expireInactiveSessions": { + "env": "PARSE_SERVER_EXPIRE_INACTIVE_SESSIONS", + "help": "Sets wether we should expire the inactive sessions, defaults to true", + "action": parsers.booleanParser, + "default": true + }, + "fileKey": { + "env": "PARSE_SERVER_FILE_KEY", + "help": "Key for your files" + }, + "filesAdapter": { + "env": "PARSE_SERVER_FILES_ADAPTER", + "help": "Adapter module for the files sub-system", + "action": parsers.moduleOrObjectParser + }, + "graphQLPath": { + "env": "PARSE_SERVER_GRAPHQL_PATH", + "help": "Mount path for the GraphQL endpoint, defaults to /graphql", + "default": "/graphql" + }, + "graphQLSchema": { + "env": "PARSE_SERVER_GRAPH_QLSCHEMA", + "help": "Full path to your GraphQL custom schema.graphql file" + }, + "host": { + "env": "PARSE_SERVER_HOST", + "help": "The host to serve ParseServer on, defaults to 0.0.0.0", + "default": "0.0.0.0" + }, + "idempotencyOptions": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS", + "help": "Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production.", + "action": parsers.objectParser, + "default": {} + }, + "javascriptKey": { + "env": "PARSE_SERVER_JAVASCRIPT_KEY", + "help": "Key for the Javascript SDK" + }, + "jsonLogs": { + "env": "JSON_LOGS", + "help": "Log as structured JSON objects", + "action": parsers.booleanParser + }, + "liveQuery": { + "env": "PARSE_SERVER_LIVE_QUERY", + "help": "parse-server's LiveQuery configuration object", + "action": parsers.objectParser + }, + "liveQueryServerOptions": { + "env": "PARSE_SERVER_LIVE_QUERY_SERVER_OPTIONS", + "help": "Live query server configuration options (will start the liveQuery server)", + "action": parsers.objectParser + }, + "loggerAdapter": { + "env": "PARSE_SERVER_LOGGER_ADAPTER", + "help": "Adapter module for the logging sub-system", + "action": parsers.moduleOrObjectParser + }, + "logLevel": { + "env": "PARSE_SERVER_LOG_LEVEL", + "help": "Sets the level for logs" + }, + "logsFolder": { + "env": "PARSE_SERVER_LOGS_FOLDER", + "help": "Folder for the logs (defaults to './logs'); set to null to disable file based logging", + "default": "./logs" + }, + "masterKey": { + "env": "PARSE_SERVER_MASTER_KEY", + "help": "Your Parse Master Key", + "required": true + }, + "masterKeyIps": { + "env": "PARSE_SERVER_MASTER_KEY_IPS", + "help": "Restrict masterKey to be used by only these ips, defaults to [] (allow all ips)", + "action": parsers.arrayParser, + "default": [] + }, + "maxLimit": { + "env": "PARSE_SERVER_MAX_LIMIT", + "help": "Max value for limit option on queries, defaults to unlimited", + "action": parsers.numberParser("maxLimit") + }, + "maxLogFiles": { + "env": "PARSE_SERVER_MAX_LOG_FILES", + "help": "Maximum number of logs to keep. If not set, no logs will be removed. This can be a number of files or number of days. If using days, add 'd' as the suffix. (default: null)", + "action": parsers.objectParser + }, + "maxUploadSize": { + "env": "PARSE_SERVER_MAX_UPLOAD_SIZE", + "help": "Max file size for uploads, defaults to 20mb", + "default": "20mb" + }, + "middleware": { + "env": "PARSE_SERVER_MIDDLEWARE", + "help": "middleware for express server, can be string or function" + }, + "mountGraphQL": { + "env": "PARSE_SERVER_MOUNT_GRAPHQL", + "help": "Mounts the GraphQL endpoint", + "action": parsers.booleanParser, + "default": false + }, + "mountPath": { + "env": "PARSE_SERVER_MOUNT_PATH", + "help": "Mount path for the server, defaults to /parse", + "default": "/parse" + }, + "mountPlayground": { + "env": "PARSE_SERVER_MOUNT_PLAYGROUND", + "help": "Mounts the GraphQL Playground - never use this option in production", + "action": parsers.booleanParser, + "default": false + }, + "objectIdSize": { + "env": "PARSE_SERVER_OBJECT_ID_SIZE", + "help": "Sets the number of characters in generated object id's, default 10", + "action": parsers.numberParser("objectIdSize"), + "default": 10 + }, + "passwordPolicy": { + "env": "PARSE_SERVER_PASSWORD_POLICY", + "help": "Password policy for enforcing password related rules", + "action": parsers.objectParser + }, + "playgroundPath": { + "env": "PARSE_SERVER_PLAYGROUND_PATH", + "help": "Mount path for the GraphQL Playground, defaults to /playground", + "default": "/playground" + }, + "port": { + "env": "PORT", + "help": "The port to run the ParseServer, defaults to 1337.", + "action": parsers.numberParser("port"), + "default": 1337 + }, + "preserveFileName": { + "env": "PARSE_SERVER_PRESERVE_FILE_NAME", + "help": "Enable (or disable) the addition of a unique hash to the file names", + "action": parsers.booleanParser, + "default": false + }, + "preventLoginWithUnverifiedEmail": { + "env": "PARSE_SERVER_PREVENT_LOGIN_WITH_UNVERIFIED_EMAIL", + "help": "Prevent user from login if email is not verified and PARSE_SERVER_VERIFY_USER_EMAILS is true, defaults to false", + "action": parsers.booleanParser, + "default": false + }, + "protectedFields": { + "env": "PARSE_SERVER_PROTECTED_FIELDS", + "help": "Protected fields that should be treated with extra security when fetching details.", + "action": parsers.objectParser, + "default": { + "_User": { + "*": ["email"] + } + } + }, + "publicServerURL": { + "env": "PARSE_PUBLIC_SERVER_URL", + "help": "Public URL to your parse server with http:// or https://." + }, + "push": { + "env": "PARSE_SERVER_PUSH", + "help": "Configuration for push, as stringified JSON. See http://docs.parseplatform.org/parse-server/guide/#push-notifications", + "action": parsers.objectParser + }, + "readOnlyMasterKey": { + "env": "PARSE_SERVER_READ_ONLY_MASTER_KEY", + "help": "Read-only key, which has the same capabilities as MasterKey without writes" + }, + "restAPIKey": { + "env": "PARSE_SERVER_REST_API_KEY", + "help": "Key for REST calls" + }, + "revokeSessionOnPasswordReset": { + "env": "PARSE_SERVER_REVOKE_SESSION_ON_PASSWORD_RESET", + "help": "When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.", + "action": parsers.booleanParser, + "default": true + }, + "scheduledPush": { + "env": "PARSE_SERVER_SCHEDULED_PUSH", + "help": "Configuration for push scheduling, defaults to false.", + "action": parsers.booleanParser, + "default": false + }, + "schemaCacheTTL": { + "env": "PARSE_SERVER_SCHEMA_CACHE_TTL", + "help": "The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 5000; set 0 to disable.", + "action": parsers.numberParser("schemaCacheTTL"), + "default": 5000 + }, + "serverCloseComplete": { + "env": "PARSE_SERVER_SERVER_CLOSE_COMPLETE", + "help": "Callback when server has closed" + }, + "serverStartComplete": { + "env": "PARSE_SERVER_SERVER_START_COMPLETE", + "help": "Callback when server has started" + }, + "serverURL": { + "env": "PARSE_SERVER_URL", + "help": "URL to your parse server with http:// or https://.", + "required": true + }, + "sessionLength": { + "env": "PARSE_SERVER_SESSION_LENGTH", + "help": "Session duration, in seconds, defaults to 1 year", + "action": parsers.numberParser("sessionLength"), + "default": 31536000 + }, + "silent": { + "env": "SILENT", + "help": "Disables console output", + "action": parsers.booleanParser + }, + "startLiveQueryServer": { + "env": "PARSE_SERVER_START_LIVE_QUERY_SERVER", + "help": "Starts the liveQuery server", + "action": parsers.booleanParser + }, + "userSensitiveFields": { + "env": "PARSE_SERVER_USER_SENSITIVE_FIELDS", + "help": "Personally identifiable information fields in the user table the should be removed for non-authorized users. Deprecated @see protectedFields", + "action": parsers.arrayParser + }, + "verbose": { + "env": "VERBOSE", + "help": "Set the logging to verbose", + "action": parsers.booleanParser + }, + "verifyUserEmails": { + "env": "PARSE_SERVER_VERIFY_USER_EMAILS", + "help": "Enable (or disable) user email validation, defaults to false", + "action": parsers.booleanParser, + "default": false + }, + "webhookKey": { + "env": "PARSE_SERVER_WEBHOOK_KEY", + "help": "Key sent with outgoing webhook calls" + } }; module.exports.CustomPagesOptions = { - choosePassword: { - env: 'PARSE_SERVER_CUSTOM_PAGES_CHOOSE_PASSWORD', - help: 'choose password page path', - }, - invalidLink: { - env: 'PARSE_SERVER_CUSTOM_PAGES_INVALID_LINK', - help: 'invalid link page path', - }, - invalidVerificationLink: { - env: 'PARSE_SERVER_CUSTOM_PAGES_INVALID_VERIFICATION_LINK', - help: 'invalid verification link page path', - }, - linkSendFail: { - env: 'PARSE_SERVER_CUSTOM_PAGES_LINK_SEND_FAIL', - help: 'verification link send fail page path', - }, - linkSendSuccess: { - env: 'PARSE_SERVER_CUSTOM_PAGES_LINK_SEND_SUCCESS', - help: 'verification link send success page path', - }, - parseFrameURL: { - env: 'PARSE_SERVER_CUSTOM_PAGES_PARSE_FRAME_URL', - help: 'for masking user-facing pages', - }, - passwordResetSuccess: { - env: 'PARSE_SERVER_CUSTOM_PAGES_PASSWORD_RESET_SUCCESS', - help: 'password reset success page path', - }, - verifyEmailSuccess: { - env: 'PARSE_SERVER_CUSTOM_PAGES_VERIFY_EMAIL_SUCCESS', - help: 'verify email success page path', - }, + "choosePassword": { + "env": "PARSE_SERVER_CUSTOM_PAGES_CHOOSE_PASSWORD", + "help": "choose password page path" + }, + "invalidLink": { + "env": "PARSE_SERVER_CUSTOM_PAGES_INVALID_LINK", + "help": "invalid link page path" + }, + "invalidVerificationLink": { + "env": "PARSE_SERVER_CUSTOM_PAGES_INVALID_VERIFICATION_LINK", + "help": "invalid verification link page path" + }, + "linkSendFail": { + "env": "PARSE_SERVER_CUSTOM_PAGES_LINK_SEND_FAIL", + "help": "verification link send fail page path" + }, + "linkSendSuccess": { + "env": "PARSE_SERVER_CUSTOM_PAGES_LINK_SEND_SUCCESS", + "help": "verification link send success page path" + }, + "parseFrameURL": { + "env": "PARSE_SERVER_CUSTOM_PAGES_PARSE_FRAME_URL", + "help": "for masking user-facing pages" + }, + "passwordResetSuccess": { + "env": "PARSE_SERVER_CUSTOM_PAGES_PASSWORD_RESET_SUCCESS", + "help": "password reset success page path" + }, + "verifyEmailSuccess": { + "env": "PARSE_SERVER_CUSTOM_PAGES_VERIFY_EMAIL_SUCCESS", + "help": "verify email success page path" + } }; module.exports.LiveQueryOptions = { - classNames: { - env: 'PARSE_SERVER_LIVEQUERY_CLASSNAMES', - help: "parse-server's LiveQuery classNames", - action: parsers.arrayParser, - }, - pubSubAdapter: { - env: 'PARSE_SERVER_LIVEQUERY_PUB_SUB_ADAPTER', - help: 'LiveQuery pubsub adapter', - action: parsers.moduleOrObjectParser, - }, - redisOptions: { - env: 'PARSE_SERVER_LIVEQUERY_REDIS_OPTIONS', - help: "parse-server's LiveQuery redisOptions", - action: parsers.objectParser, - }, - redisURL: { - env: 'PARSE_SERVER_LIVEQUERY_REDIS_URL', - help: "parse-server's LiveQuery redisURL", - }, - wssAdapter: { - env: 'PARSE_SERVER_LIVEQUERY_WSS_ADAPTER', - help: 'Adapter module for the WebSocketServer', - action: parsers.moduleOrObjectParser, - }, + "classNames": { + "env": "PARSE_SERVER_LIVEQUERY_CLASSNAMES", + "help": "parse-server's LiveQuery classNames", + "action": parsers.arrayParser + }, + "pubSubAdapter": { + "env": "PARSE_SERVER_LIVEQUERY_PUB_SUB_ADAPTER", + "help": "LiveQuery pubsub adapter", + "action": parsers.moduleOrObjectParser + }, + "redisOptions": { + "env": "PARSE_SERVER_LIVEQUERY_REDIS_OPTIONS", + "help": "parse-server's LiveQuery redisOptions", + "action": parsers.objectParser + }, + "redisURL": { + "env": "PARSE_SERVER_LIVEQUERY_REDIS_URL", + "help": "parse-server's LiveQuery redisURL" + }, + "wssAdapter": { + "env": "PARSE_SERVER_LIVEQUERY_WSS_ADAPTER", + "help": "Adapter module for the WebSocketServer", + "action": parsers.moduleOrObjectParser + } }; module.exports.LiveQueryServerOptions = { - appId: { - env: 'PARSE_LIVE_QUERY_SERVER_APP_ID', - help: - 'This string should match the appId in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same appId.', - }, - cacheTimeout: { - env: 'PARSE_LIVE_QUERY_SERVER_CACHE_TIMEOUT', - help: - "Number in milliseconds. When clients provide the sessionToken to the LiveQuery server, the LiveQuery server will try to fetch its ParseUser's objectId from parse server and store it in the cache. The value defines the duration of the cache. Check the following Security section and our protocol specification for details, defaults to 30 * 24 * 60 * 60 * 1000 ms (~30 days).", - action: parsers.numberParser('cacheTimeout'), - }, - keyPairs: { - env: 'PARSE_LIVE_QUERY_SERVER_KEY_PAIRS', - help: - 'A JSON object that serves as a whitelist of keys. It is used for validating clients when they try to connect to the LiveQuery server. Check the following Security section and our protocol specification for details.', - action: parsers.objectParser, - }, - logLevel: { - env: 'PARSE_LIVE_QUERY_SERVER_LOG_LEVEL', - help: - 'This string defines the log level of the LiveQuery server. We support VERBOSE, INFO, ERROR, NONE, defaults to INFO.', - }, - masterKey: { - env: 'PARSE_LIVE_QUERY_SERVER_MASTER_KEY', - help: - 'This string should match the masterKey in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same masterKey.', - }, - port: { - env: 'PARSE_LIVE_QUERY_SERVER_PORT', - help: 'The port to run the LiveQuery server, defaults to 1337.', - action: parsers.numberParser('port'), - default: 1337, - }, - pubSubAdapter: { - env: 'PARSE_LIVE_QUERY_SERVER_PUB_SUB_ADAPTER', - help: 'LiveQuery pubsub adapter', - action: parsers.moduleOrObjectParser, - }, - redisOptions: { - env: 'PARSE_LIVE_QUERY_SERVER_REDIS_OPTIONS', - help: "parse-server's LiveQuery redisOptions", - action: parsers.objectParser, - }, - redisURL: { - env: 'PARSE_LIVE_QUERY_SERVER_REDIS_URL', - help: "parse-server's LiveQuery redisURL", - }, - serverURL: { - env: 'PARSE_LIVE_QUERY_SERVER_SERVER_URL', - help: - 'This string should match the serverURL in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same serverURL.', - }, - websocketTimeout: { - env: 'PARSE_LIVE_QUERY_SERVER_WEBSOCKET_TIMEOUT', - help: - 'Number of milliseconds between ping/pong frames. The WebSocket server sends ping/pong frames to the clients to keep the WebSocket alive. This value defines the interval of the ping/pong frame from the server to clients, defaults to 10 * 1000 ms (10 s).', - action: parsers.numberParser('websocketTimeout'), - }, - wssAdapter: { - env: 'PARSE_LIVE_QUERY_SERVER_WSS_ADAPTER', - help: 'Adapter module for the WebSocketServer', - action: parsers.moduleOrObjectParser, - }, + "appId": { + "env": "PARSE_LIVE_QUERY_SERVER_APP_ID", + "help": "This string should match the appId in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same appId." + }, + "cacheTimeout": { + "env": "PARSE_LIVE_QUERY_SERVER_CACHE_TIMEOUT", + "help": "Number in milliseconds. When clients provide the sessionToken to the LiveQuery server, the LiveQuery server will try to fetch its ParseUser's objectId from parse server and store it in the cache. The value defines the duration of the cache. Check the following Security section and our protocol specification for details, defaults to 30 * 24 * 60 * 60 * 1000 ms (~30 days).", + "action": parsers.numberParser("cacheTimeout") + }, + "keyPairs": { + "env": "PARSE_LIVE_QUERY_SERVER_KEY_PAIRS", + "help": "A JSON object that serves as a whitelist of keys. It is used for validating clients when they try to connect to the LiveQuery server. Check the following Security section and our protocol specification for details.", + "action": parsers.objectParser + }, + "logLevel": { + "env": "PARSE_LIVE_QUERY_SERVER_LOG_LEVEL", + "help": "This string defines the log level of the LiveQuery server. We support VERBOSE, INFO, ERROR, NONE, defaults to INFO." + }, + "masterKey": { + "env": "PARSE_LIVE_QUERY_SERVER_MASTER_KEY", + "help": "This string should match the masterKey in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same masterKey." + }, + "port": { + "env": "PARSE_LIVE_QUERY_SERVER_PORT", + "help": "The port to run the LiveQuery server, defaults to 1337.", + "action": parsers.numberParser("port"), + "default": 1337 + }, + "pubSubAdapter": { + "env": "PARSE_LIVE_QUERY_SERVER_PUB_SUB_ADAPTER", + "help": "LiveQuery pubsub adapter", + "action": parsers.moduleOrObjectParser + }, + "redisOptions": { + "env": "PARSE_LIVE_QUERY_SERVER_REDIS_OPTIONS", + "help": "parse-server's LiveQuery redisOptions", + "action": parsers.objectParser + }, + "redisURL": { + "env": "PARSE_LIVE_QUERY_SERVER_REDIS_URL", + "help": "parse-server's LiveQuery redisURL" + }, + "serverURL": { + "env": "PARSE_LIVE_QUERY_SERVER_SERVER_URL", + "help": "This string should match the serverURL in use by your Parse Server. If you deploy the LiveQuery server alongside Parse Server, the LiveQuery server will try to use the same serverURL." + }, + "websocketTimeout": { + "env": "PARSE_LIVE_QUERY_SERVER_WEBSOCKET_TIMEOUT", + "help": "Number of milliseconds between ping/pong frames. The WebSocket server sends ping/pong frames to the clients to keep the WebSocket alive. This value defines the interval of the ping/pong frame from the server to clients, defaults to 10 * 1000 ms (10 s).", + "action": parsers.numberParser("websocketTimeout") + }, + "wssAdapter": { + "env": "PARSE_LIVE_QUERY_SERVER_WSS_ADAPTER", + "help": "Adapter module for the WebSocketServer", + "action": parsers.moduleOrObjectParser + } }; - module.exports.IdempotencyOptions = { - functions: { - env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS', - help: 'Array of function names, use `*` for all, default is none.', - action: parsers.arrayParser, - }, - jobs: { - env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS', - help: 'Array of job names, use `*` for all, default is none.', - action: parsers.arrayParser, - }, - classes: { - env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES', - help: 'Array of class names, use `*` for all, default is none.', - action: parsers.arrayParser, - }, - ttl: { - env: 'PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL', - help: 'The duration in seconds after which a request record is discarded from the database, default is 300s.', - action: parsers.numberParser, - default: 300, - }, + "classes": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES", + "help": "Array of class names, use `*` for all, default is none.", + "action": parsers.arrayParser + }, + "functions": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS", + "help": "Array of function names, use `*` for all, default is none.", + "action": parsers.arrayParser + }, + "jobs": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS", + "help": "Array of job names, use `*` for all, default is none.", + "action": parsers.arrayParser + }, + "ttl": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL", + "help": "The duration in seconds after which a request record is discarded from the database, default is 300s.", + "action": parsers.numberParser("ttl") + } }; diff --git a/src/Options/docs.js b/src/Options/docs.js index 1fd3fb939b..da3e46c9f9 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -32,11 +32,11 @@ * @property {String} graphQLPath Mount path for the GraphQL endpoint, defaults to /graphql * @property {String} graphQLSchema Full path to your GraphQL custom schema.graphql file * @property {String} host The host to serve ParseServer on, defaults to 0.0.0.0 + * @property {IdempotencyOptions} idempotencyOptions Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production. * @property {String} javascriptKey Key for the Javascript SDK * @property {Boolean} jsonLogs Log as structured JSON objects * @property {LiveQueryOptions} liveQuery parse-server's LiveQuery configuration object * @property {LiveQueryServerOptions} liveQueryServerOptions Live query server configuration options (will start the liveQuery server) - * @property {IdempotencyOptions} idempotencyOptions Options for request idempotency to deduplicate identical requests that may be caused by network issues. Caution, this is an experimental feature that may not be appropriate for production. * @property {Adapter} loggerAdapter Adapter module for the logging sub-system * @property {String} logLevel Sets the level for logs * @property {String} logsFolder Folder for the logs (defaults to './logs'); set to null to disable file based logging @@ -114,8 +114,9 @@ /** * @interface IdempotencyOptions + * @property {String[]} classes Array of class names, use `*` for all, default is none. * @property {String[]} functions Array of function names, use `*` for all, default is none. * @property {String[]} jobs Array of job names, use `*` for all, default is none. - * @property {String[]} classes Array of class names, use `*` for all, default is none. * @property {Number} ttl The duration in seconds after which a request record is discarded from the database, default is 300s. */ + From e36a49f209f9dd4e24e89b95626de2a8ab73aa34 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 23:16:06 +0200 Subject: [PATCH 14/58] fixed incorrect import path --- src/middlewares.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares.js b/src/middlewares.js index 12ce78aac6..1ad6c2fa0f 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -5,7 +5,7 @@ import Config from './Config'; import ClientSDK from './ClientSDK'; import defaultLogger from './logger'; import rest from './rest'; -import MongoStorageAdapter from '../lib/Adapters/Storage/Mongo/MongoStorageAdapter'; +import MongoStorageAdapter from './Adapters/Storage/Mongo/MongoStorageAdapter'; export const DEFAULT_ALLOWED_HEADERS = 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, Content-Type, Pragma, Cache-Control'; From b2204b0cd308a7ff72b8f168ea2585f8f95c034f Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Wed, 24 Jun 2020 23:18:36 +0200 Subject: [PATCH 15/58] added new request ID header to allowed headers --- src/middlewares.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares.js b/src/middlewares.js index 1ad6c2fa0f..53cc122416 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -8,7 +8,7 @@ import rest from './rest'; import MongoStorageAdapter from './Adapters/Storage/Mongo/MongoStorageAdapter'; export const DEFAULT_ALLOWED_HEADERS = - 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, Content-Type, Pragma, Cache-Control'; + 'X-Parse-Master-Key, X-Parse-REST-API-Key, X-Parse-Javascript-Key, X-Parse-Application-Id, X-Parse-Client-Version, X-Parse-Session-Token, X-Requested-With, X-Parse-Revocable-Session, X-Parse-Request-Id, Content-Type, Pragma, Cache-Control'; const getMountForRequest = function (req) { const mountPathLength = req.originalUrl.length - req.url.length; From 772f942223dda0bb49753cb6cb8b9d288fa24c90 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 00:03:45 +0200 Subject: [PATCH 16/58] fixed typescript typos --- src/Adapters/Storage/Mongo/MongoStorageAdapter.js | 2 +- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 2 +- src/Controllers/DatabaseController.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 9c08d3a073..a46da0edf0 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -692,7 +692,7 @@ export class MongoStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - options?: Object = {}, + options: ?Object = {}, ): Promise { schema = convertParseSchemaToMongoSchema(schema); const indexCreationRequest = {}; diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index ce7b746f4f..4b4c659277 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2577,7 +2577,7 @@ export class PostgresStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - options?: Object = {}, + options: ?Object = {}, ): Promise { const conn = options.conn !== undefined ? options.conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 2f359e8708..2880d45d49 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1822,8 +1822,8 @@ class DatabaseController { }); const idempotencyExpireIndex = idempotencyClassPromise - .then(() => { - return this.adapter.ensureIndex( + .then(() => + this.adapter.ensureIndex( '_Idempotency', requiredUserFields, ['expire'], @@ -1831,7 +1831,7 @@ class DatabaseController { false, { ttl: 0 }, ) - }) + ) .catch((error) => { logger.warn( 'Unable to create TTL index for idempotency expire date: ', From ae928f36a2e276e9eb42084d404bbb0081e04a64 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 00:05:27 +0200 Subject: [PATCH 17/58] add new system class to spec helper --- spec/helper.js | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/helper.js b/spec/helper.js index 16d25ba1b8..84e704f7e3 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -230,6 +230,7 @@ afterEach(function(done) { '_Session', '_Product', '_Audience', + '_Idempotency' ].indexOf(className) >= 0 ); } From a6232c0815fcd4f399d6ed159c8ce423b67ac725 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 00:20:21 +0200 Subject: [PATCH 18/58] fixed typescript typos --- src/Adapters/Storage/Mongo/MongoStorageAdapter.js | 2 +- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index a46da0edf0..9c08d3a073 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -692,7 +692,7 @@ export class MongoStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - options: ?Object = {}, + options?: Object = {}, ): Promise { schema = convertParseSchemaToMongoSchema(schema); const indexCreationRequest = {}; diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 4b4c659277..ce7b746f4f 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2577,7 +2577,7 @@ export class PostgresStorageAdapter implements StorageAdapter { fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false, - options: ?Object = {}, + options?: Object = {}, ): Promise { const conn = options.conn !== undefined ? options.conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; From d3e7ee986c77220a50c0ffb5d745f8cde3efac9e Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 00:28:17 +0200 Subject: [PATCH 19/58] re-added postgres conn parameter --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index ce7b746f4f..8bcc2e8c11 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2578,8 +2578,9 @@ export class PostgresStorageAdapter implements StorageAdapter { indexName: ?string, caseInsensitive: boolean = false, options?: Object = {}, + conn: ?any = null ): Promise { - const conn = options.conn !== undefined ? options.conn : this._client; + conn = conn != null ? conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName != null ? { name: indexName } : { name: defaultIndexName }; From 0d58fef30073aa8c21eba65ed4591450a88a5f3b Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 00:45:51 +0200 Subject: [PATCH 20/58] removed postgres conn parameter --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 8bcc2e8c11..ce7b746f4f 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2578,9 +2578,8 @@ export class PostgresStorageAdapter implements StorageAdapter { indexName: ?string, caseInsensitive: boolean = false, options?: Object = {}, - conn: ?any = null ): Promise { - conn = conn != null ? conn : this._client; + const conn = options.conn !== undefined ? options.conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName != null ? { name: indexName } : { name: defaultIndexName }; From 9c625632b0b57ec11c361ae327e6c223672d270a Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 01:15:59 +0200 Subject: [PATCH 21/58] fixed incorrect schema for index creation --- src/Controllers/DatabaseController.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 2880d45d49..f8a1ce0370 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1740,6 +1740,12 @@ class DatabaseController { ...SchemaController.defaultColumns._Role, }, }; + const requiredIdempotencyFields = { + fields: { + ...SchemaController.defaultColumns._Default, + ...SchemaController.defaultColumns._Idempotency, + }, + }; const userClassPromise = this.loadSchema().then((schema) => schema.enforceClassExists('_User') @@ -1809,7 +1815,7 @@ class DatabaseController { .then(() => { return this.adapter.ensureUniqueness( '_Idempotency', - requiredUserFields, + requiredIdempotencyFields, ['reqId'] ); }) @@ -1825,7 +1831,7 @@ class DatabaseController { .then(() => this.adapter.ensureIndex( '_Idempotency', - requiredUserFields, + requiredIdempotencyFields, ['expire'], 'ttl', false, From 884fd356ce055ce928bfa3b4e7f1a4162293cf95 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 01:33:11 +0200 Subject: [PATCH 22/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 72 +++++++++++++-------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index f8a1ce0370..b9de1194e3 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1811,40 +1811,40 @@ class DatabaseController { throw error; }); - const idempotencyRequestIdUniqueness = idempotencyClassPromise - .then(() => { - return this.adapter.ensureUniqueness( - '_Idempotency', - requiredIdempotencyFields, - ['reqId'] - ); - }) - .catch((error) => { - logger.warn( - 'Unable to ensure uniqueness for idempotency request ID: ', - error - ); - throw error; - }); - - const idempotencyExpireIndex = idempotencyClassPromise - .then(() => - this.adapter.ensureIndex( - '_Idempotency', - requiredIdempotencyFields, - ['expire'], - 'ttl', - false, - { ttl: 0 }, - ) - ) - .catch((error) => { - logger.warn( - 'Unable to create TTL index for idempotency expire date: ', - error - ); - throw error; - }); + // const idempotencyRequestIdUniqueness = idempotencyClassPromise + // .then(() => { + // return this.adapter.ensureUniqueness( + // '_Idempotency', + // requiredIdempotencyFields, + // ['reqId'] + // ); + // }) + // .catch((error) => { + // logger.warn( + // 'Unable to ensure uniqueness for idempotency request ID: ', + // error + // ); + // throw error; + // }); + + // const idempotencyExpireIndex = idempotencyClassPromise + // .then(() => + // this.adapter.ensureIndex( + // '_Idempotency', + // requiredIdempotencyFields, + // ['expire'], + // 'ttl', + // false, + // { ttl: 0 }, + // ) + // ) + // .catch((error) => { + // logger.warn( + // 'Unable to create TTL index for idempotency expire date: ', + // error + // ); + // throw error; + // }); const roleUniqueness = roleClassPromise .then(() => @@ -1867,8 +1867,8 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - idempotencyRequestIdUniqueness, - idempotencyExpireIndex, + // idempotencyRequestIdUniqueness, + // idempotencyExpireIndex, adapterInit, indexPromise, ]); From 33fc25c3b4c585c074afb380ab74f215ca9b77f4 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 01:38:39 +0200 Subject: [PATCH 23/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index b9de1194e3..91f7d270ce 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1827,24 +1827,24 @@ class DatabaseController { // throw error; // }); - // const idempotencyExpireIndex = idempotencyClassPromise - // .then(() => - // this.adapter.ensureIndex( - // '_Idempotency', - // requiredIdempotencyFields, - // ['expire'], - // 'ttl', - // false, - // { ttl: 0 }, - // ) - // ) - // .catch((error) => { - // logger.warn( - // 'Unable to create TTL index for idempotency expire date: ', - // error - // ); - // throw error; - // }); + const idempotencyExpireIndex = idempotencyClassPromise + .then(() => + this.adapter.ensureIndex( + '_Idempotency', + requiredIdempotencyFields, + ['expire'], + 'ttl', + false, + { ttl: 0 }, + ) + ) + .catch((error) => { + logger.warn( + 'Unable to create TTL index for idempotency expire date: ', + error + ); + throw error; + }); const roleUniqueness = roleClassPromise .then(() => @@ -1867,7 +1867,7 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - // idempotencyRequestIdUniqueness, + idempotencyRequestIdUniqueness, // idempotencyExpireIndex, adapterInit, indexPromise, From bf32dd162a3db7a95db8a84cbb8b748822c9af69 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 01:43:32 +0200 Subject: [PATCH 24/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 52 +++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 91f7d270ce..bad528ddec 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1811,41 +1811,41 @@ class DatabaseController { throw error; }); - // const idempotencyRequestIdUniqueness = idempotencyClassPromise - // .then(() => { - // return this.adapter.ensureUniqueness( - // '_Idempotency', - // requiredIdempotencyFields, - // ['reqId'] - // ); - // }) - // .catch((error) => { - // logger.warn( - // 'Unable to ensure uniqueness for idempotency request ID: ', - // error - // ); - // throw error; - // }); - - const idempotencyExpireIndex = idempotencyClassPromise - .then(() => - this.adapter.ensureIndex( + const idempotencyRequestIdUniqueness = idempotencyClassPromise + .then(() => { + return this.adapter.ensureUniqueness( '_Idempotency', requiredIdempotencyFields, - ['expire'], - 'ttl', - false, - { ttl: 0 }, - ) - ) + ['reqId'] + ); + }) .catch((error) => { logger.warn( - 'Unable to create TTL index for idempotency expire date: ', + 'Unable to ensure uniqueness for idempotency request ID: ', error ); throw error; }); + // const idempotencyExpireIndex = idempotencyClassPromise + // .then(() => + // this.adapter.ensureIndex( + // '_Idempotency', + // requiredIdempotencyFields, + // ['expire'], + // 'ttl', + // false, + // { ttl: 0 }, + // ) + // ) + // .catch((error) => { + // logger.warn( + // 'Unable to create TTL index for idempotency expire date: ', + // error + // ); + // throw error; + // }); + const roleUniqueness = roleClassPromise .then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']) From 46f19b07954e4a6c0d188f497f983edaf1ea7621 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 02:15:18 +0200 Subject: [PATCH 25/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index bad528ddec..c5b3a321c6 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1811,41 +1811,41 @@ class DatabaseController { throw error; }); - const idempotencyRequestIdUniqueness = idempotencyClassPromise - .then(() => { - return this.adapter.ensureUniqueness( - '_Idempotency', - requiredIdempotencyFields, - ['reqId'] - ); - }) - .catch((error) => { - logger.warn( - 'Unable to ensure uniqueness for idempotency request ID: ', - error - ); - throw error; - }); - - // const idempotencyExpireIndex = idempotencyClassPromise + // const idempotencyRequestIdUniqueness = idempotencyClassPromise // .then(() => - // this.adapter.ensureIndex( + // this.adapter.ensureUniqueness( // '_Idempotency', // requiredIdempotencyFields, - // ['expire'], - // 'ttl', - // false, - // { ttl: 0 }, + // ['reqId'] // ) // ) // .catch((error) => { // logger.warn( - // 'Unable to create TTL index for idempotency expire date: ', + // 'Unable to ensure uniqueness for idempotency request ID: ', // error // ); // throw error; // }); + const idempotencyExpireIndex = idempotencyClassPromise + .then(() => + this.adapter.ensureIndex( + '_Idempotency', + requiredIdempotencyFields, + ['expire'], + 'ttl', + false, + { ttl: 0 }, + ) + ) + .catch((error) => { + logger.warn( + 'Unable to create TTL index for idempotency expire date: ', + error + ); + throw error; + }); + const roleUniqueness = roleClassPromise .then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']) @@ -1867,8 +1867,8 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - idempotencyRequestIdUniqueness, - // idempotencyExpireIndex, + // idempotencyRequestIdUniqueness, + idempotencyExpireIndex, adapterInit, indexPromise, ]); From e14138e6e3cfb3dd2efbfa2906c4826857765a73 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 03:09:28 +0200 Subject: [PATCH 26/58] temporarily disabling index creation to fix postgres issue --- spec/helper.js | 1 - src/Controllers/DatabaseController.js | 1 - src/Controllers/SchemaController.js | 11 ++--------- src/rest.js | 1 + 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/spec/helper.js b/spec/helper.js index 84e704f7e3..16d25ba1b8 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -230,7 +230,6 @@ afterEach(function(done) { '_Session', '_Product', '_Audience', - '_Idempotency' ].indexOf(className) >= 0 ); } diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index c5b3a321c6..0a7e6928a4 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1742,7 +1742,6 @@ class DatabaseController { }; const requiredIdempotencyFields = { fields: { - ...SchemaController.defaultColumns._Default, ...SchemaController.defaultColumns._Idempotency, }, }; diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index d26e9bc2b8..dfacd078ba 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -164,8 +164,7 @@ const systemClasses = Object.freeze([ '_PushStatus', '_JobStatus', '_JobSchedule', - '_Audience', - '_Idempotency' + '_Audience' ]); const volatileClasses = Object.freeze([ @@ -666,13 +665,7 @@ const _AudienceSchema = convertSchemaToAdapterSchema( classLevelPermissions: {}, }) ); -const _IdempotencySchema = convertSchemaToAdapterSchema( - injectDefaultSchema({ - className: '_Idempotency', - fields: defaultColumns._Idempotency, - classLevelPermissions: {}, - }) -); +const _IdempotencySchema = { className: '_Idempotency', fields: defaultColumns._Idempotency }; const VolatileClassesSchemas = [ _HooksSchema, _JobStatusSchema, diff --git a/src/rest.js b/src/rest.js index 201cd89bbc..2c039d65b8 100644 --- a/src/rest.js +++ b/src/rest.js @@ -278,6 +278,7 @@ const classesWithMasterOnlyAccess = [ '_Hooks', '_GlobalConfig', '_JobSchedule', + '_Idempotency', ]; // Disallowing access to the _Role collection except by master key function enforceRoleSecurity(method, className, auth) { From f80730eb2f84adee5edf2e48ffbc0891be8d0de2 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 03:23:03 +0200 Subject: [PATCH 27/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 33 ++++++++++++++------------- src/Controllers/SchemaController.js | 11 +++++++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 0a7e6928a4..48a90c00c3 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1742,6 +1742,7 @@ class DatabaseController { }; const requiredIdempotencyFields = { fields: { + ...SchemaController.defaultColumns._Default, ...SchemaController.defaultColumns._Idempotency, }, }; @@ -1810,21 +1811,21 @@ class DatabaseController { throw error; }); - // const idempotencyRequestIdUniqueness = idempotencyClassPromise - // .then(() => - // this.adapter.ensureUniqueness( - // '_Idempotency', - // requiredIdempotencyFields, - // ['reqId'] - // ) - // ) - // .catch((error) => { - // logger.warn( - // 'Unable to ensure uniqueness for idempotency request ID: ', - // error - // ); - // throw error; - // }); + const idempotencyRequestIdUniqueness = idempotencyClassPromise + .then(() => + this.adapter.ensureUniqueness( + '_Idempotency', + requiredIdempotencyFields, + ['reqId'] + ) + ) + .catch((error) => { + logger.warn( + 'Unable to ensure uniqueness for idempotency request ID: ', + error + ); + throw error; + }); const idempotencyExpireIndex = idempotencyClassPromise .then(() => @@ -1867,7 +1868,7 @@ class DatabaseController { emailCaseInsensitiveIndex, roleUniqueness, // idempotencyRequestIdUniqueness, - idempotencyExpireIndex, + // idempotencyExpireIndex, adapterInit, indexPromise, ]); diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index dfacd078ba..d26e9bc2b8 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -164,7 +164,8 @@ const systemClasses = Object.freeze([ '_PushStatus', '_JobStatus', '_JobSchedule', - '_Audience' + '_Audience', + '_Idempotency' ]); const volatileClasses = Object.freeze([ @@ -665,7 +666,13 @@ const _AudienceSchema = convertSchemaToAdapterSchema( classLevelPermissions: {}, }) ); -const _IdempotencySchema = { className: '_Idempotency', fields: defaultColumns._Idempotency }; +const _IdempotencySchema = convertSchemaToAdapterSchema( + injectDefaultSchema({ + className: '_Idempotency', + fields: defaultColumns._Idempotency, + classLevelPermissions: {}, + }) +); const VolatileClassesSchemas = [ _HooksSchema, _JobStatusSchema, From f02b3fd7915570ab2aa19daf3b7d97a8bf98b3d3 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 03:29:46 +0200 Subject: [PATCH 28/58] temporarily disabling index creation to fix postgres issue --- src/Controllers/DatabaseController.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 48a90c00c3..10cafe4b79 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1867,8 +1867,8 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - // idempotencyRequestIdUniqueness, - // idempotencyExpireIndex, + idempotencyRequestIdUniqueness, + idempotencyExpireIndex, adapterInit, indexPromise, ]); From 1e3ad1577a103ba350569c012e2b18e7e4d70f5a Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 09:21:23 +0200 Subject: [PATCH 29/58] trying to fix postgres issue --- src/Controllers/SchemaController.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index d26e9bc2b8..57c5fea7fc 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -164,8 +164,7 @@ const systemClasses = Object.freeze([ '_PushStatus', '_JobStatus', '_JobSchedule', - '_Audience', - '_Idempotency' + '_Audience' ]); const volatileClasses = Object.freeze([ @@ -669,7 +668,7 @@ const _AudienceSchema = convertSchemaToAdapterSchema( const _IdempotencySchema = convertSchemaToAdapterSchema( injectDefaultSchema({ className: '_Idempotency', - fields: defaultColumns._Idempotency, + fields: {}, classLevelPermissions: {}, }) ); From 6175f418708e4dc0c7a0fcd75cdc0b21c26d22e2 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 09:31:37 +0200 Subject: [PATCH 30/58] fixed incorrect auth when writing to _Idempotency --- src/Controllers/SchemaController.js | 3 ++- src/middlewares.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 57c5fea7fc..784f7e1020 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -164,7 +164,8 @@ const systemClasses = Object.freeze([ '_PushStatus', '_JobStatus', '_JobSchedule', - '_Audience' + '_Audience', + '_Idempotency' ]); const volatileClasses = Object.freeze([ diff --git a/src/middlewares.js b/src/middlewares.js index 53cc122416..025d51d17e 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -427,7 +427,7 @@ export function promiseEnsureIdempotency(req) { const expiryDate = new Date(new Date().setSeconds(new Date().getSeconds() + ttl)); return rest.create( req.config, - auth.nobody(req.config), + auth.master(req.config), '_Idempotency', { reqId: requestId, expire: Parse._encode(expiryDate) } ).catch (e => { From c7e2d19cac7880577e75b53e20c19d527a88a77f Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 12:14:47 +0200 Subject: [PATCH 31/58] trying to fix postgres issue --- spec/helper.js | 1 + src/Controllers/DatabaseController.js | 75 ++++++++++++++------------- src/Controllers/SchemaController.js | 2 +- 3 files changed, 40 insertions(+), 38 deletions(-) diff --git a/spec/helper.js b/spec/helper.js index 16d25ba1b8..84e704f7e3 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -230,6 +230,7 @@ afterEach(function(done) { '_Session', '_Product', '_Audience', + '_Idempotency' ].indexOf(className) >= 0 ); } diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 10cafe4b79..2a92398c65 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -244,6 +244,7 @@ const filterSensitiveData = ( }; import type { LoadSchemaOptions } from './types'; +import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter'; // Runs an update on the database. // Returns a promise for an object with the new values for field @@ -1811,41 +1812,6 @@ class DatabaseController { throw error; }); - const idempotencyRequestIdUniqueness = idempotencyClassPromise - .then(() => - this.adapter.ensureUniqueness( - '_Idempotency', - requiredIdempotencyFields, - ['reqId'] - ) - ) - .catch((error) => { - logger.warn( - 'Unable to ensure uniqueness for idempotency request ID: ', - error - ); - throw error; - }); - - const idempotencyExpireIndex = idempotencyClassPromise - .then(() => - this.adapter.ensureIndex( - '_Idempotency', - requiredIdempotencyFields, - ['expire'], - 'ttl', - false, - { ttl: 0 }, - ) - ) - .catch((error) => { - logger.warn( - 'Unable to create TTL index for idempotency expire date: ', - error - ); - throw error; - }); - const roleUniqueness = roleClassPromise .then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']) @@ -1855,6 +1821,42 @@ class DatabaseController { throw error; }); + const idempotencyIndices = this.adapter instanceof MongoStorageAdapter ? [ + idempotencyClassPromise + .then(() => + this.adapter.ensureUniqueness( + '_Idempotency', + requiredIdempotencyFields, + ['reqId'] + ) + ) + .catch((error) => { + logger.warn( + 'Unable to ensure uniqueness for idempotency request ID: ', + error + ); + throw error; + }), + idempotencyClassPromise + .then(() => + this.adapter.ensureIndex( + '_Idempotency', + requiredIdempotencyFields, + ['expire'], + 'ttl', + false, + { ttl: 0 }, + ) + ) + .catch((error) => { + logger.warn( + 'Unable to create TTL index for idempotency expire date: ', + error + ); + throw error; + }) + ] : []; + const indexPromise = this.adapter.updateSchemaWithIndexes(); // Create tables for volatile classes @@ -1867,8 +1869,7 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - idempotencyRequestIdUniqueness, - idempotencyExpireIndex, + ...idempotencyIndices, adapterInit, indexPromise, ]); diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 784f7e1020..d26e9bc2b8 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -669,7 +669,7 @@ const _AudienceSchema = convertSchemaToAdapterSchema( const _IdempotencySchema = convertSchemaToAdapterSchema( injectDefaultSchema({ className: '_Idempotency', - fields: {}, + fields: defaultColumns._Idempotency, classLevelPermissions: {}, }) ); From e6536b060249aa83f947c456482ab42f4542a81d Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 12:37:17 +0200 Subject: [PATCH 32/58] Travis CI trigger From 8f670d3b872f0ff86a1daf04b0f8cb80ad17db92 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 15:17:18 +0200 Subject: [PATCH 33/58] added test cases --- spec/Idempotency.spec.js | 166 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 spec/Idempotency.spec.js diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js new file mode 100644 index 0000000000..97a2b3983b --- /dev/null +++ b/spec/Idempotency.spec.js @@ -0,0 +1,166 @@ +'use strict'; +const Config = require('../lib/Config'); +const request = require('../lib/request'); +const rest = require('../lib/rest'); +const auth = require('../lib/Auth'); +const uuid = require('uuid'); + +describe_only_db('mongo')('idempotency for cloud code functions', () => { + // Parameters + /** Enable TTL expiration simulated by removing entry instead of waiting for MongoDB TTL monitor which + runss only every 60s, so it can take up to 119s until entry removal - ain't nobody got time for that */ + const SIMULATE_TTL = true; + // Helpers + async function deleteRequestEntry(reqId) { + const config = Config.get(Parse.applicationId); + const res = await rest.find( + config, + auth.master(config), + '_Idempotency', + { reqId: reqId }, + { limit: 1 } + ); + await rest.del( + config, + auth.master(config), + '_Idempotency', + res.results[0].objectId); + } + // Setups + beforeEach(async () => { + if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200_000; } + await reconfigureServer({ + appId: Parse.applicationId, + masterKey: Parse.masterKey, + serverURL: Parse.serverURL, + idempotencyOptions: { + ttl: 30, + functions: ["*"], + jobs: ["*"], + classes: ["*"], + }, + }) + }); + // Tests + it('should enforce idempotency for cloud code function', async () => { + // Declare function + let counter = 0; + Parse.Cloud.define('myFunction', () => { + counter++; + }); + // Run function + const params = { + method: 'POST', + url: 'http://localhost:8378/1/functions/myFunction', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + expect(Config.get(Parse.applicationId).idempotencyOptions.ttl).toBe(30); + await request(params); + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("Duplicate request"); + }); + expect(counter).toBe(1); + }); + + it('should delete request entry after TTL', async () => { + // Declare function + let counter = 0; + Parse.Cloud.define('myFunction', () => { + counter++; + }); + // Run function + const params = { + method: 'POST', + url: 'http://localhost:8378/1/functions/myFunction', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await expectAsync(request(params)).toBeResolved(); + if (SIMULATE_TTL) { + await deleteRequestEntry('abc-123'); + } else { + await new Promise(resolve => setTimeout(resolve, 130_000)); + } + await expectAsync(request(params)).toBeResolved(); + expect(counter).toBe(2); + }); + + it('should enforce idempotency for cloud code jobs', async () => { + // Declare job + let counter = 0; + Parse.Cloud.job('myJob', () => { + counter++; + }); + // Run job + const params = { + method: 'POST', + url: 'http://localhost:8378/1/jobs/myJob', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await expectAsync(request(params)).toBeResolved(); + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("Duplicate request"); + }); + expect(counter).toBe(1); + }); + + it('should enforce idempotency for class object creation', async () => { + // Declare trigger + let counter = 0; + Parse.Cloud.afterSave('MyClass', () => { + counter++; + }); + // Create object + const params = { + method: 'POST', + url: 'http://localhost:8378/1/classes/MyClass', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await expectAsync(request(params)).toBeResolved(); + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("Duplicate request"); + }); + expect(counter).toBe(1); + }); + + it('should not interfere with calls of different request ID', async () => { + // Declare trigger + let counter = 0; + Parse.Cloud.afterSave('MyClass', () => { + counter++; + }); + // Create 100 objects + const promises = [...Array(100).keys()].map(() => { + const params = { + method: 'POST', + url: 'http://localhost:8378/1/classes/MyClass', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': uuid.v4() + } + }; + return request(params); + }); + await expectAsync(Promise.all(promises)).toBeResolved(); + expect(counter).toBe(100); + }); +}); From ae384b93a96e9bf3c743d980b502aac1904b0868 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 15:23:15 +0200 Subject: [PATCH 34/58] removed number grouping --- spec/Idempotency.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 97a2b3983b..5f850e069c 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -28,7 +28,7 @@ describe_only_db('mongo')('idempotency for cloud code functions', () => { } // Setups beforeEach(async () => { - if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200_000; } + if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; } await reconfigureServer({ appId: Parse.applicationId, masterKey: Parse.masterKey, @@ -87,7 +87,7 @@ describe_only_db('mongo')('idempotency for cloud code functions', () => { if (SIMULATE_TTL) { await deleteRequestEntry('abc-123'); } else { - await new Promise(resolve => setTimeout(resolve, 130_000)); + await new Promise(resolve => setTimeout(resolve, 130000)); } await expectAsync(request(params)).toBeResolved(); expect(counter).toBe(2); From 8c06add1ada97786df169d13c55db77b6f84d74b Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 15:35:30 +0200 Subject: [PATCH 35/58] fixed test description --- spec/Idempotency.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 5f850e069c..e8956be29b 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -5,7 +5,7 @@ const rest = require('../lib/rest'); const auth = require('../lib/Auth'); const uuid = require('uuid'); -describe_only_db('mongo')('idempotency for cloud code functions', () => { +describe_only_db('mongo')('Idempotency', () => { // Parameters /** Enable TTL expiration simulated by removing entry instead of waiting for MongoDB TTL monitor which runss only every 60s, so it can take up to 119s until entry removal - ain't nobody got time for that */ From 7b31767995f99f3ff67d64a3f37b9c4013f28730 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 21:06:57 +0200 Subject: [PATCH 36/58] trying to fix postgres issue --- src/Controllers/DatabaseController.js | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 2a92398c65..d42a99e6f5 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -1754,9 +1754,9 @@ class DatabaseController { const roleClassPromise = this.loadSchema().then((schema) => schema.enforceClassExists('_Role') ); - const idempotencyClassPromise = this.loadSchema().then((schema) => - schema.enforceClassExists('_Idempotency') - ); + const idempotencyClassPromise = this.adapter instanceof MongoStorageAdapter + ? this.loadSchema().then((schema) => schema.enforceClassExists('_Idempotency')) + : Promise.resolve(); const usernameUniqueness = userClassPromise .then(() => @@ -1821,23 +1821,25 @@ class DatabaseController { throw error; }); - const idempotencyIndices = this.adapter instanceof MongoStorageAdapter ? [ - idempotencyClassPromise + const idempotencyRequestIdIndex = this.adapter instanceof MongoStorageAdapter + ? idempotencyClassPromise .then(() => this.adapter.ensureUniqueness( '_Idempotency', requiredIdempotencyFields, ['reqId'] - ) - ) + )) .catch((error) => { logger.warn( 'Unable to ensure uniqueness for idempotency request ID: ', error ); throw error; - }), - idempotencyClassPromise + }) + : Promise.resolve(); + + const idempotencyExpireIndex = this.adapter instanceof MongoStorageAdapter + ? idempotencyClassPromise .then(() => this.adapter.ensureIndex( '_Idempotency', @@ -1846,8 +1848,7 @@ class DatabaseController { 'ttl', false, { ttl: 0 }, - ) - ) + )) .catch((error) => { logger.warn( 'Unable to create TTL index for idempotency expire date: ', @@ -1855,7 +1856,7 @@ class DatabaseController { ); throw error; }) - ] : []; + : Promise.resolve(); const indexPromise = this.adapter.updateSchemaWithIndexes(); @@ -1869,7 +1870,8 @@ class DatabaseController { emailUniqueness, emailCaseInsensitiveIndex, roleUniqueness, - ...idempotencyIndices, + idempotencyRequestIdIndex, + idempotencyExpireIndex, adapterInit, indexPromise, ]); From 40a74f08862240a8d34349e438ee2b526a1754dd Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 22:11:16 +0200 Subject: [PATCH 37/58] added Github readme docs --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index b1ee9b18e2..8a836d6856 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,42 @@ Parse Server allows developers to choose from several options when hosting files `GridFSBucketAdapter` is used by default and requires no setup, but if you're interested in using S3 or Google Cloud Storage, additional configuration information is available in the [Parse Server guide](http://docs.parseplatform.org/parse-server/guide/#configuring-file-adapters). +### Idempodency Enforcement + +**Caution, this is an experimental feature that may not be appropriate for production.** + +This feature deduplicates identical requests that are received by Parse Server mutliple times, typically due to network issues or network adapter access restrictions on mobile operating systems. + +Identical requests are identified by their request header `X-Parse-Request-Id`. Therefore a client request has to include this header for deduplication to be applied. Requests that do not contain this header cannot be deduplicated and are processed normally by Parse Server. This means rolling out this feature to clients is seamless as Parse Server still processes request without this header when this feature is enbabled. + +> This feature needs to be enabled on the client side to send the header and on the server to process the header. Refer to the specific Parse SDK docs to see whether the feature is supported yet. + +#### Configuration example +``` +let api = new ParseServer({ + idempotencyOptions: { + functions: ["*"], // enforce for all functions + classes: ["User"] // enforce only for the User class + jobs: ["jobA", "jobB"], // enforce only for jobA and jobB + ttl: 120 // keep request IDs for 120s + } + ... +} +``` +#### Parameters + +| Parameter | Optional | Type | Default value | Environment variable | Description | +|-----------|----------|--------|---------------|----------------------|-------------| +| `idempotencyOptions` | yes | `Object` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified routes. | +| `idempotencyOptions.functions`| yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS | An array of function names for which the feature should be enabled. Add `*` for all. | +| `idempotencyOptions.jobs` | yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS | An array of job names for which the feature should be enabled. Add `*` for all. | +| `idempotencyOptions.classes` | yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES | An array of class names for which the feature should be enabled. Add `*` for all. | +| `idempotencyOptions.ttl` | yes | `Integer` | `300` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. | + +#### Notes + +- This feature is currently only available for MongoDB and not for Postgres. + ### Logging Parse Server will, by default, log: From 87c6432f6b2d15eae22f8fd70f3b2a59b0a1bbd8 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 22:11:32 +0200 Subject: [PATCH 38/58] added change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d719f9fca2..3ee55a1936 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### master [Full Changelog](https://github.com/parse-community/parse-server/compare/4.2.0...master) +- NEW (EXPERIMENTAL): Idempotency enforcement for client requests. This deduplicates requests where the client intends to send one request to Parse Server but due to network issues the server receives the request multiple times. **Caution, this is an experimental feature that may not be appropriate for production.** [#6744](https://github.com/parse-community/parse-server/issues/6744). Thanks to [Manuel Trezza](https://github.com/mtrezza). ### 4.2.0 [Full Changelog](https://github.com/parse-community/parse-server/compare/4.1.0...4.2.0) From 5c6d30f6f43870873ee097f663d317d4817ebfea Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 25 Jun 2020 23:29:27 +0200 Subject: [PATCH 39/58] refactored tests; fixed some typos --- spec/Idempotency.spec.js | 24 +++++++++++++----------- src/Options/Definitions.js | 5 +++-- src/Options/docs.js | 2 +- src/Options/index.js | 3 ++- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index e8956be29b..278473045f 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -8,7 +8,7 @@ const uuid = require('uuid'); describe_only_db('mongo')('Idempotency', () => { // Parameters /** Enable TTL expiration simulated by removing entry instead of waiting for MongoDB TTL monitor which - runss only every 60s, so it can take up to 119s until entry removal - ain't nobody got time for that */ + runs only every 60s, so it can take up to 119s until entry removal - ain't nobody got time for that */ const SIMULATE_TTL = true; // Helpers async function deleteRequestEntry(reqId) { @@ -26,20 +26,22 @@ describe_only_db('mongo')('Idempotency', () => { '_Idempotency', res.results[0].objectId); } - // Setups - beforeEach(async () => { - if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; } + async function setup(options) { await reconfigureServer({ appId: Parse.applicationId, masterKey: Parse.masterKey, serverURL: Parse.serverURL, - idempotencyOptions: { - ttl: 30, - functions: ["*"], - jobs: ["*"], - classes: ["*"], - }, - }) + idempotencyOptions: options, + }); + } + // Setups + beforeEach(async () => { + if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; } + await setup({ + functions: ["*"], + jobs: ["*"], + classes: ["*"], + }); }); // Tests it('should enforce idempotency for cloud code function', async () => { diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 24a7044435..e909b21e4a 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -522,7 +522,8 @@ module.exports.IdempotencyOptions = { }, "ttl": { "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL", - "help": "The duration in seconds after which a request record is discarded from the database, default is 300s.", - "action": parsers.numberParser("ttl") + "help": "The duration in seconds after which a request record is discarded from the database, defaults to 300s.", + "action": parsers.numberParser("ttl"), + "default": 300 } }; diff --git a/src/Options/docs.js b/src/Options/docs.js index da3e46c9f9..3742ffb760 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -117,6 +117,6 @@ * @property {String[]} classes Array of class names, use `*` for all, default is none. * @property {String[]} functions Array of function names, use `*` for all, default is none. * @property {String[]} jobs Array of job names, use `*` for all, default is none. - * @property {Number} ttl The duration in seconds after which a request record is discarded from the database, default is 300s. + * @property {Number} ttl The duration in seconds after which a request record is discarded from the database, defaults to 300s. */ diff --git a/src/Options/index.js b/src/Options/index.js index 82de1d9f0b..17ff18909f 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -282,6 +282,7 @@ export interface IdempotencyOptions { jobs: ?(string[]); /* Array of class names, use `*` for all, default is none. */ classes: ?(string[]); - /* The duration in seconds after which a request record is discarded from the database, default is 300s. */ + /* The duration in seconds after which a request record is discarded from the database, defaults to 300s. + :DEFAULT: 300 */ ttl: ?number; } From 8ee5e3e685e46dd882bc9c0e2604336fa8c6accf Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 02:25:41 +0200 Subject: [PATCH 40/58] fixed test case --- spec/Idempotency.spec.js | 1 + src/middlewares.js | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 278473045f..71dd18b12d 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -41,6 +41,7 @@ describe_only_db('mongo')('Idempotency', () => { functions: ["*"], jobs: ["*"], classes: ["*"], + ttl: 30, }); }); // Tests diff --git a/src/middlewares.js b/src/middlewares.js index 025d51d17e..bb044504b7 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -412,9 +412,10 @@ export function promiseEnsureIdempotency(req) { // Enable feature only for MongoDB if (!(req.config.database.adapter instanceof MongoStorageAdapter)) { return Promise.resolve(); } // Get request ID + // Get parameters + const { functions, jobs, classes, ttl = 300 } = req.config.idempotencyOptions; const requestId = ((req || {}).headers || {})["x-parse-request-id"]; - if (!requestId) { return Promise.resolve(); } - const { functions, jobs, classes, ttl } = req.config.idempotencyOptions; + if (!requestId || !req.config.idempotencyOptions) { return Promise.resolve(); } // Determine whether idempotency is enabled for current request path const split = req.path.match(/^\/([^\/]*)\/([^\/]*)/i); const route = split[1]; From c0fb4ea85acdc15acc170a8a2be218980ef16845 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 02:54:55 +0200 Subject: [PATCH 41/58] fixed default TTL value --- src/Config.js | 16 ++++++++++++---- src/middlewares.js | 5 ++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Config.js b/src/Config.js index 2077626ff8..9bea94de40 100644 --- a/src/Config.js +++ b/src/Config.js @@ -6,6 +6,7 @@ import AppCache from './cache'; import SchemaCache from './Controllers/SchemaCache'; import DatabaseController from './Controllers/DatabaseController'; import net from 'net'; +import { IdempotencyOptions } from './Options/Definitions'; function removeTrailingSlash(str) { if (!str) { @@ -73,6 +74,7 @@ export class Config { masterKey, readOnlyMasterKey, allowHeaders, + idempotencyOptions, }) { if (masterKey === readOnlyMasterKey) { throw new Error('masterKey and readOnlyMasterKey should be different'); @@ -104,14 +106,20 @@ export class Config { throw 'publicServerURL should be a valid HTTPS URL starting with https://'; } } - this.validateSessionConfiguration(sessionLength, expireInactiveSessions); - this.validateMasterKeyIps(masterKeyIps); - this.validateMaxLimit(maxLimit); - this.validateAllowHeaders(allowHeaders); + this.validateIdempotencyOptions(idempotencyOptions); + } + + static validateIdempotencyOptions(idempotencyOptions) { + if (!idempotencyOptions) { return; } + if (!isNaN(idempotencyOptions.ttl) && idempotencyOptions.ttl <= 0) { + throw 'idempotency TTL value must be greater than 0 seconds'; + } else if (isNaN(idempotencyOptions.ttl)) { + idempotencyOptions.ttl = IdempotencyOptions.ttl.default; + } } static validateAccountLockoutPolicy(accountLockout) { diff --git a/src/middlewares.js b/src/middlewares.js index bb044504b7..c7e5bd72a2 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -411,9 +411,8 @@ export function promiseEnforceMasterKeyAccess(request) { export function promiseEnsureIdempotency(req) { // Enable feature only for MongoDB if (!(req.config.database.adapter instanceof MongoStorageAdapter)) { return Promise.resolve(); } - // Get request ID // Get parameters - const { functions, jobs, classes, ttl = 300 } = req.config.idempotencyOptions; + const { functions, jobs, classes, ttl } = req.config.idempotencyOptions; const requestId = ((req || {}).headers || {})["x-parse-request-id"]; if (!requestId || !req.config.idempotencyOptions) { return Promise.resolve(); } // Determine whether idempotency is enabled for current request path @@ -424,7 +423,7 @@ export function promiseEnsureIdempotency(req) { const jobMatch = jobs && route == "jobs" && (jobs.includes("*") || jobs.includes(item)); const classMatch = classes && route == "classes" && (classes.includes("*") || classes.includes(item)); if (!functionMatch && !jobMatch && !classMatch) { return Promise.resolve(); } - // Try to track request + // Try to store request const expiryDate = new Date(new Date().setSeconds(new Date().getSeconds() + ttl)); return rest.create( req.config, From d409ea1db5db14b03ba91997f5c138323924a30f Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 03:06:16 +0200 Subject: [PATCH 42/58] Travis CI Trigger From 3ec141b3e530d16240ace24014a849a61563be44 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 03:26:06 +0200 Subject: [PATCH 43/58] Travis CI Trigger From 53d623a7e62cb18c47a98e70fff42fde3493fead Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 03:40:41 +0200 Subject: [PATCH 44/58] Travis CI Trigger From 471766398d45984cd6829affc453814b841618d7 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 17:55:44 +0200 Subject: [PATCH 45/58] added test case to increase coverage --- spec/Idempotency.spec.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 71dd18b12d..b1976b0e03 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -166,4 +166,26 @@ describe_only_db('mongo')('Idempotency', () => { await expectAsync(Promise.all(promises)).toBeResolved(); expect(counter).toBe(100); }); + + it('should re-throw any other error unchanged when writing request entry fails for any other reason', async () => { + // Throw on DB write + spyOn(rest, 'create').and.callFake(() => { + throw new Parse.Error(0, "some other error"); + }); + // Run function + Parse.Cloud.define('myFunction', () => {}); + const params = { + method: 'POST', + url: 'http://localhost:8378/1/functions/myFunction', + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("some other error"); + }); + }); }); From b69ca44745c10552f0c919bf772e5abb6734041d Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 18:07:33 +0200 Subject: [PATCH 46/58] Trigger Travis CI From 18b318659f9fce41da77ff0921285815b9574c44 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 21:15:19 +0200 Subject: [PATCH 47/58] changed configuration syntax to use regex; added test cases --- README.md | 4 +- spec/Idempotency.spec.js | 82 ++++++++++++++++++++++++++++++++++++-- src/Config.js | 13 +++++- src/Options/Definitions.js | 19 +++------ src/Options/docs.js | 4 +- src/Options/index.js | 9 ++--- src/middlewares.js | 29 +++++++++----- 7 files changed, 118 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 8a836d6856..7d8b5987f2 100644 --- a/README.md +++ b/README.md @@ -415,9 +415,7 @@ let api = new ParseServer({ | Parameter | Optional | Type | Default value | Environment variable | Description | |-----------|----------|--------|---------------|----------------------|-------------| | `idempotencyOptions` | yes | `Object` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified routes. | -| `idempotencyOptions.functions`| yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS | An array of function names for which the feature should be enabled. Add `*` for all. | -| `idempotencyOptions.jobs` | yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS | An array of job names for which the feature should be enabled. Add `*` for all. | -| `idempotencyOptions.classes` | yes | `Array` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES | An array of class names for which the feature should be enabled. Add `*` for all. | +| `idempotencyOptions.paths`| yes | `Array` | `[]` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of paths for which the feature should be enabled. The mount path must not be included, for example instead of `/parse/functions/myFunction` specifiy `functions/myFunction`. The entries are interpreted as regular expression, for example `functions/.*` matches all functions, `jobs/.*` matches all jobs, `classes/.*` matches all classes, `.*` matches all paths. | | `idempotencyOptions.ttl` | yes | `Integer` | `300` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. | #### Notes diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index b1976b0e03..33d85eb25f 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -1,5 +1,6 @@ 'use strict'; const Config = require('../lib/Config'); +const Definitions = require('../lib/Options/Definitions'); const request = require('../lib/request'); const rest = require('../lib/rest'); const auth = require('../lib/Auth'); @@ -38,9 +39,13 @@ describe_only_db('mongo')('Idempotency', () => { beforeEach(async () => { if (SIMULATE_TTL) { jasmine.DEFAULT_TIMEOUT_INTERVAL = 200000; } await setup({ - functions: ["*"], - jobs: ["*"], - classes: ["*"], + paths: [ + "functions/.*", + "jobs/.*", + "classes/.*", + "users", + "installations" + ], ttl: 30, }); }); @@ -144,6 +149,62 @@ describe_only_db('mongo')('Idempotency', () => { expect(counter).toBe(1); }); + it('should enforce idempotency for user object creation', async () => { + // Declare trigger + let counter = 0; + Parse.Cloud.afterSave('_User', () => { + counter++; + }); + // Create object + const params = { + method: 'POST', + url: 'http://localhost:8378/1/users', + body: { + username: "user", + password: "pass" + }, + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await expectAsync(request(params)).toBeResolved(); + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("Duplicate request"); + }); + expect(counter).toBe(1); + }); + + it('should enforce idempotency for installation object creation', async () => { + // Declare trigger + let counter = 0; + Parse.Cloud.afterSave('_Installation', () => { + counter++; + }); + // Create object + const params = { + method: 'POST', + url: 'http://localhost:8378/1/installations', + body: { + installationId: "1", + deviceType: "ios" + }, + headers: { + 'X-Parse-Application-Id': Parse.applicationId, + 'X-Parse-Master-Key': Parse.masterKey, + 'X-Parse-Request-Id': 'abc-123' + } + }; + await expectAsync(request(params)).toBeResolved(); + await request(params).then(fail, e => { + expect(e.status).toEqual(400); + expect(e.data.error).toEqual("Duplicate request"); + }); + expect(counter).toBe(1); + }); + it('should not interfere with calls of different request ID', async () => { // Declare trigger let counter = 0; @@ -188,4 +249,19 @@ describe_only_db('mongo')('Idempotency', () => { expect(e.data.error).toEqual("some other error"); }); }); + + it('should use default configuration when none is set', async () => { + // Configure server with minimal params + await setup({}); + expect(Config.get(Parse.applicationId).idempotencyOptions.ttl).toBe(Definitions.IdempotencyOptions.ttl.default); + expect(Config.get(Parse.applicationId).idempotencyOptions.paths).toBe(Definitions.IdempotencyOptions.paths.default); + }); + + it('should throw on invalid configuration', async () => { + // Configure server with invalid params + await expectAsync(setup({ paths: 1 })).toBeRejected(); + await expectAsync(setup({ ttl: 'a' })).toBeRejected(); + await expectAsync(setup({ ttl: 0 })).toBeRejected(); + await expectAsync(setup({ ttl: -1 })).toBeRejected(); + }); }); diff --git a/src/Config.js b/src/Config.js index 9bea94de40..feae7dffd1 100644 --- a/src/Config.js +++ b/src/Config.js @@ -7,6 +7,8 @@ import SchemaCache from './Controllers/SchemaCache'; import DatabaseController from './Controllers/DatabaseController'; import net from 'net'; import { IdempotencyOptions } from './Options/Definitions'; +import { isType } from 'graphql'; +import { isTypedArray } from 'lodash'; function removeTrailingSlash(str) { if (!str) { @@ -115,10 +117,17 @@ export class Config { static validateIdempotencyOptions(idempotencyOptions) { if (!idempotencyOptions) { return; } - if (!isNaN(idempotencyOptions.ttl) && idempotencyOptions.ttl <= 0) { + if (idempotencyOptions.ttl === undefined) { + idempotencyOptions.ttl = IdempotencyOptions.ttl.default; + } else if (!isNaN(idempotencyOptions.ttl) && idempotencyOptions.ttl <= 0) { throw 'idempotency TTL value must be greater than 0 seconds'; } else if (isNaN(idempotencyOptions.ttl)) { - idempotencyOptions.ttl = IdempotencyOptions.ttl.default; + throw 'idempotency TTL value must be a number'; + } + if (!idempotencyOptions.paths) { + idempotencyOptions.paths = IdempotencyOptions.paths.default; + } else if (!(idempotencyOptions.paths instanceof Array)) { + throw 'idempotency paths must be of an array of strings'; } } diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index e909b21e4a..603e7747cc 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -505,20 +505,11 @@ module.exports.LiveQueryServerOptions = { } }; module.exports.IdempotencyOptions = { - "classes": { - "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_CLASSES", - "help": "Array of class names, use `*` for all, default is none.", - "action": parsers.arrayParser - }, - "functions": { - "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_FUNCTIONS", - "help": "Array of function names, use `*` for all, default is none.", - "action": parsers.arrayParser - }, - "jobs": { - "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_JOBS", - "help": "Array of job names, use `*` for all, default is none.", - "action": parsers.arrayParser + "paths": { + "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS", + "help": "An array of paths for which the feature should be enabled. The mount path must not be included, for example instead of `/parse/functions/myFunction` specifiy `functions/myFunction`. The entries are interpreted as regular expression, for example `functions/.*` matches all functions, `jobs/.*` matches all jobs, `classes/.*` matches all classes, `.*` matches all paths.", + "action": parsers.arrayParser, + "default": [] }, "ttl": { "env": "PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL", diff --git a/src/Options/docs.js b/src/Options/docs.js index 3742ffb760..1cadf8a76a 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -114,9 +114,7 @@ /** * @interface IdempotencyOptions - * @property {String[]} classes Array of class names, use `*` for all, default is none. - * @property {String[]} functions Array of function names, use `*` for all, default is none. - * @property {String[]} jobs Array of job names, use `*` for all, default is none. + * @property {String[]} paths An array of paths for which the feature should be enabled. The mount path must not be included, for example instead of `/parse/functions/myFunction` specifiy `functions/myFunction`. The entries are interpreted as regular expression, for example `functions/.*` matches all functions, `jobs/.*` matches all jobs, `classes/.*` matches all classes, `.*` matches all paths. * @property {Number} ttl The duration in seconds after which a request record is discarded from the database, defaults to 300s. */ diff --git a/src/Options/index.js b/src/Options/index.js index 17ff18909f..c0efdd6a27 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -276,12 +276,9 @@ export interface LiveQueryServerOptions { } export interface IdempotencyOptions { - /* Array of function names, use `*` for all, default is none. */ - functions: ?(string[]); - /* Array of job names, use `*` for all, default is none. */ - jobs: ?(string[]); - /* Array of class names, use `*` for all, default is none. */ - classes: ?(string[]); + /* An array of paths for which the feature should be enabled. The mount path must not be included, for example instead of `/parse/functions/myFunction` specifiy `functions/myFunction`. The entries are interpreted as regular expression, for example `functions/.*` matches all functions, `jobs/.*` matches all jobs, `classes/.*` matches all classes, `.*` matches all paths. + :DEFAULT: [] */ + paths: ?(string[]); /* The duration in seconds after which a request record is discarded from the database, defaults to 300s. :DEFAULT: 300 */ ttl: ?number; diff --git a/src/middlewares.js b/src/middlewares.js index c7e5bd72a2..490d13783d 100644 --- a/src/middlewares.js +++ b/src/middlewares.js @@ -412,22 +412,29 @@ export function promiseEnsureIdempotency(req) { // Enable feature only for MongoDB if (!(req.config.database.adapter instanceof MongoStorageAdapter)) { return Promise.resolve(); } // Get parameters - const { functions, jobs, classes, ttl } = req.config.idempotencyOptions; + const config = req.config; const requestId = ((req || {}).headers || {})["x-parse-request-id"]; - if (!requestId || !req.config.idempotencyOptions) { return Promise.resolve(); } + const { paths, ttl } = config.idempotencyOptions; + if (!requestId || !config.idempotencyOptions) { return Promise.resolve(); } + // Request path may contain trailing slashes, depending on the original request, so remove + // leading and trailing slashes to make it easier to specify paths in the configuration + const reqPath = req.path.replace(/^\/|\/$/, ''); // Determine whether idempotency is enabled for current request path - const split = req.path.match(/^\/([^\/]*)\/([^\/]*)/i); - const route = split[1]; - const item = split[2]; - const functionMatch = functions && route == "functions" && (functions.includes("*") || functions.includes(item)); - const jobMatch = jobs && route == "jobs" && (jobs.includes("*") || jobs.includes(item)); - const classMatch = classes && route == "classes" && (classes.includes("*") || classes.includes(item)); - if (!functionMatch && !jobMatch && !classMatch) { return Promise.resolve(); } + let match = false; + for (const path of paths) { + // Assume one wants a path to always match from the beginning to prevent any mistakes + const regex = new RegExp(path.charAt(0) === '^' ? path : '^' + path); + if (reqPath.match(regex)) { + match = true; + break; + } + } + if (!match) { return Promise.resolve(); } // Try to store request const expiryDate = new Date(new Date().setSeconds(new Date().getSeconds() + ttl)); return rest.create( - req.config, - auth.master(req.config), + config, + auth.master(config), '_Idempotency', { reqId: requestId, expire: Parse._encode(expiryDate) } ).catch (e => { From abc5c7ed6389b6b8953438b57baa444af511bc00 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 21:57:54 +0200 Subject: [PATCH 48/58] removed unused vars --- src/Config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Config.js b/src/Config.js index feae7dffd1..214e22ca57 100644 --- a/src/Config.js +++ b/src/Config.js @@ -7,8 +7,6 @@ import SchemaCache from './Controllers/SchemaCache'; import DatabaseController from './Controllers/DatabaseController'; import net from 'net'; import { IdempotencyOptions } from './Options/Definitions'; -import { isType } from 'graphql'; -import { isTypedArray } from 'lodash'; function removeTrailingSlash(str) { if (!str) { From 66fe783cce6769a6bc9f159126d1da7cc4d71448 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 22:25:02 +0200 Subject: [PATCH 49/58] removed IdempotencyRouter --- src/ParseServer.js | 2 -- src/Routers/IdempotencyRouter.js | 21 --------------------- 2 files changed, 23 deletions(-) delete mode 100644 src/Routers/IdempotencyRouter.js diff --git a/src/ParseServer.js b/src/ParseServer.js index 9077ed6502..081a8c15a4 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -36,7 +36,6 @@ import { SessionsRouter } from './Routers/SessionsRouter'; import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; import { AudiencesRouter } from './Routers/AudiencesRouter'; -import { IdempotencyRouter } from './Routers/IdempotencyRouter'; import { AggregateRouter } from './Routers/AggregateRouter'; import { ParseServerRESTController } from './ParseServerRESTController'; import * as controllers from './Controllers'; @@ -232,7 +231,6 @@ class ParseServer { new HooksRouter(), new CloudCodeRouter(), new AudiencesRouter(), - new IdempotencyRouter(), new AggregateRouter(), ]; diff --git a/src/Routers/IdempotencyRouter.js b/src/Routers/IdempotencyRouter.js deleted file mode 100644 index 427d08e370..0000000000 --- a/src/Routers/IdempotencyRouter.js +++ /dev/null @@ -1,21 +0,0 @@ -import ClassesRouter from './ClassesRouter'; -import * as middleware from '../middlewares'; - -export class IdempotencyRouter extends ClassesRouter { - className() { - return '_Idempotency'; - } - - mountRoutes() { - this.route( - 'POST', - '/idempotency', - middleware.promiseEnforceMasterKeyAccess, - req => { - return this.handleCreate(req); - } - ); - } -} - -export default IdempotencyRouter; From 38fc9527b73a63c019991efd2cb03523ad735d5f Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 22:35:15 +0200 Subject: [PATCH 50/58] Trigger Travis CI From 46107b724e70f06283f8a42bc997b5b809f89b19 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Fri, 26 Jun 2020 23:57:29 +0200 Subject: [PATCH 51/58] updated docs --- README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7d8b5987f2..9081cf6f38 100644 --- a/README.md +++ b/README.md @@ -398,25 +398,24 @@ Identical requests are identified by their request header `X-Parse-Request-Id`. > This feature needs to be enabled on the client side to send the header and on the server to process the header. Refer to the specific Parse SDK docs to see whether the feature is supported yet. +Deduplication is only done for object creation and update (`POST` and `PUT` requests). Deduplication is not done for object finding and deletion (`GET` and `DELETE` requests), as these operations are already idempotent by definition. + #### Configuration example ``` let api = new ParseServer({ idempotencyOptions: { - functions: ["*"], // enforce for all functions - classes: ["User"] // enforce only for the User class - jobs: ["jobA", "jobB"], // enforce only for jobA and jobB - ttl: 120 // keep request IDs for 120s + paths: [".*"], // enforce for all requests + ttl: 120 // keep request IDs for 120s } - ... } ``` #### Parameters -| Parameter | Optional | Type | Default value | Environment variable | Description | -|-----------|----------|--------|---------------|----------------------|-------------| -| `idempotencyOptions` | yes | `Object` | `undefined` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified routes. | -| `idempotencyOptions.paths`| yes | `Array` | `[]` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of paths for which the feature should be enabled. The mount path must not be included, for example instead of `/parse/functions/myFunction` specifiy `functions/myFunction`. The entries are interpreted as regular expression, for example `functions/.*` matches all functions, `jobs/.*` matches all jobs, `classes/.*` matches all classes, `.*` matches all paths. | -| `idempotencyOptions.ttl` | yes | `Integer` | `300` | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. | +| Parameter | Optional | Type | Default value | Example values | Environment variable | Description | +|-----------|----------|--------|---------------|-----------|-----------|-------------| +| `idempotencyOptions` | yes | `Object` | `undefined` | | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified routes. | +| `idempotencyOptions.paths`| yes | `Array` | `[]` | `.*` (all paths, includes the examples below),
`functions/.*` (all functions),
`jobs/.*` (all jobs),
`classes/.*` (all classes),
`functions/.*` (all functions),
`users` (user creation / update),
`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. | +| `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. | #### Notes From 7bf5a6a1ec3114b1d1b7b5f84695e3ad3fec067b Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Sat, 27 Jun 2020 00:03:08 +0200 Subject: [PATCH 52/58] updated docs --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9081cf6f38..5e97172d57 100644 --- a/README.md +++ b/README.md @@ -411,12 +411,16 @@ let api = new ParseServer({ ``` #### Parameters + + | Parameter | Optional | Type | Default value | Example values | Environment variable | Description | |-----------|----------|--------|---------------|-----------|-----------|-------------| -| `idempotencyOptions` | yes | `Object` | `undefined` | | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified routes. | +| `idempotencyOptions` | yes | `Object` | `undefined` | | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified paths. | | `idempotencyOptions.paths`| yes | `Array` | `[]` | `.*` (all paths, includes the examples below),
`functions/.*` (all functions),
`jobs/.*` (all jobs),
`classes/.*` (all classes),
`functions/.*` (all functions),
`users` (user creation / update),
`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. | | `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. | + + #### Notes - This feature is currently only available for MongoDB and not for Postgres. From bf60c8c54a8f8e0e985c471e9404e5716f2283d7 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Sat, 27 Jun 2020 00:06:51 +0200 Subject: [PATCH 53/58] updated docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5e97172d57..0a588fc062 100644 --- a/README.md +++ b/README.md @@ -411,7 +411,7 @@ let api = new ParseServer({ ``` #### Parameters - +
| Parameter | Optional | Type | Default value | Example values | Environment variable | Description | |-----------|----------|--------|---------------|-----------|-----------|-------------| @@ -419,7 +419,7 @@ let api = new ParseServer({ | `idempotencyOptions.paths`| yes | `Array` | `[]` | `.*` (all paths, includes the examples below),
`functions/.*` (all functions),
`jobs/.*` (all jobs),
`classes/.*` (all classes),
`functions/.*` (all functions),
`users` (user creation / update),
`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. | | `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. | - +
#### Notes From a28685e9ce3823809940460d5c0817482d5777b5 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Sat, 27 Jun 2020 00:10:56 +0200 Subject: [PATCH 54/58] updated docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a588fc062..07eef9f754 100644 --- a/README.md +++ b/README.md @@ -411,7 +411,7 @@ let api = new ParseServer({ ``` #### Parameters -
+ | Parameter | Optional | Type | Default value | Example values | Environment variable | Description | |-----------|----------|--------|---------------|-----------|-----------|-------------| @@ -419,7 +419,7 @@ let api = new ParseServer({ | `idempotencyOptions.paths`| yes | `Array` | `[]` | `.*` (all paths, includes the examples below),
`functions/.*` (all functions),
`jobs/.*` (all jobs),
`classes/.*` (all classes),
`functions/.*` (all functions),
`users` (user creation / update),
`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. | | `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. | -
+ #### Notes From a661a3e75ccc2d8b3298d15c4b2fabb8ea92e5a4 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Sat, 27 Jun 2020 00:12:13 +0200 Subject: [PATCH 55/58] update docs --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 07eef9f754..ae8a14d175 100644 --- a/README.md +++ b/README.md @@ -411,16 +411,12 @@ let api = new ParseServer({ ``` #### Parameters - - | Parameter | Optional | Type | Default value | Example values | Environment variable | Description | |-----------|----------|--------|---------------|-----------|-----------|-------------| | `idempotencyOptions` | yes | `Object` | `undefined` | | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_OPTIONS | Setting this enables idempotency enforcement for the specified paths. | | `idempotencyOptions.paths`| yes | `Array` | `[]` | `.*` (all paths, includes the examples below),
`functions/.*` (all functions),
`jobs/.*` (all jobs),
`classes/.*` (all classes),
`functions/.*` (all functions),
`users` (user creation / update),
`installations` (installation creation / update) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_PATHS | An array of path patterns that have to match the request path for request deduplication to be enabled. The mount path must not be included, for example to match the request path `/parse/functions/myFunction` specifiy the path pattern `functions/myFunction`. A trailing slash of the request path is ignored, for example the path pattern `functions/myFunction` matches both `/parse/functions/myFunction` and `/parse/functions/myFunction/`. | | `idempotencyOptions.ttl` | yes | `Integer` | `300` | `60` (60 seconds) | PARSE_SERVER_EXPERIMENTAL_IDEMPOTENCY_TTL | The duration in seconds after which a request record is discarded from the database. Duplicate requests due to network issues can be expected to arrive within milliseconds up to several seconds. This value must be greater than `0`. | - - #### Notes - This feature is currently only available for MongoDB and not for Postgres. From b408ac55abaf05595274cff8e49360f1d1874bc7 Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Sat, 27 Jun 2020 00:33:28 +0200 Subject: [PATCH 56/58] Trigger Travis CI From 220236fd2bec0360a39889eb6e63212c2f06efff Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 9 Jul 2020 20:58:10 +0200 Subject: [PATCH 57/58] fixed coverage --- spec/Idempotency.spec.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 33d85eb25f..7ea5ad1606 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -229,11 +229,7 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should re-throw any other error unchanged when writing request entry fails for any other reason', async () => { - // Throw on DB write - spyOn(rest, 'create').and.callFake(() => { - throw new Parse.Error(0, "some other error"); - }); - // Run function + spyOn(rest, 'create').and.rejectWith(new Parse.Error(0, "some other error")); Parse.Cloud.define('myFunction', () => {}); const params = { method: 'POST', From 6c03028b039f4cf26bc253c34218d5cf2d53fcfa Mon Sep 17 00:00:00 2001 From: Manuel Trezza Date: Thu, 9 Jul 2020 20:59:25 +0200 Subject: [PATCH 58/58] removed code comments --- spec/Idempotency.spec.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/spec/Idempotency.spec.js b/spec/Idempotency.spec.js index 7ea5ad1606..2d0e99aa19 100644 --- a/spec/Idempotency.spec.js +++ b/spec/Idempotency.spec.js @@ -51,12 +51,10 @@ describe_only_db('mongo')('Idempotency', () => { }); // Tests it('should enforce idempotency for cloud code function', async () => { - // Declare function let counter = 0; Parse.Cloud.define('myFunction', () => { counter++; }); - // Run function const params = { method: 'POST', url: 'http://localhost:8378/1/functions/myFunction', @@ -76,12 +74,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should delete request entry after TTL', async () => { - // Declare function let counter = 0; Parse.Cloud.define('myFunction', () => { counter++; }); - // Run function const params = { method: 'POST', url: 'http://localhost:8378/1/functions/myFunction', @@ -102,12 +98,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should enforce idempotency for cloud code jobs', async () => { - // Declare job let counter = 0; Parse.Cloud.job('myJob', () => { counter++; }); - // Run job const params = { method: 'POST', url: 'http://localhost:8378/1/jobs/myJob', @@ -126,12 +120,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should enforce idempotency for class object creation', async () => { - // Declare trigger let counter = 0; Parse.Cloud.afterSave('MyClass', () => { counter++; }); - // Create object const params = { method: 'POST', url: 'http://localhost:8378/1/classes/MyClass', @@ -150,12 +142,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should enforce idempotency for user object creation', async () => { - // Declare trigger let counter = 0; Parse.Cloud.afterSave('_User', () => { counter++; }); - // Create object const params = { method: 'POST', url: 'http://localhost:8378/1/users', @@ -178,12 +168,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should enforce idempotency for installation object creation', async () => { - // Declare trigger let counter = 0; Parse.Cloud.afterSave('_Installation', () => { counter++; }); - // Create object const params = { method: 'POST', url: 'http://localhost:8378/1/installations', @@ -206,12 +194,10 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should not interfere with calls of different request ID', async () => { - // Declare trigger let counter = 0; Parse.Cloud.afterSave('MyClass', () => { counter++; }); - // Create 100 objects const promises = [...Array(100).keys()].map(() => { const params = { method: 'POST', @@ -247,14 +233,12 @@ describe_only_db('mongo')('Idempotency', () => { }); it('should use default configuration when none is set', async () => { - // Configure server with minimal params await setup({}); expect(Config.get(Parse.applicationId).idempotencyOptions.ttl).toBe(Definitions.IdempotencyOptions.ttl.default); expect(Config.get(Parse.applicationId).idempotencyOptions.paths).toBe(Definitions.IdempotencyOptions.paths.default); }); it('should throw on invalid configuration', async () => { - // Configure server with invalid params await expectAsync(setup({ paths: 1 })).toBeRejected(); await expectAsync(setup({ ttl: 'a' })).toBeRejected(); await expectAsync(setup({ ttl: 0 })).toBeRejected();