Feat/bulk/all get job server

README.md

@@ -134,10 +134,14 @@ After these have been set up, set the environment variables according to the tab
 |BULK_UPLOAD_MAX_NUM|No|Maximum number of links that can be bulk uploaded at once. Defaults to 1000|
 |BULK_UPLOAD_RANDOM_STR_LENGTH|No|String length of randomly generated shortUrl in bulk upload. Defaults to 8|
 |BULK_QR_CODE_BATCH_SIZE|No|Maximum batch size of QR codes to generate in a single Lambda run. Defaults to 1000|
+|BULK_QR_CODE_BUCKET_URL|No|Link to download QR codes from|
+|ACTIVATE_BULK_QR_CODE_GENERATION|No|Whether to start Lambda for bulk QR code generation or not. Defaults to false|
 |REPLICA_URI|Yes|The postgres connection string, e.g. `postgres://postgres:postgres@postgres:5432/postgres`|
 |SQS_BULK_QRCODE_GENERATE_START_URL|No|The SQS queue for starting QR code bulk generation Lambda|
 |SQS_TIMEOUT|No|Duration of time in ms for sending to SQS queue before timeout. Defaults to 10000ms (10s)|
 |SQS_REGION|No|AWS Region of SQS queue for starting QR code bulk generation Lambda|
+|JOB_POLL_ATTEMPTS|No|Number of attempts for long polling of job status before timeout of 408 is returned. Defaults to 12|
+|JOB_POLL_INTERVAL|No|Interval of time between attempts for long polling of job status in ms. Defaults to 5000ms (5s)|
 
 
 #### Serverless functions for link migration

docker-compose.yml

@@ -56,6 +56,11 @@ services:
       - SQS_TIMEOUT=10000
       - SQS_REGION=
       - BULK_QR_CODE_BATCH_SIZE=1000
+      - LAMBDA_HASH_SECRET=
+      - ACTIVATE_BULK_QR_CODE_GENERATION=false
+      - JOB_POLL_ATTEMPTS=12
+      - JOB_POLL_INTERVAL=5000
+      - BULK_QR_CODE_BUCKET_URL=
     volumes:
       - ./public:/usr/src/gogovsg/public
       - ./src:/usr/src/gogovsg/src

src/server/api/callback.ts

@@ -0,0 +1,32 @@
+import Express from 'express'
+import Joi from '@hapi/joi'
+import { createValidator } from 'express-joi-validation'
+import { container } from '../util/inversify'
+import { JobController } from '../modules/job'
+import { DependencyIds } from '../constants'
+
+const router = Express.Router()
+const validator = createValidator({ passError: true })
+
+const jobController = container.get<JobController>(DependencyIds.jobController)
+
+const jobItemCallbackSchema = Joi.object({
+  jobItemId: Joi.string().required(),
+  status: Joi.object()
+    .keys({
+      isSuccess: Joi.boolean().required(),
+      errorMessage: Joi.string().allow(null, ''),
+    })
+    .required(),
+})
+/**
+ * Update job status based on callback.
+ */
+router.post(
+  '/',
+  validator.body(jobItemCallbackSchema),
+  jobController.updateJobItem,
+  jobController.updateJob,
+)
+
+module.exports = router

src/server/api/index.ts

@@ -1,7 +1,7 @@
 import Express from 'express'
 import jsonMessage from '../util/json'
 import { ERROR_404_PATH } from '../constants'
-import { displayHostname } from '../config'
+import { displayHostname, lambdaHashSecret } from '../config'
 import assetVariant from '../../shared/util/asset-variant'
 
 const router = Express.Router()
@@ -30,6 +30,31 @@ function userGuard(
   next()
 }
 
+/**
+ * To protect lambda callback route. Temporary.
+ * */
+const lambdaCallbackGuard = (
+  req: Express.Request,
+  res: Express.Response,
+  next: Express.NextFunction,
+) => {
+  const authToken = req.headers.authorization
+  if (!authToken) {
+    res.unauthorized()
+    return
+  }
+  const [headerKey, key] = authToken.trim().split(' ')
+  if (
+    headerKey.toLowerCase() !== 'bearer' ||
+    !key ||
+    key !== lambdaHashSecret
+  ) {
+    res.unauthorized()
+    return
+  }
+  next()
+}
+
 /**
  *  Preprocess request parameters.
  * */
@@ -52,6 +77,8 @@ router.use('/link-stats', userGuard, require('./link-statistics'))
 router.use('/link-audit', userGuard, require('./link-audit'))
 router.use('/directory', userGuard, require('./directory'))
 
+router.use('/callback', lambdaCallbackGuard, require('./callback'))
+
 router.use((_, res) => {
   res.status(404).render(ERROR_404_PATH, {
     assetVariant,

src/server/api/user/index.ts

@@ -10,6 +10,7 @@ import {
 } from '../../../shared/constants'
 import {
   ownershipTransferSchema,
+  pollJobInformationSchema,
   tagRetrievalSchema,
   urlBulkSchema,
   urlEditSchema,
@@ -163,6 +164,14 @@ router.get(
   userController.getTagsWithConditions,
 )
 
+router.get(
+  '/job/status',
+  validator.query(pollJobInformationSchema),
+  jobController.pollJobStatusUpdate,
+)
+
+router.get('/job/latest', jobController.getLatestJob)
+
 router.get('/message', userController.getUserMessage)
 
 router.get('/announcement', userController.getUserAnnouncement)

src/server/api/user/validators.ts

@@ -142,3 +142,7 @@ export const ownershipTransferSchema = Joi.object({
   shortUrl: Joi.string().required(),
   newUserEmail: Joi.string().required(),
 })
+
+export const pollJobInformationSchema = Joi.object({
+  jobId: Joi.number().required(),
+})

src/server/config.ts

@@ -251,3 +251,11 @@ export const bulkUploadRandomStrLength: number =
   Number(process.env.BULK_UPLOAD_RANDOM_STR_LENGTH) || 8
 export const qrCodeJobBatchSize: number =
   Number(process.env.BULK_QR_CODE_BATCH_SIZE) || 1000
+export const qrCodeBucketUrl: string = process.env.BULK_QR_CODE_BUCKET_URL || ''
+export const shouldGenerateQRCodes: boolean =
+  process.env.ACTIVATE_BULK_QR_CODE_GENERATION === 'true'
+export const lambdaHashSecret: string = process.env.LAMBDA_HASH_SECRET as string
+export const jobPollInterval: number =
+  Number(process.env.JOB_POLL_INTERVAL) || 5000 // in ms
+export const jobPollAttempts: number =
+  Number(process.env.JOB_POLL_ATTEMPTS) || 12

src/server/inversify.config.ts

@@ -76,7 +76,7 @@ import { FileCheckController, UrlCheckController } from './modules/threat'
 import { QrCodeService } from './modules/qr/services'
 import { QrCodeController } from './modules/qr'
 import TagManagementService from './modules/user/services/TagManagementService'
-import JobManagementService from './modules/job/services/JobManagementService'
+import { JobManagementService } from './modules/job/services'
 import { BulkService } from './modules/bulk/services'
 import { BulkController } from './modules/bulk'
 import { SQSService } from './services/sqs'

src/server/models/job.ts

@@ -1,13 +1,14 @@
 import Sequelize from 'sequelize'
 import { IdType } from '../../types/server/models'
-import { JobItemStatusEnum } from '../repositories/enums'
+import { JobItemStatusEnum, JobStatusEnum } from '../repositories/enums'
 import { sequelize } from '../util/sequelize'
 
 export interface JobType extends IdType, Sequelize.Model {
   readonly uuid: string
   readonly userId: Number
   readonly createdAt: string
   readonly updatedAt: string
+  readonly status: JobStatusEnum
 }
 
 type JobStatic = typeof Sequelize.Model & {
@@ -22,6 +23,16 @@ export const Job = <JobStatic>sequelize.define(
       defaultValue: Sequelize.UUIDV4,
       unique: true,
     },
+    status: {
+      type: Sequelize.ENUM,
+      values: [
+        JobStatusEnum.InProgress,
+        JobStatusEnum.Success,
+        JobStatusEnum.Failure,
+      ],
+      defaultValue: JobItemStatusEnum.InProgress,
+      allowNull: false,
+    },
   },
   {
     defaultScope: {
@@ -35,6 +46,7 @@ export interface JobItemType extends IdType, Sequelize.Model {
   readonly message: string
   readonly params: JSON
   readonly jobId: number
+  readonly jobItemId: string
   readonly createdAt: string
   readonly updatedAt: string
 }
@@ -51,20 +63,23 @@ export const JobItem = <JobItemStatic>sequelize.define(
       values: [
         JobItemStatusEnum.InProgress,
         JobItemStatusEnum.Success,
-        JobItemStatusEnum.Failed,
+        JobItemStatusEnum.Failure,
       ],
       defaultValue: JobItemStatusEnum.InProgress,
       allowNull: false,
     },
     message: {
       type: Sequelize.STRING,
       defaultValue: '',
-      allowNull: false,
     },
     params: {
       type: Sequelize.JSON,
       allowNull: false,
     },
+    jobItemId: {
+      type: Sequelize.STRING,
+      allowNull: false,
+    },
   },
   {
     defaultScope: {

src/server/modules/bulk/BulkController.ts

@@ -6,6 +6,7 @@ import { DependencyIds } from '../../constants'
 import { BulkService } from './interfaces'
 import { UrlManagementService } from '../user/interfaces'
 import dogstatsd from '../../util/dogstatsd'
+import { logger, shouldGenerateQRCodes } from '../../config'
 
 @injectable()
 export class BulkController {
@@ -62,12 +63,18 @@ export class BulkController {
       return
     }
 
-    // put jobParamsList on the req body so that it can be used by JobController
-    req.body.jobParamsList = urlMappings
-
     dogstatsd.increment('bulk.hash.success', 1, 1)
-    res.ok(jsonMessage(`${urlMappings.length} links created`))
-    next()
+    if (shouldGenerateQRCodes) {
+      logger.info('shouldGenerateQRCodes true, triggering QR code generation')
+      // put jobParamsList on the req body so that it can be used by JobController
+      req.body.jobParamsList = urlMappings
+      next()
+    } else {
+      logger.info(
+        'shouldGenerateQRCodes false, not triggering QR code generation',
+      )
+      res.ok({ count: urlMappings.length })
+    }
   }
 }