feat: support workers in docker SOFIE-4071 (#236)

Author: Julusian

.editorconfig

@@ -2,4 +2,9 @@
 indent_style = tab
 
 [*.{cs,js,ts,json}]
-indent_size = 4
+indent_size = 4
+
+[*.{yml,yaml}]
+indent_size = 2
+tab_width = 2
+indent_style = space

.github/workflows/publish-prerelease-docker.yaml

@@ -62,7 +62,7 @@ jobs:
 
     strategy:
       matrix:
-        repo: [package-manager, workforce, http-server, quantel-http-transformer-proxy]
+        repo: [package-manager, workforce, http-server, quantel-http-transformer-proxy, worker]
 
     steps:
       - uses: actions/checkout@v5

apps/appcontainer-node/packages/generic/src/appContainer.ts

@@ -101,12 +101,12 @@ export class AppContainer {
 									throw new Error(`Unknown app "${clientId}" just connected to the appContainer`)
 								}
 								client.once('close', () => {
-									this.logger.warn(`Connection to Worker "${clientId}" closed`)
+									this.logger.warn(`Connection from Worker "${clientId}" closed`)
 									app.workerAgentApi = null
 
 									this.workerStorage.releaseLockForTag(unprotectString(clientId))
 								})
-								this.logger.info(`Connection to Worker "${client.clientId}" established`)
+								this.logger.info(`Connection from Worker "${client.clientId}" established`)
 								app.workerAgentApi = api
 
 								// Set upp the app for pinging and automatic spin-down:

apps/worker/app/Dockerfile

@@ -0,0 +1,58 @@
+FROM node:18-alpine as builder
+
+# Note: Build this from the root directory:
+# cd package-manager
+# docker build -f apps/worker/app/Dockerfile -t pm-worker .
+# docker build -t pm-worker ../../../..
+
+# Environment
+
+WORKDIR /src
+
+# Common
+
+COPY package.json tsconfig.json tsconfig.build.json yarn.lock lerna.json commonPackage.json .yarnrc.yml ./
+COPY scripts ./scripts
+COPY .yarn ./.yarn
+
+# Shared dependencies
+COPY shared ./shared
+
+
+# App dependencies
+RUN mkdir -p apps/worker
+COPY apps/worker/packages apps/worker/packages
+
+# App
+COPY apps/worker/app apps/worker/app
+
+# Install
+RUN yarn install
+
+# Build
+RUN yarn build
+
+# Purge dev-dependencies:
+RUN yarn workspaces focus -A --production
+
+RUN rm -r scripts
+
+
+# Create deploy-image:
+FROM node:18-alpine
+
+COPY --from=builder /src /src
+
+RUN apk add --no-cache dumb-init ffmpeg
+
+# Run as non-root user
+USER 1000
+WORKDIR /src/apps/worker/app
+# ENV APP_CONTAINER_PORT=8090
+# # ENV HTTP_SERVER_API_KEY_READ=
+# # ENV HTTP_SERVER_API_KEY_WRITE=
+# ENV HTTP_SERVER_BASE_PATH="/data"
+# EXPOSE 8090
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "./docker-entrypoint.sh"]
+

apps/worker/app/docker-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+
+HOSTNAME=$(hostname)
+
+# Inject a unique worker ID based on the hostname, and disable the appContainer connection
+node dist/index.js --workerId=$HOSTNAME --appContainerURL="" $*

shared/packages/api/src/config.ts

@@ -29,6 +29,11 @@ const workforceArguments = defineArguments({
 		default: parseInt(process.env.WORKFORCE_PORT || '', 10) || 8070,
 		describe: 'The port number to start the Workforce websocket server on',
 	},
+	allowNoAppContainers: {
+		type: 'boolean',
+		default: process.env.WORKFORCE_ALLOW_NO_APP_CONTAINERS === '1' || false,
+		describe: 'If true, the workforce will not check if it has no appContainers connected',
+	},
 })
 /** CLI-argument-definitions for the HTTP-Server process */
 const httpServerArguments = defineArguments({
@@ -227,7 +232,7 @@ const appContainerArguments = defineArguments({
 	// These are passed-through to the spun-up workers:
 	resourceId: {
 		type: 'string',
-		default: process.env.WORKER_NETWORK_ID || 'default',
+		default: process.env.WORKER_RESOURCE_ID || 'default',
 		describe: 'Identifier of the local resource/computer this worker runs on',
 	},
 	networkIds: {
@@ -334,6 +339,7 @@ export interface WorkforceConfig {
 	process: ProcessConfig
 	workforce: {
 		port: number | null
+		allowNoAppContainers: boolean
 	}
 }
 
@@ -349,6 +355,7 @@ export async function getWorkforceConfig(): Promise<WorkforceConfig> {
 		process: getProcessConfig(argv),
 		workforce: {
 			port: argv.port,
+			allowNoAppContainers: argv.allowNoAppContainers,
 		},
 	}
 }

shared/packages/expectationManager/src/internalManager/lib/expectationManagerServer.ts

@@ -50,15 +50,15 @@ export class ExpectationManagerServer {
 							})
 							this.manager.workerAgents.upsert(clientId, { api, connected: true })
 							client.once('close', () => {
-								this.logger.warn(`Connection to Worker "${clientId}" closed`)
+								this.logger.warn(`Connection from Worker "${clientId}" closed`)
 
 								const workerAgent = this.manager.workerAgents.get(clientId)
 								if (workerAgent) {
 									workerAgent.connected = false
 									this.manager.workerAgents.remove(clientId)
 								}
 							})
-							this.logger.info(`Connection to Worker "${clientId}" established`)
+							this.logger.info(`Connection from Worker "${clientId}" established`)
 							this.manager.tracker.triggerEvaluationNow()
 							break
 						}

shared/packages/worker/src/worker/accessorHandlers/accessor.ts

@@ -46,7 +46,6 @@ export function getAccessorStaticHandle(accessor: AccessorOnPackage.Any) {
 	} else if (accessor.type === Accessor.AccessType.HTTP_PROXY) {
 		return HTTPProxyAccessorHandle
 	} else if (accessor.type === Accessor.AccessType.FILE_SHARE) {
-		if (process.platform !== 'win32') throw new Error(`FileShareAccessor: not supported on ${process.platform}`)
 		return FileShareAccessorHandle
 	} else if (accessor.type === Accessor.AccessType.QUANTEL) {
 		return QuantelAccessorHandle

shared/packages/worker/src/worker/accessorHandlers/fileShare.ts

@@ -31,6 +31,7 @@ import {
 	MonitorId,
 	betterPathResolve,
 	betterPathIsAbsolute,
+	isRunningInTest,
 } from '@sofie-package-manager/api'
 import { BaseWorker } from '../worker'
 import { GenericWorker } from '../workers/genericWorker/genericWorker'
@@ -111,12 +112,25 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		return this.getFullPath(this.filePath)
 	}
 	static doYouSupportAccess(worker: BaseWorker, accessor: AccessorOnPackage.Any): boolean {
+		if (!isFileShareSupportedOnCurrentPlatform()) return false
+
 		return defaultDoYouSupportAccess(worker, accessor)
 	}
 	get packageName(): string {
 		return this.fullPath
 	}
 	checkHandleBasic(): AccessorHandlerCheckHandleBasicResult {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		if (this.accessor.type !== Accessor.AccessType.FILE_SHARE) {
 			return {
 				success: false,
@@ -168,14 +182,46 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		return { success: true }
 	}
 	checkCompatibilityWithAccessor(): AccessorHandlerCheckHandleCompatibilityResult {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
 		return { success: true } // no special compatibility checks
 	}
 	checkHandleRead(): AccessorHandlerCheckHandleReadResult {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		const defaultResult = defaultCheckHandleRead(this.accessor)
 		if (defaultResult) return defaultResult
 		return { success: true }
 	}
 	checkHandleWrite(): AccessorHandlerCheckHandleWriteResult {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		const defaultResult = defaultCheckHandleWrite(this.accessor)
 		if (defaultResult) return defaultResult
 		return { success: true }
@@ -199,6 +245,18 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		return { success: true }
 	}
 	async tryPackageRead(): Promise<AccessorHandlerTryPackageReadResult> {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+				packageExists: false,
+			}
+		}
+
 		try {
 			// Check if we can open the file for reading:
 			const fd = await fsOpen(this.fullPath, 'r')
@@ -232,6 +290,17 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		return { success: true }
 	}
 	private async _checkPackageReadAccess(): Promise<AccessorHandlerCheckPackageReadAccessResult> {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		await this.prepareFileAccess()
 
 		try {
@@ -364,6 +433,17 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		await this.unlinkIfExists(this.metadataPath)
 	}
 	async runCronJob(packageContainerExp: PackageContainerExpectation): Promise<AccessorHandlerRunCronJobResult> {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		// Always check read/write access first:
 		const checkRead = await this.checkPackageContainerReadAccess()
 		if (!checkRead.success) return checkRead
@@ -396,6 +476,17 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 	async setupPackageContainerMonitors(
 		packageContainerExp: PackageContainerExpectation
 	): Promise<SetupPackageContainerMonitorsResult> {
+		if (!isFileShareSupportedOnCurrentPlatform()) {
+			return {
+				success: false,
+				knownReason: true,
+				reason: {
+					user: `File share is not supported on this worker`,
+					tech: `File share is not supported on ${process.platform}`,
+				},
+			}
+		}
+
 		const resultingMonitors: Record<MonitorId, MonitorInProgress> = {}
 		const monitorIds = Object.keys(
 			packageContainerExp.monitors
@@ -419,6 +510,9 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		operationName: string,
 		source: string | GenericAccessorHandle<any>
 	): Promise<PackageOperation> {
+		if (!isFileShareSupportedOnCurrentPlatform())
+			throw new Error(`FileShareAccessor: not supported on ${process.platform}`)
+
 		await this.fileHandler.clearPackageRemoval(this.filePath)
 		return this.logWorkOperation(operationName, source, this.packageName)
 	}
@@ -443,6 +537,9 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 	 * This method should be called prior to any file access being made.
 	 */
 	async prepareFileAccess(forceRemount = false): Promise<void> {
+		if (!isFileShareSupportedOnCurrentPlatform())
+			throw new Error(`FileShareAccessor: not supported on ${process.platform}`)
+
 		if (!this.originalFolderPath) throw new Error(`FileShareAccessor: accessor.folderPath not set!`)
 		const folderPath = this.originalFolderPath
 
@@ -680,3 +777,8 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 interface MappedDriveLetters {
 	[driveLetter: string]: string
 }
+
+function isFileShareSupportedOnCurrentPlatform(): boolean {
+	// This is only supported on windows currently
+	return isRunningInTest() || process.platform === 'win32'
+}