diff --git a/.gitignore b/.gitignore index 9de7a7047d..73cc8a1bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ secrets.auto.tfvars *.tgz *.env* .vscode +.project +.settings/ **/coverage/* diff --git a/README.md b/README.md index 19cf7fa98e..15e72658c4 100644 --- a/README.md +++ b/README.md @@ -415,6 +415,7 @@ In case the setup does not work as intended follow the trace of events: | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app. Ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`). | 
object({
    key_base64     = string
    id             = string
    webhook_secret = string
  })
| n/a | yes | +| [http\_proxy](#input\http\_proxy) | Http(s) proxy used by scale up/down runners Lambda to interact with AWS APIs (SSM, EC2), and usable in user data template. To use when `vpc_id` has no direct internet connection. | `string` | `null` | no | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron      = string
    timeZone  = string
    idleCount = number
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | diff --git a/main.tf b/main.tf index 3a3525bd3b..9b2ae8b306 100644 --- a/main.tf +++ b/main.tf @@ -182,6 +182,7 @@ module "runners" { lambda_timeout_scale_down = var.runners_scale_down_lambda_timeout lambda_subnet_ids = var.lambda_subnet_ids lambda_security_group_ids = var.lambda_security_group_ids + http_proxy = var.http_proxy logging_retention_in_days = var.logging_retention_in_days logging_kms_key_id = var.logging_kms_key_id enable_cloudwatch_agent = var.enable_cloudwatch_agent diff --git a/modules/runners/README.md b/modules/runners/README.md index 6fb103555a..739ba1ca84 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -135,6 +135,7 @@ yarn run dist | [ghes\_ssl\_verify](#input\_ghes\_ssl\_verify) | GitHub Enterprise SSL verification. Set to 'false' when custom certificate (chains) is used for GitHub Enterprise Server (insecure). | `bool` | `true` | no | | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB | `string` | `null` | no | | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | +| [http\_proxy](#input\http\_proxy) | Http(s) proxy used by scale up/down runners Lambda to interact with AWS APIs (SSM, EC2), and usable in user data template. To use when `vpc_id` has no direct internet connection. | `string` | `null` | no | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron      = string
    timeZone  = string
    idleCount = number
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot intances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | diff --git a/modules/runners/lambdas/runners/package.json b/modules/runners/lambdas/runners/package.json index c0616275b6..5cfa28a78f 100644 --- a/modules/runners/lambdas/runners/package.json +++ b/modules/runners/lambdas/runners/package.json @@ -10,7 +10,7 @@ "lint": "yarn eslint src", "watch": "ts-node-dev --respawn --exit-child src/local.ts", "build": "ncc build src/lambda.ts -o dist", - "dist": "yarn build && cd dist && zip ../runners.zip index.js", + "dist": "yarn build && cd dist && zip ../runners.zip *.js", "format": "prettier --write \"**/*.ts\"", "format-check": "prettier --check \"**/*.ts\"", "all": "yarn build && yarn format && yarn lint && yarn test" @@ -44,6 +44,7 @@ "@types/express": "^4.17.11", "@types/node": "^18.7.6", "aws-sdk": "^2.1196.0", + "proxy-agent": "^5.0.0", "cron-parser": "^4.6.0", "tslog": "^3.3.3", "typescript": "^4.7.4" diff --git a/modules/runners/lambdas/runners/src/aws/ssm.test.ts b/modules/runners/lambdas/runners/src/aws/ssm.test.ts index 8cd82b0d46..c72ff5bfbf 100644 --- a/modules/runners/lambdas/runners/src/aws/ssm.test.ts +++ b/modules/runners/lambdas/runners/src/aws/ssm.test.ts @@ -1,9 +1,11 @@ import { GetParameterCommandOutput, SSM } from '@aws-sdk/client-ssm'; import nock from 'nock'; +import proxy from 'proxy-agent'; import { getParameterValue } from './ssm'; jest.mock('@aws-sdk/client-ssm'); +jest.mock('proxy-agent'); const cleanEnv = process.env; @@ -12,6 +14,8 @@ beforeEach(() => { jest.clearAllMocks(); process.env = { ...cleanEnv }; nock.disableNetConnect(); + // Remove any proxy if existing (from user/execution 'clean' env or from previous test) + process.env.HTTPS_PROXY = undefined; }); describe('Test getParameterValue', () => { @@ -56,4 +60,57 @@ describe('Test getParameterValue', () => { // Assert expect(result).toBe(undefined); }); + + test('Check that proxy is not used if not defined', async () => { + // Mock it + const mockedProxy = proxy as unknown as jest.Mock; + + // Act + await getParameterValue('testParam'); + + // Assert not called + expect(mockedProxy).not.toHaveBeenCalled(); + }); + + test('Check that proxy is used', async () => { + // Define fake proxy + process.env.HTTPS_PROXY = 'http://proxy.company.com'; + + // Mock it + const mockedProxy = proxy as unknown as jest.Mock; + + // Act + await getParameterValue('testParam'); + + // Assert correctly called + expect(mockedProxy).toBeCalledWith(process.env.HTTPS_PROXY); + }); + + test('Check that invalid proxy is not used', async () => { + // Define fake invalid proxy + process.env.HTTPS_PROXY = 'invalidPrefix://proxy.company.com'; + + // Mock it + const mockedProxy = proxy as unknown as jest.Mock; + + // Act + await getParameterValue('testParam'); + + // Assert not called + expect(mockedProxy).not.toHaveBeenCalled(); + }); + + test('Check proxy unknown host', async () => { + // Define proxy which is unknown + process.env.HTTPS_PROXY = 'http://unknown.company.com'; + + // Mock it + const mockedProxy = proxy as unknown as jest.Mock; + mockedProxy.mockImplementation(() => { + throw new Error('Unknown host'); + }); + + // Assert exception + await expect(getParameterValue('testParam')).rejects.toHaveProperty('message', 'Unknown host'); + }); }); diff --git a/modules/runners/lambdas/runners/src/aws/ssm.ts b/modules/runners/lambdas/runners/src/aws/ssm.ts index 5212914321..19c321a011 100644 --- a/modules/runners/lambdas/runners/src/aws/ssm.ts +++ b/modules/runners/lambdas/runners/src/aws/ssm.ts @@ -1,6 +1,22 @@ import { SSM } from '@aws-sdk/client-ssm'; +import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; +import proxy from 'proxy-agent'; + +import { logger } from '../logger'; +import { hideUrlPassword } from '../utils/url'; export async function getParameterValue(parameter_name: string): Promise { - const client = new SSM({ region: process.env.AWS_REGION }); + // Proxy with aws-sdk v3 + // Configured by client (global configuration like v2 doesn't work) + // https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/node-configuring-proxies.html + let rh; + const httpsProxy = process.env.HTTPS_PROXY; + if (httpsProxy?.startsWith('http')) { + logger.debug('Http proxy used for AWS SSM SDK (v3): ' + hideUrlPassword(httpsProxy)); + rh = new NodeHttpHandler({ + httpsAgent: new proxy(httpsProxy), + }); + } + const client = new SSM({ region: process.env.AWS_REGION, requestHandler: rh }); return (await client.getParameter({ Name: parameter_name, WithDecryption: true })).Parameter?.Value as string; } diff --git a/modules/runners/lambdas/runners/src/lambda.test.ts b/modules/runners/lambdas/runners/src/lambda.test.ts index 8fa3b9a6c5..af9344dc58 100644 --- a/modules/runners/lambdas/runners/src/lambda.test.ts +++ b/modules/runners/lambdas/runners/src/lambda.test.ts @@ -1,7 +1,8 @@ import { Context, SQSEvent, SQSRecord } from 'aws-lambda'; +import { config } from 'aws-sdk'; import { mocked } from 'jest-mock'; -import { adjustPool, scaleDownHandler, scaleUpHandler } from './lambda'; +import { adjustPool, configureProxyAwsSdkV2Only, scaleDownHandler, scaleUpHandler } from './lambda'; import { logger } from './logger'; import { adjust } from './pool/pool'; import ScaleError from './scale-runners/ScaleError'; @@ -157,3 +158,31 @@ describe('Adjust pool.', () => { expect(logSpy).lastCalledWith(error); }); }); + +describe('Test proxy configuration for AWS SDK v2', () => { + beforeEach(() => { + // Remove any proxy if existing (from user/execution env or from previous test) + process.env.HTTPS_PROXY = undefined; + }); + + afterEach(() => { + // Remove any set proxy + process.env.HTTPS_PROXY = undefined; + + // Reset potential agent in AWS config + delete config.httpOptions?.agent; + }); + + it('Proxy configured', async () => { + process.env.HTTPS_PROXY = 'http://proxy.company.com'; + expect(configureProxyAwsSdkV2Only(config)).toBe(true); + const agent = config.httpOptions?.agent; + expect(agent).toBeDefined(); + expect(JSON.stringify(agent)).toContain(process.env.HTTPS_PROXY); + }); + + it('Proxy not configured', async () => { + expect(configureProxyAwsSdkV2Only(config)).toBe(false); + expect(config.httpOptions?.agent).toBeUndefined(); + }); +}); diff --git a/modules/runners/lambdas/runners/src/lambda.ts b/modules/runners/lambdas/runners/src/lambda.ts index 5e413a36a2..32758e11f4 100644 --- a/modules/runners/lambdas/runners/src/lambda.ts +++ b/modules/runners/lambdas/runners/src/lambda.ts @@ -1,4 +1,6 @@ import { Context, SQSEvent } from 'aws-lambda'; +import { config } from 'aws-sdk'; +import proxy from 'proxy-agent'; import 'source-map-support/register'; import { LogFields, logger } from './logger'; @@ -6,6 +8,7 @@ import { PoolEvent, adjust } from './pool/pool'; import ScaleError from './scale-runners/ScaleError'; import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; +import { hideUrlPassword } from './utils/url'; export async function scaleUpHandler(event: SQSEvent, context: Context): Promise { logger.setSettings({ requestId: context.awsRequestId }); @@ -19,6 +22,7 @@ export async function scaleUpHandler(event: SQSEvent, context: Context): Promise } try { + configureProxyAwsSdkV2Only(config); await scaleUp(event.Records[0].eventSource, JSON.parse(event.Records[0].body)); } catch (e) { if (e instanceof ScaleError) { @@ -33,6 +37,7 @@ export async function scaleDownHandler(context: Context): Promise { logger.setSettings({ requestId: context.awsRequestId }); try { + configureProxyAwsSdkV2Only(config); await scaleDown(); } catch (e) { logger.error(e); @@ -48,3 +53,22 @@ export async function adjustPool(event: PoolEvent, context: Context): Promise SSM & EC2 in 'runners.ts' file ('ssm.ts' is already in aws-sdk v3) + * https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-configuring-proxies.html + * + * Configure proxy for AWS config in parameter, return true if updated. + */ +export function configureProxyAwsSdkV2Only(awsConfig: AWS.Config) { + const httpsProxy = process.env.HTTPS_PROXY; + if (httpsProxy?.startsWith('http')) { + logger.debug('Http proxy for AWS SDK (v2): ' + hideUrlPassword(httpsProxy)); + awsConfig.update({ + httpOptions: { agent: proxy(httpsProxy) }, + }); + return true; + } + return false; +} diff --git a/modules/runners/lambdas/runners/src/local-down.ts b/modules/runners/lambdas/runners/src/local-down.ts index 2d0ac05fca..d38e114153 100644 --- a/modules/runners/lambdas/runners/src/local-down.ts +++ b/modules/runners/lambdas/runners/src/local-down.ts @@ -1,10 +1,15 @@ +import { config } from 'aws-sdk'; + +import { configureProxyAwsSdkV2Only } from './lambda'; +import { logger } from './logger'; import { scaleDown } from './scale-runners/scale-down'; export function run(): void { + configureProxyAwsSdkV2Only(config); scaleDown() .then() .catch((e) => { - console.log(e); + logger.error(e); }); } diff --git a/modules/runners/lambdas/runners/src/local-pool.ts b/modules/runners/lambdas/runners/src/local-pool.ts index ab8c74a1a0..e5faf27b89 100644 --- a/modules/runners/lambdas/runners/src/local-pool.ts +++ b/modules/runners/lambdas/runners/src/local-pool.ts @@ -1,10 +1,15 @@ +import { config } from 'aws-sdk'; + +import { configureProxyAwsSdkV2Only } from './lambda'; +import { logger } from './logger'; import { adjust } from './pool/pool'; export function run(): void { + configureProxyAwsSdkV2Only(config); adjust({ poolSize: 1 }) .then() .catch((e) => { - console.log(e); + logger.error(e); }); } diff --git a/modules/runners/lambdas/runners/src/local.ts b/modules/runners/lambdas/runners/src/local.ts index 5c79110d76..d52ba60dbb 100644 --- a/modules/runners/lambdas/runners/src/local.ts +++ b/modules/runners/lambdas/runners/src/local.ts @@ -1,3 +1,6 @@ +import { config } from 'aws-sdk'; + +import { configureProxyAwsSdkV2Only } from './lambda'; import { logger } from './logger'; import { ActionRequestMessage, scaleUp } from './scale-runners/scale-up'; @@ -34,6 +37,7 @@ const sqsEvent = { }; export function run(): void { + configureProxyAwsSdkV2Only(config); scaleUp(sqsEvent.Records[0].eventSource, sqsEvent.Records[0].body as ActionRequestMessage) .then() .catch((e) => { diff --git a/modules/runners/lambdas/runners/src/utils/url.test.ts b/modules/runners/lambdas/runners/src/utils/url.test.ts new file mode 100644 index 0000000000..8ba8b7b4f3 --- /dev/null +++ b/modules/runners/lambdas/runners/src/utils/url.test.ts @@ -0,0 +1,12 @@ +import { hideUrlPassword } from './url'; + +describe('Test URL proxy validation', () => { + test('URL credentials port 80', async () => { + const url = hideUrlPassword('http://foo:bar@proxy.company.com:80'); + expect(url).toBe('http://foo:*****@proxy.company.com/'); + }); + test('URL port 8080', async () => { + const url = hideUrlPassword('http://proxy.company.com:8080'); + expect(url).toBe('http://proxy.company.com:8080/'); + }); +}); diff --git a/modules/runners/lambdas/runners/src/utils/url.ts b/modules/runners/lambdas/runners/src/utils/url.ts new file mode 100644 index 0000000000..ed9dfa1b12 --- /dev/null +++ b/modules/runners/lambdas/runners/src/utils/url.ts @@ -0,0 +1,7 @@ +export function hideUrlPassword(url: string): string { + const urlProxy = new URL(url); + if (urlProxy.password) { + urlProxy.password = '*****'; + } + return urlProxy.toString(); +} diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 7fea4e8940..340e9623b2 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -132,6 +132,7 @@ resource "aws_launch_template" "runner" { start_runner = templatefile(local.userdata_start_runner[var.runner_os], {}) ghes_url = var.ghes_url ghes_ssl_verify = var.ghes_ssl_verify + http_proxy = var.http_proxy ## retain these for backwards compatibility environment = var.prefix enable_cloudwatch_agent = var.enable_cloudwatch_agent diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index 08181485c8..fddac8216f 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -32,6 +32,7 @@ resource "aws_lambda_function" "scale_down" { PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.github_app_parameters.key_base64.name RUNNER_BOOT_TIME_IN_MINUTES = var.runner_boot_time_in_minutes SCALE_DOWN_CONFIG = jsonencode(var.idle_config) + HTTPS_PROXY = var.http_proxy } } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 858224bf5a..f274513dce 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -22,6 +22,7 @@ resource "aws_lambda_function" "scale_up" { ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners ENVIRONMENT = var.prefix GHES_URL = var.ghes_url + HTTPS_PROXY = var.http_proxy INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 60c1d25e62..0b71d61ea1 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -407,6 +407,12 @@ variable "lambda_security_group_ids" { default = [] } +variable "http_proxy" { + description = "Http(s) proxy used by scale up/down runners Lambda to interact with AWS APIs (SSM, EC2), and usable in user data template. To use when `vpc_id` has no direct internet connection." + type = string + default = null +} + variable "key_name" { description = "Key pair name" type = string diff --git a/variables.tf b/variables.tf index 7c45e06179..e7560797f8 100644 --- a/variables.tf +++ b/variables.tf @@ -371,6 +371,12 @@ variable "lambda_security_group_ids" { default = [] } +variable "http_proxy" { + description = "Http(s) proxy used by scale up/down runners Lambda to interact with AWS APIs (SSM, EC2), and usable in user data template. To use when `vpc_id` has no direct internet connection." + type = string + default = null +} + variable "key_name" { description = "Key pair name" type = string