From 5586709d03b077c795b65511785154b4ce12022b Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 17:54:19 -0500 Subject: [PATCH 01/23] Initial pass at DDNS support for client addresses This is a first pass attempt at adding support for using ddns (really any resolvable domain name) as the address in access list clients. This helps make it possible to restrict access to hosts using a dynamic public IP (e.g. allow access to a proxied host from your local network only via ddns address). Current approach is hacky since it was developed by manually replacing files in an existing npm docker container. Future commits will integrate this better and avoid needing to patch/intercept existing APIs. See associated PR for more details. --- backend/index.js | 2 + backend/lib/ddns_resolver/ddns_resolver.js | 308 +++++++++++++++++++++ backend/schema/endpoints/access-lists.json | 4 + 3 files changed, 314 insertions(+) create mode 100644 backend/lib/ddns_resolver/ddns_resolver.js diff --git a/backend/index.js b/backend/index.js index 3d6d60071..82f9be5f4 100644 --- a/backend/index.js +++ b/backend/index.js @@ -9,6 +9,7 @@ async function appStart () { const apiValidator = require('./lib/validator/api'); const internalCertificate = require('./internal/certificate'); const internalIpRanges = require('./internal/ip_ranges'); + const ddnsResolver = require('./lib/ddns_resolver/ddns_resolver'); return migrate.latest() .then(setup) @@ -20,6 +21,7 @@ async function appStart () { internalCertificate.initTimer(); internalIpRanges.initTimer(); + ddnsResolver.initTimer(); const server = app.listen(3000, () => { logger.info('Backend PID ' + process.pid + ' listening on port 3000 ...'); diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js new file mode 100644 index 000000000..fc759133d --- /dev/null +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -0,0 +1,308 @@ +const error = require('../error') +const logger = require('../../logger').global; +const internalAccessList = require('../../internal/access-list'); +const internalNginx = require('../../internal/nginx'); +const spawn = require('child_process').spawn; + +const cmdHelper = { + /** + * Run the given command. Safer than using exec since args are passed as a list instead of in shell mode as a single string. + * @param {string} cmd The command to run + * @param {string} args The args to pass to the command + * @returns Promise that resolves to stdout or an object with error code and stderr if there's an error + */ + run: (cmd, args) => { + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + const proc = spawn(cmd, args); + proc.stdout.on('data', (data) => { + stdout += data; + }); + proc.stderr.on('data', (data) => { + stderr += data; + }); + + proc.on('close', (exitCode) => { + if (!exitCode) { + resolve(stdout.trim()); + } else { + reject({ + exitCode: exitCode, + stderr: stderr + }); + } + }); + }); + } +}; + +const ddnsResolver = { + /** + * Starts a timer to periodically check for ddns updates + */ + initTimer: () => { + ddnsResolver._initialize(); + ddnsResolver._interval = setInterval(ddnsResolver._checkForDDNSUpdates, ddnsResolver._updateIntervalMs); + logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsResolver._updateIntervalMs / 1000)}s)`); + // Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up + setTimeout(ddnsResolver._checkForDDNSUpdates, 10 * 1000); + }, + + /** + * Checks whether the address requires resolution (i.e. starts with ddns:) + * @param {String} address + * @returns {boolean} + */ + requiresResolution: (address) => { + if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { + return true; + } + return false; + }, + + /** + * Resolves the given address to its IP + * @param {String} address + * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value + */ + resolveAddress: (address, forceUpdate=false) => { + if (!forceUpdate && ddnsResolver._cache.has(address)) { + // Check if it is still valid + const value = ddnsResolver._cache.get(address); + const ip = value[0]; + const lastUpdated = value[1]; + const nowSeconds = Date.now(); + const delta = nowSeconds - lastUpdated; + if (delta < ddnsResolver._updateIntervalMs) { + return Promise.resolve(ip); + } + } + ddnsResolver._cache.delete(address); + // Reach here only if cache value doesn't exist or needs to be updated + let host = address.toLowerCase(); + if (host.startsWith('ddns:')) { + host = host.substring(5); + } + return ddnsResolver._queryHost(host) + .then((resolvedIP) => { + ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); + return resolvedIP; + }) + .catch((_error) => { + // return input address in case of failure + return address; + }); + }, + + + /** Private **/ + // Properties + _initialized: false, + _updateIntervalMs: 1000 * 60 * 60, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) + /** + * cache mapping host to (ip address, last updated time) + */ + _cache: new Map(), + _interval: null, // reference to created interval id + _processingDDNSUpdate: false, + + _originalGenerateConfig: null, // Used for patching config generation to resolve hosts + + // Methods + + _initialize: () => { + if (ddnsResolver._initialized) { + return; + } + // Init the resolver + // Read and set custom update interval from env if needed + if (typeof process.env.DDNS_UPDATE_INTERVAL !== 'undefined') { + const interval = Number(process.env.DDNS_UPDATE_INTERVAL.toLowerCase()); + if (!isNaN(interval)) { + // Interval value from env is in seconds. Set min to 60s. + ddnsResolver._updateIntervalMs = Math.max(interval * 1000, 60 * 1000); + } else { + logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`); + } + } + + // Patch nginx config generation if needed (check env var) + if (typeof process.env.DDNS_UPDATE_PATCH !== 'undefined') { + const enabled = Number(process.env.DDNS_UPDATE_PATCH.toLowerCase()); + if (!isNaN(enabled) && enabled) { + logger.info('Patching nginx config generation'); + ddnsResolver._originalGenerateConfig = internalNginx.generateConfig; + internalNginx.generateConfig = ddnsResolver._patchedGenerateConfig; + } + } + ddnsResolver._initialized = true; + }, + + /** + * + * @param {String} host + * @returns {Promise} + */ + _queryHost: (host) => { + logger.info('Looking up IP for ', host); + return cmdHelper.run('getent', ['hosts', host]) + .then((result) => { + if (result.length < 8) { + logger.error('IP lookup returned invalid output: ', result); + throw error.ValidationError('Invalid output from getent hosts'); + } + const out = result.split(/\s+/); + logger.info(`Resolved ${host} to ${out[0]}`); + return out[0]; + }, + (error) => { + logger.error('Error looking up IP for ' + host + ': ', error); + throw error; + }); + }, + + _patchedGenerateConfig: (host_type, host) => { + const promises = []; + if (host_type === 'proxy_host') { + if (typeof host.access_list !== 'undefined' && typeof host.access_list.clients !== 'undefined') { + for (const client of host.access_list.clients) { + if (ddnsResolver.requiresResolution(client.address)) { + const p = ddnsResolver.resolveAddress(client.address) + .then((resolvedIP) => { + client.address = `${resolvedIP}; # ${client.address}`; + return Promise.resolve(); + }); + promises.push(p); + } + } + } + } + if (promises.length) { + return Promise.all(promises) + .then(() => { + return ddnsResolver._originalGenerateConfig(host_type, host); + }); + } + return ddnsResolver._originalGenerateConfig(host_type, host); + }, + + /** + * Triggered by a timer, will check for and update ddns hosts in access list clients + */ + _checkForDDNSUpdates: () => { + logger.info('Checking for DDNS updates...'); + if (!ddnsResolver._processingDDNSUpdate) { + ddnsResolver._processingDDNSUpdate = true; + + const updatedAddresses = new Map(); + + // Get all ddns hostnames in use + return ddnsResolver._getAccessLists() + .then((rows) => { + // Build map of used addresses that require resolution + const usedAddresses = new Map(); + for (const row of rows) { + if (!row.proxy_host_count) { + // Ignore rows (access lists) that are not associated to any hosts + continue; + } + for (const client of row.clients) { + if (!ddnsResolver.requiresResolution(client.address)) { + continue; + } + if (!usedAddresses.has(client.address)) { + usedAddresses.set(client.address, [row]); + } else { + usedAddresses.get(client.address).push(row); + } + } + } + logger.info(`Found ${usedAddresses.size} address(es) in use.`); + // Remove unused addresses + const addressesToRemove = []; + for (const address of ddnsResolver._cache.keys()) { + if (!usedAddresses.has(address)) { + addressesToRemove.push(address); + } + } + addressesToRemove.forEach((address) => { ddnsResolver._cache.delete(address); }); + + const promises = []; + + for (const [address, rows] of usedAddresses) { + let oldIP = ''; + if (ddnsResolver._cache.has(address)) { + oldIP = ddnsResolver._cache.get(address)[0]; + } + const p = ddnsResolver.resolveAddress(address, true) + .then((resolvedIP) => { + if (resolvedIP !== address && resolvedIP !== oldIP) { + // Mark this as an updated address + updatedAddresses.set(address, rows); + } + }); + promises.push(p); + } + + if (promises.length) { + return Promise.all(promises); + } + return Promise.resolve(); + }) + .then(() => { + logger.info(`${updatedAddresses.size} DDNS IP(s) updated.`); + const updatedRows = new Map(); + const proxy_hosts = []; + for (const rows of updatedAddresses.values()) { + for (const row of rows) { + if (!updatedRows.has(row.id)) { + updatedRows.set(row.id, 1); + proxy_hosts.push(...row.proxy_hosts); + } + } + } + if (proxy_hosts.length) { + logger.info(`Updating ${proxy_hosts.length} proxy host(s) affected by DDNS changes`); + return internalNginx.bulkGenerateConfigs('proxy_host', proxy_hosts) + .then(internalNginx.reload); + } + return Promise.resolve(); + }) + .then(() => { + logger.info('Finished checking for DDNS updates'); + ddnsResolver._processingDDNSUpdate = false; + }); + } else { + logger.info('Skipping since previous DDNS update check is in progress'); + } + }, + + _getAccessLists: () => { + const fakeAccess = { + can: (capabilityStr) => { + return Promise.resolve({ + permission_visibility: 'all' + }) + } + }; + + return internalAccessList.getAll(fakeAccess) + .then((rows) => { + const promises = []; + for (const row of rows) { + const p = internalAccessList.get(fakeAccess, { + id: row.id, + expand: ['owner', 'items', 'clients', 'proxy_hosts.[certificate,access_list.[clients,items]]'] + }, true /* <- skip masking */); + promises.push(p); + } + if (promises.length) { + return Promise.all(promises); + } + return Promise.resolve([]); + }); + } +}; + +module.exports = ddnsResolver; diff --git a/backend/schema/endpoints/access-lists.json b/backend/schema/endpoints/access-lists.json index 404e32376..94585f30b 100644 --- a/backend/schema/endpoints/access-lists.json +++ b/backend/schema/endpoints/access-lists.json @@ -36,6 +36,10 @@ { "type": "string", "pattern": "^all$" + }, + { + "type": "string", + "pattern": "^ddns:[\\w\\.]+$" } ] }, From ec9eb0dd6078ad3e892bac5437f30c723f5b0982 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 19:13:47 -0500 Subject: [PATCH 02/23] Refactor and integrate ddns resolution with nginx module Refactored ddns resolver so that no patching is done. nginx.js will automatically resolve ddns addresses if needed. Added dedicated logger scope for ddns resovler. --- backend/index.js | 2 +- backend/internal/nginx.js | 40 ++++++++- .../lib/{ddns_resolver => }/ddns_resolver.js | 85 ++----------------- backend/lib/utils.js | 32 +++++++ backend/logger.js | 3 +- 5 files changed, 80 insertions(+), 82 deletions(-) rename backend/lib/{ddns_resolver => }/ddns_resolver.js (74%) diff --git a/backend/index.js b/backend/index.js index 82f9be5f4..129cf060d 100644 --- a/backend/index.js +++ b/backend/index.js @@ -9,7 +9,7 @@ async function appStart () { const apiValidator = require('./lib/validator/api'); const internalCertificate = require('./internal/certificate'); const internalIpRanges = require('./internal/ip_ranges'); - const ddnsResolver = require('./lib/ddns_resolver/ddns_resolver'); + const ddnsResolver = require('./lib/ddns_resolver'); return migrate.latest() .then(setup) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 77933e733..2f5dc376e 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -4,6 +4,7 @@ const logger = require('../logger').nginx; const config = require('../lib/config'); const utils = require('../lib/utils'); const error = require('../lib/error'); +const ddnsResolver = require('../lib/ddns_resolver'); const internalNginx = { @@ -131,6 +132,31 @@ const internalNginx = { return '/data/nginx/' + internalNginx.getFileFriendlyHostType(host_type) + '/' + host_id + '.conf'; }, + /** + * Resolves any ddns addresses that need to be resolved for clients in the host's access list. + * @param {Object} host + * @returns {Promise} + */ + resolveDDNSAddresses: (host) => { + const promises = []; + if (typeof host.access_list !== 'undefined' && typeof host.access_list.clients !== 'undefined') { + for (const client of host.access_list.clients) { + if (ddnsResolver.requiresResolution(client.address)) { + const p = ddnsResolver.resolveAddress(client.address) + .then((resolvedIP) => { + client.address = `${resolvedIP}; # ${client.address}`; + return Promise.resolve(); + }); + promises.push(p); + } + } + } + if (promises.length) { + return Promise.all(promises); + } + return Promise.resolve(); + }, + /** * Generates custom locations * @param {Object} host @@ -201,6 +227,12 @@ const internalNginx = { return; } + // Resolve ddns addresses if needed + let resolverPromise = Promise.resolve(); + if (host_type === 'proxy_host') { + resolverPromise = internalNginx.resolveDDNSAddresses(host); + } + let locationsPromise; let origLocations; @@ -215,8 +247,10 @@ const internalNginx = { if (host.locations) { //logger.info ('host.locations = ' + JSON.stringify(host.locations, null, 2)); origLocations = [].concat(host.locations); - locationsPromise = internalNginx.renderLocations(host).then((renderedLocations) => { - host.locations = renderedLocations; + locationsPromise = resolverPromise.then(() => { + return internalNginx.renderLocations(host).then((renderedLocations) => { + host.locations = renderedLocations; + }); }); // Allow someone who is using / custom location path to use it, and skip the default / location @@ -227,7 +261,7 @@ const internalNginx = { }); } else { - locationsPromise = Promise.resolve(); + locationsPromise = resolverPromise; } // Set the IPv6 setting for the host diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver.js similarity index 74% rename from backend/lib/ddns_resolver/ddns_resolver.js rename to backend/lib/ddns_resolver.js index fc759133d..b339ca476 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver.js @@ -1,41 +1,7 @@ -const error = require('../error') -const logger = require('../../logger').global; -const internalAccessList = require('../../internal/access-list'); -const internalNginx = require('../../internal/nginx'); -const spawn = require('child_process').spawn; - -const cmdHelper = { - /** - * Run the given command. Safer than using exec since args are passed as a list instead of in shell mode as a single string. - * @param {string} cmd The command to run - * @param {string} args The args to pass to the command - * @returns Promise that resolves to stdout or an object with error code and stderr if there's an error - */ - run: (cmd, args) => { - return new Promise((resolve, reject) => { - let stdout = ''; - let stderr = ''; - const proc = spawn(cmd, args); - proc.stdout.on('data', (data) => { - stdout += data; - }); - proc.stderr.on('data', (data) => { - stderr += data; - }); - - proc.on('close', (exitCode) => { - if (!exitCode) { - resolve(stdout.trim()); - } else { - reject({ - exitCode: exitCode, - stderr: stderr - }); - } - }); - }); - } -}; +const error = require('./error') +const logger = require('../logger').ddns; +const internalAccessList = require('../internal/access-list'); +const utils = require('./utils'); const ddnsResolver = { /** @@ -99,15 +65,13 @@ const ddnsResolver = { /** Private **/ // Properties _initialized: false, - _updateIntervalMs: 1000 * 60 * 60, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) + _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) /** * cache mapping host to (ip address, last updated time) */ _cache: new Map(), _interval: null, // reference to created interval id _processingDDNSUpdate: false, - - _originalGenerateConfig: null, // Used for patching config generation to resolve hosts // Methods @@ -126,16 +90,6 @@ const ddnsResolver = { logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`); } } - - // Patch nginx config generation if needed (check env var) - if (typeof process.env.DDNS_UPDATE_PATCH !== 'undefined') { - const enabled = Number(process.env.DDNS_UPDATE_PATCH.toLowerCase()); - if (!isNaN(enabled) && enabled) { - logger.info('Patching nginx config generation'); - ddnsResolver._originalGenerateConfig = internalNginx.generateConfig; - internalNginx.generateConfig = ddnsResolver._patchedGenerateConfig; - } - } ddnsResolver._initialized = true; }, @@ -146,7 +100,7 @@ const ddnsResolver = { */ _queryHost: (host) => { logger.info('Looking up IP for ', host); - return cmdHelper.run('getent', ['hosts', host]) + return utils.execSafe('getent', ['hosts', host]) .then((result) => { if (result.length < 8) { logger.error('IP lookup returned invalid output: ', result); @@ -162,35 +116,12 @@ const ddnsResolver = { }); }, - _patchedGenerateConfig: (host_type, host) => { - const promises = []; - if (host_type === 'proxy_host') { - if (typeof host.access_list !== 'undefined' && typeof host.access_list.clients !== 'undefined') { - for (const client of host.access_list.clients) { - if (ddnsResolver.requiresResolution(client.address)) { - const p = ddnsResolver.resolveAddress(client.address) - .then((resolvedIP) => { - client.address = `${resolvedIP}; # ${client.address}`; - return Promise.resolve(); - }); - promises.push(p); - } - } - } - } - if (promises.length) { - return Promise.all(promises) - .then(() => { - return ddnsResolver._originalGenerateConfig(host_type, host); - }); - } - return ddnsResolver._originalGenerateConfig(host_type, host); - }, - /** * Triggered by a timer, will check for and update ddns hosts in access list clients */ _checkForDDNSUpdates: () => { + const internalNginx = require('../internal/nginx'); // Prevent circular import + logger.info('Checking for DDNS updates...'); if (!ddnsResolver._processingDDNSUpdate) { ddnsResolver._processingDDNSUpdate = true; diff --git a/backend/lib/utils.js b/backend/lib/utils.js index bcdb3341c..188a8cbe1 100644 --- a/backend/lib/utils.js +++ b/backend/lib/utils.js @@ -4,6 +4,7 @@ const execFile = require('child_process').execFile; const { Liquid } = require('liquidjs'); const logger = require('../logger').global; const error = require('./error'); +const spawn = require('child_process').spawn; module.exports = { @@ -26,6 +27,37 @@ module.exports = { return stdout; }, + /** + * Run the given command. Safer than using exec since args are passed as a list instead of in shell mode as a single string. + * @param {string} cmd The command to run + * @param {string} args The args to pass to the command + * @returns Promise that resolves to stdout or an object with error code and stderr if there's an error + */ + execSafe: (cmd, args) => { + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + const proc = spawn(cmd, args); + proc.stdout.on('data', (data) => { + stdout += data; + }); + proc.stderr.on('data', (data) => { + stderr += data; + }); + + proc.on('close', (exitCode) => { + if (!exitCode) { + resolve(stdout.trim()); + } else { + reject({ + exitCode: exitCode, + stderr: stderr + }); + } + }); + }); + }, + /** * @param {String} cmd * @param {Array} args diff --git a/backend/logger.js b/backend/logger.js index 0ebb07c58..78553f515 100644 --- a/backend/logger.js +++ b/backend/logger.js @@ -10,5 +10,6 @@ module.exports = { certbot: new Signale({scope: 'Certbot '}), import: new Signale({scope: 'Importer '}), setup: new Signale({scope: 'Setup '}), - ip_ranges: new Signale({scope: 'IP Ranges'}) + ip_ranges: new Signale({scope: 'IP Ranges'}), + ddns: new Signale({scope: 'DDNS '}) }; From 972d158161654481bd36532626b1e395fa19b559 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 20:09:10 -0500 Subject: [PATCH 03/23] fix linter warnings --- backend/internal/nginx.js | 14 +- backend/lib/ddns_resolver.js | 394 +++++++++++++++++------------------ 2 files changed, 204 insertions(+), 204 deletions(-) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 2f5dc376e..8256ae32b 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -1,9 +1,9 @@ -const _ = require('lodash'); -const fs = require('fs'); -const logger = require('../logger').nginx; -const config = require('../lib/config'); -const utils = require('../lib/utils'); -const error = require('../lib/error'); +const _ = require('lodash'); +const fs = require('fs'); +const logger = require('../logger').nginx; +const config = require('../lib/config'); +const utils = require('../lib/utils'); +const error = require('../lib/error'); const ddnsResolver = require('../lib/ddns_resolver'); const internalNginx = { @@ -151,7 +151,7 @@ const internalNginx = { } } } - if (promises.length) { + if (promises.length) { return Promise.all(promises); } return Promise.resolve(); diff --git a/backend/lib/ddns_resolver.js b/backend/lib/ddns_resolver.js index b339ca476..31b3f5874 100644 --- a/backend/lib/ddns_resolver.js +++ b/backend/lib/ddns_resolver.js @@ -1,239 +1,239 @@ -const error = require('./error') -const logger = require('../logger').ddns; -const internalAccessList = require('../internal/access-list'); -const utils = require('./utils'); +const error = require('./error'); +const logger = require('../logger').ddns; +const internalAccessList = require('../internal/access-list'); +const utils = require('./utils'); const ddnsResolver = { - /** + /** * Starts a timer to periodically check for ddns updates */ - initTimer: () => { - ddnsResolver._initialize(); - ddnsResolver._interval = setInterval(ddnsResolver._checkForDDNSUpdates, ddnsResolver._updateIntervalMs); - logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsResolver._updateIntervalMs / 1000)}s)`); - // Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up - setTimeout(ddnsResolver._checkForDDNSUpdates, 10 * 1000); - }, + initTimer: () => { + ddnsResolver._initialize(); + ddnsResolver._interval = setInterval(ddnsResolver._checkForDDNSUpdates, ddnsResolver._updateIntervalMs); + logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsResolver._updateIntervalMs / 1000)}s)`); + // Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up + setTimeout(ddnsResolver._checkForDDNSUpdates, 10 * 1000); + }, - /** + /** * Checks whether the address requires resolution (i.e. starts with ddns:) * @param {String} address * @returns {boolean} */ - requiresResolution: (address) => { - if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { + requiresResolution: (address) => { + if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { return true; } return false; - }, + }, - /** + /** * Resolves the given address to its IP * @param {String} address * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value */ - resolveAddress: (address, forceUpdate=false) => { - if (!forceUpdate && ddnsResolver._cache.has(address)) { - // Check if it is still valid - const value = ddnsResolver._cache.get(address); - const ip = value[0]; - const lastUpdated = value[1]; - const nowSeconds = Date.now(); - const delta = nowSeconds - lastUpdated; - if (delta < ddnsResolver._updateIntervalMs) { - return Promise.resolve(ip); - } - } - ddnsResolver._cache.delete(address); - // Reach here only if cache value doesn't exist or needs to be updated - let host = address.toLowerCase(); - if (host.startsWith('ddns:')) { - host = host.substring(5); - } - return ddnsResolver._queryHost(host) - .then((resolvedIP) => { - ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); - return resolvedIP; - }) - .catch((_error) => { - // return input address in case of failure - return address; - }); - }, + resolveAddress: (address, forceUpdate=false) => { + if (!forceUpdate && ddnsResolver._cache.has(address)) { + // Check if it is still valid + const value = ddnsResolver._cache.get(address); + const ip = value[0]; + const lastUpdated = value[1]; + const nowSeconds = Date.now(); + const delta = nowSeconds - lastUpdated; + if (delta < ddnsResolver._updateIntervalMs) { + return Promise.resolve(ip); + } + } + ddnsResolver._cache.delete(address); + // Reach here only if cache value doesn't exist or needs to be updated + let host = address.toLowerCase(); + if (host.startsWith('ddns:')) { + host = host.substring(5); + } + return ddnsResolver._queryHost(host) + .then((resolvedIP) => { + ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); + return resolvedIP; + }) + .catch((/*error*/) => { + // return input address in case of failure + return address; + }); + }, - /** Private **/ - // Properties - _initialized: false, - _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) - /** + /** Private **/ + // Properties + _initialized: false, + _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) + /** * cache mapping host to (ip address, last updated time) */ - _cache: new Map(), - _interval: null, // reference to created interval id - _processingDDNSUpdate: false, + _cache: new Map(), + _interval: null, // reference to created interval id + _processingDDNSUpdate: false, - // Methods + // Methods - _initialize: () => { - if (ddnsResolver._initialized) { - return; - } - // Init the resolver - // Read and set custom update interval from env if needed - if (typeof process.env.DDNS_UPDATE_INTERVAL !== 'undefined') { - const interval = Number(process.env.DDNS_UPDATE_INTERVAL.toLowerCase()); - if (!isNaN(interval)) { - // Interval value from env is in seconds. Set min to 60s. - ddnsResolver._updateIntervalMs = Math.max(interval * 1000, 60 * 1000); - } else { - logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`); - } - } - ddnsResolver._initialized = true; - }, + _initialize: () => { + if (ddnsResolver._initialized) { + return; + } + // Init the resolver + // Read and set custom update interval from env if needed + if (typeof process.env.DDNS_UPDATE_INTERVAL !== 'undefined') { + const interval = Number(process.env.DDNS_UPDATE_INTERVAL.toLowerCase()); + if (!isNaN(interval)) { + // Interval value from env is in seconds. Set min to 60s. + ddnsResolver._updateIntervalMs = Math.max(interval * 1000, 60 * 1000); + } else { + logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`); + } + } + ddnsResolver._initialized = true; + }, - /** + /** * * @param {String} host * @returns {Promise} */ - _queryHost: (host) => { - logger.info('Looking up IP for ', host); - return utils.execSafe('getent', ['hosts', host]) - .then((result) => { - if (result.length < 8) { - logger.error('IP lookup returned invalid output: ', result); - throw error.ValidationError('Invalid output from getent hosts'); - } - const out = result.split(/\s+/); - logger.info(`Resolved ${host} to ${out[0]}`); - return out[0]; - }, - (error) => { - logger.error('Error looking up IP for ' + host + ': ', error); - throw error; - }); - }, + _queryHost: (host) => { + logger.info('Looking up IP for ', host); + return utils.execSafe('getent', ['hosts', host]) + .then((result) => { + if (result.length < 8) { + logger.error('IP lookup returned invalid output: ', result); + throw error.ValidationError('Invalid output from getent hosts'); + } + const out = result.split(/\s+/); + logger.info(`Resolved ${host} to ${out[0]}`); + return out[0]; + }, + (error) => { + logger.error('Error looking up IP for ' + host + ': ', error); + throw error; + }); + }, - /** + /** * Triggered by a timer, will check for and update ddns hosts in access list clients */ - _checkForDDNSUpdates: () => { - const internalNginx = require('../internal/nginx'); // Prevent circular import + _checkForDDNSUpdates: () => { + const internalNginx = require('../internal/nginx'); // Prevent circular import - logger.info('Checking for DDNS updates...'); - if (!ddnsResolver._processingDDNSUpdate) { - ddnsResolver._processingDDNSUpdate = true; + logger.info('Checking for DDNS updates...'); + if (!ddnsResolver._processingDDNSUpdate) { + ddnsResolver._processingDDNSUpdate = true; - const updatedAddresses = new Map(); + const updatedAddresses = new Map(); - // Get all ddns hostnames in use - return ddnsResolver._getAccessLists() - .then((rows) => { - // Build map of used addresses that require resolution - const usedAddresses = new Map(); - for (const row of rows) { - if (!row.proxy_host_count) { - // Ignore rows (access lists) that are not associated to any hosts - continue; - } - for (const client of row.clients) { - if (!ddnsResolver.requiresResolution(client.address)) { - continue; - } - if (!usedAddresses.has(client.address)) { - usedAddresses.set(client.address, [row]); - } else { - usedAddresses.get(client.address).push(row); - } - } - } - logger.info(`Found ${usedAddresses.size} address(es) in use.`); - // Remove unused addresses - const addressesToRemove = []; - for (const address of ddnsResolver._cache.keys()) { - if (!usedAddresses.has(address)) { - addressesToRemove.push(address); - } - } - addressesToRemove.forEach((address) => { ddnsResolver._cache.delete(address); }); + // Get all ddns hostnames in use + return ddnsResolver._getAccessLists() + .then((rows) => { + // Build map of used addresses that require resolution + const usedAddresses = new Map(); + for (const row of rows) { + if (!row.proxy_host_count) { + // Ignore rows (access lists) that are not associated to any hosts + continue; + } + for (const client of row.clients) { + if (!ddnsResolver.requiresResolution(client.address)) { + continue; + } + if (!usedAddresses.has(client.address)) { + usedAddresses.set(client.address, [row]); + } else { + usedAddresses.get(client.address).push(row); + } + } + } + logger.info(`Found ${usedAddresses.size} address(es) in use.`); + // Remove unused addresses + const addressesToRemove = []; + for (const address of ddnsResolver._cache.keys()) { + if (!usedAddresses.has(address)) { + addressesToRemove.push(address); + } + } + addressesToRemove.forEach((address) => { ddnsResolver._cache.delete(address); }); - const promises = []; + const promises = []; - for (const [address, rows] of usedAddresses) { - let oldIP = ''; - if (ddnsResolver._cache.has(address)) { - oldIP = ddnsResolver._cache.get(address)[0]; - } - const p = ddnsResolver.resolveAddress(address, true) - .then((resolvedIP) => { - if (resolvedIP !== address && resolvedIP !== oldIP) { - // Mark this as an updated address - updatedAddresses.set(address, rows); - } - }); - promises.push(p); - } + for (const [address, rows] of usedAddresses) { + let oldIP = ''; + if (ddnsResolver._cache.has(address)) { + oldIP = ddnsResolver._cache.get(address)[0]; + } + const p = ddnsResolver.resolveAddress(address, true) + .then((resolvedIP) => { + if (resolvedIP !== address && resolvedIP !== oldIP) { + // Mark this as an updated address + updatedAddresses.set(address, rows); + } + }); + promises.push(p); + } - if (promises.length) { - return Promise.all(promises); - } - return Promise.resolve(); - }) - .then(() => { - logger.info(`${updatedAddresses.size} DDNS IP(s) updated.`); - const updatedRows = new Map(); - const proxy_hosts = []; - for (const rows of updatedAddresses.values()) { - for (const row of rows) { - if (!updatedRows.has(row.id)) { - updatedRows.set(row.id, 1); - proxy_hosts.push(...row.proxy_hosts); - } - } - } - if (proxy_hosts.length) { - logger.info(`Updating ${proxy_hosts.length} proxy host(s) affected by DDNS changes`); - return internalNginx.bulkGenerateConfigs('proxy_host', proxy_hosts) - .then(internalNginx.reload); - } - return Promise.resolve(); - }) - .then(() => { - logger.info('Finished checking for DDNS updates'); - ddnsResolver._processingDDNSUpdate = false; - }); - } else { - logger.info('Skipping since previous DDNS update check is in progress'); - } - }, + if (promises.length) { + return Promise.all(promises); + } + return Promise.resolve(); + }) + .then(() => { + logger.info(`${updatedAddresses.size} DDNS IP(s) updated.`); + const updatedRows = new Map(); + const proxy_hosts = []; + for (const rows of updatedAddresses.values()) { + for (const row of rows) { + if (!updatedRows.has(row.id)) { + updatedRows.set(row.id, 1); + proxy_hosts.push(...row.proxy_hosts); + } + } + } + if (proxy_hosts.length) { + logger.info(`Updating ${proxy_hosts.length} proxy host(s) affected by DDNS changes`); + return internalNginx.bulkGenerateConfigs('proxy_host', proxy_hosts) + .then(internalNginx.reload); + } + return Promise.resolve(); + }) + .then(() => { + logger.info('Finished checking for DDNS updates'); + ddnsResolver._processingDDNSUpdate = false; + }); + } else { + logger.info('Skipping since previous DDNS update check is in progress'); + } + }, - _getAccessLists: () => { - const fakeAccess = { - can: (capabilityStr) => { - return Promise.resolve({ - permission_visibility: 'all' - }) - } - }; + _getAccessLists: () => { + const fakeAccess = { + can: (/*role*/) => { + return Promise.resolve({ + permission_visibility: 'all' + }); + } + }; - return internalAccessList.getAll(fakeAccess) - .then((rows) => { - const promises = []; - for (const row of rows) { - const p = internalAccessList.get(fakeAccess, { - id: row.id, - expand: ['owner', 'items', 'clients', 'proxy_hosts.[certificate,access_list.[clients,items]]'] - }, true /* <- skip masking */); - promises.push(p); - } - if (promises.length) { - return Promise.all(promises); - } - return Promise.resolve([]); - }); - } + return internalAccessList.getAll(fakeAccess) + .then((rows) => { + const promises = []; + for (const row of rows) { + const p = internalAccessList.get(fakeAccess, { + id: row.id, + expand: ['owner', 'items', 'clients', 'proxy_hosts.[certificate,access_list.[clients,items]]'] + }, true /* <- skip masking */); + promises.push(p); + } + if (promises.length) { + return Promise.all(promises); + } + return Promise.resolve([]); + }); + } }; module.exports = ddnsResolver; From 33f41f7e6ff5bd52a445147a0956f280308f1fd3 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 20:12:34 -0500 Subject: [PATCH 04/23] Fix utils.js linter error --- backend/lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/utils.js b/backend/lib/utils.js index 188a8cbe1..94ed74cea 100644 --- a/backend/lib/utils.js +++ b/backend/lib/utils.js @@ -51,7 +51,7 @@ module.exports = { } else { reject({ exitCode: exitCode, - stderr: stderr + stderr: stderr }); } }); From 743cdd8b0be23923c2f7a2f039aa5b02f2dd0351 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 21:25:15 -0500 Subject: [PATCH 05/23] Eliminate circular dependency --- backend/index.js | 4 +- backend/internal/nginx.js | 2 +- backend/lib/ddns_resolver/ddns_resolver.js | 85 ++++++++++++++ .../ddns_updater.js} | 110 +++--------------- 4 files changed, 105 insertions(+), 96 deletions(-) create mode 100644 backend/lib/ddns_resolver/ddns_resolver.js rename backend/lib/{ddns_resolver.js => ddns_resolver/ddns_updater.js} (59%) diff --git a/backend/index.js b/backend/index.js index 129cf060d..7c75433d1 100644 --- a/backend/index.js +++ b/backend/index.js @@ -9,7 +9,7 @@ async function appStart () { const apiValidator = require('./lib/validator/api'); const internalCertificate = require('./internal/certificate'); const internalIpRanges = require('./internal/ip_ranges'); - const ddnsResolver = require('./lib/ddns_resolver'); + const ddnsUpdater = require('./lib/ddns_resolver/ddns_updater'); return migrate.latest() .then(setup) @@ -21,7 +21,7 @@ async function appStart () { internalCertificate.initTimer(); internalIpRanges.initTimer(); - ddnsResolver.initTimer(); + ddnsUpdater.initTimer(); const server = app.listen(3000, () => { logger.info('Backend PID ' + process.pid + ' listening on port 3000 ...'); diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 8256ae32b..3708808eb 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -4,7 +4,7 @@ const logger = require('../logger').nginx; const config = require('../lib/config'); const utils = require('../lib/utils'); const error = require('../lib/error'); -const ddnsResolver = require('../lib/ddns_resolver'); +const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver'); const internalNginx = { diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js new file mode 100644 index 000000000..0fa45e0e3 --- /dev/null +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -0,0 +1,85 @@ +const error = require('../error'); +const logger = require('../../logger').ddns; +const utils = require('../utils'); + +const ddnsResolver = { + /** + * Checks whether the address requires resolution (i.e. starts with ddns:) + * @param {String} address + * @returns {boolean} + */ + requiresResolution: (address) => { + if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { + return true; + } + return false; + }, + + /** + * Resolves the given address to its IP + * @param {String} address + * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value + */ + resolveAddress: (address, forceUpdate=false) => { + if (!forceUpdate && ddnsResolver._cache.has(address)) { + // Check if it is still valid + const value = ddnsResolver._cache.get(address); + const ip = value[0]; + const lastUpdated = value[1]; + const nowSeconds = Date.now(); + const delta = nowSeconds - lastUpdated; + if (delta < ddnsResolver._updateIntervalMs) { + return Promise.resolve(ip); + } + } + ddnsResolver._cache.delete(address); + // Reach here only if cache value doesn't exist or needs to be updated + let host = address.toLowerCase(); + if (host.startsWith('ddns:')) { + host = host.substring(5); + } + return ddnsResolver._queryHost(host) + .then((resolvedIP) => { + ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); + return resolvedIP; + }) + .catch((/*error*/) => { + // return input address in case of failure + return address; + }); + }, + + + /** Private **/ + // Properties + /** + * cache mapping host to (ip address, last updated time) + */ + _cache: new Map(), + + // Methods + /** + * + * @param {String} host + * @returns {Promise} + */ + _queryHost: (host) => { + logger.info('Looking up IP for ', host); + return utils.execSafe('getent', ['hosts', host]) + .then((result) => { + if (result.length < 8) { + logger.error('IP lookup returned invalid output: ', result); + throw error.ValidationError('Invalid output from getent hosts'); + } + const out = result.split(/\s+/); + logger.info(`Resolved ${host} to ${out[0]}`); + return out[0]; + }, + (error) => { + logger.error('Error looking up IP for ' + host + ': ', error); + throw error; + }); + }, +}; + +module.exports = ddnsResolver; diff --git a/backend/lib/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_updater.js similarity index 59% rename from backend/lib/ddns_resolver.js rename to backend/lib/ddns_resolver/ddns_updater.js index 31b3f5874..f00e588d2 100644 --- a/backend/lib/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_updater.js @@ -1,82 +1,31 @@ -const error = require('./error'); -const logger = require('../logger').ddns; -const internalAccessList = require('../internal/access-list'); -const utils = require('./utils'); +const internalNginx = require('../../internal/nginx'); +const logger = require('../../logger').ddns; +const internalAccessList = require('../../internal/access-list'); +const ddnsResolver = require('./ddns_resolver'); -const ddnsResolver = { +const ddnsUpdater = { /** * Starts a timer to periodically check for ddns updates */ initTimer: () => { - ddnsResolver._initialize(); - ddnsResolver._interval = setInterval(ddnsResolver._checkForDDNSUpdates, ddnsResolver._updateIntervalMs); - logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsResolver._updateIntervalMs / 1000)}s)`); + ddnsUpdater._initialize(); + ddnsUpdater._interval = setInterval(ddnsUpdater._checkForDDNSUpdates, ddnsUpdater._updateIntervalMs); + logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsUpdater._updateIntervalMs / 1000)}s)`); // Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up - setTimeout(ddnsResolver._checkForDDNSUpdates, 10 * 1000); + setTimeout(ddnsUpdater._checkForDDNSUpdates, 10 * 1000); }, - /** - * Checks whether the address requires resolution (i.e. starts with ddns:) - * @param {String} address - * @returns {boolean} - */ - requiresResolution: (address) => { - if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { - return true; - } - return false; - }, - - /** - * Resolves the given address to its IP - * @param {String} address - * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value - */ - resolveAddress: (address, forceUpdate=false) => { - if (!forceUpdate && ddnsResolver._cache.has(address)) { - // Check if it is still valid - const value = ddnsResolver._cache.get(address); - const ip = value[0]; - const lastUpdated = value[1]; - const nowSeconds = Date.now(); - const delta = nowSeconds - lastUpdated; - if (delta < ddnsResolver._updateIntervalMs) { - return Promise.resolve(ip); - } - } - ddnsResolver._cache.delete(address); - // Reach here only if cache value doesn't exist or needs to be updated - let host = address.toLowerCase(); - if (host.startsWith('ddns:')) { - host = host.substring(5); - } - return ddnsResolver._queryHost(host) - .then((resolvedIP) => { - ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); - return resolvedIP; - }) - .catch((/*error*/) => { - // return input address in case of failure - return address; - }); - }, - - /** Private **/ // Properties _initialized: false, _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) - /** - * cache mapping host to (ip address, last updated time) - */ - _cache: new Map(), _interval: null, // reference to created interval id _processingDDNSUpdate: false, // Methods _initialize: () => { - if (ddnsResolver._initialized) { + if (ddnsUpdater._initialized) { return; } // Init the resolver @@ -85,51 +34,26 @@ const ddnsResolver = { const interval = Number(process.env.DDNS_UPDATE_INTERVAL.toLowerCase()); if (!isNaN(interval)) { // Interval value from env is in seconds. Set min to 60s. - ddnsResolver._updateIntervalMs = Math.max(interval * 1000, 60 * 1000); + ddnsUpdater._updateIntervalMs = Math.max(interval * 1000, 60 * 1000); } else { logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`); } } - ddnsResolver._initialized = true; - }, - - /** - * - * @param {String} host - * @returns {Promise} - */ - _queryHost: (host) => { - logger.info('Looking up IP for ', host); - return utils.execSafe('getent', ['hosts', host]) - .then((result) => { - if (result.length < 8) { - logger.error('IP lookup returned invalid output: ', result); - throw error.ValidationError('Invalid output from getent hosts'); - } - const out = result.split(/\s+/); - logger.info(`Resolved ${host} to ${out[0]}`); - return out[0]; - }, - (error) => { - logger.error('Error looking up IP for ' + host + ': ', error); - throw error; - }); + ddnsUpdater._initialized = true; }, /** * Triggered by a timer, will check for and update ddns hosts in access list clients */ _checkForDDNSUpdates: () => { - const internalNginx = require('../internal/nginx'); // Prevent circular import - logger.info('Checking for DDNS updates...'); - if (!ddnsResolver._processingDDNSUpdate) { - ddnsResolver._processingDDNSUpdate = true; + if (!ddnsUpdater._processingDDNSUpdate) { + ddnsUpdater._processingDDNSUpdate = true; const updatedAddresses = new Map(); // Get all ddns hostnames in use - return ddnsResolver._getAccessLists() + return ddnsUpdater._getAccessLists() .then((rows) => { // Build map of used addresses that require resolution const usedAddresses = new Map(); @@ -202,7 +126,7 @@ const ddnsResolver = { }) .then(() => { logger.info('Finished checking for DDNS updates'); - ddnsResolver._processingDDNSUpdate = false; + ddnsUpdater._processingDDNSUpdate = false; }); } else { logger.info('Skipping since previous DDNS update check is in progress'); @@ -236,4 +160,4 @@ const ddnsResolver = { } }; -module.exports = ddnsResolver; +module.exports = ddnsUpdater; \ No newline at end of file From 7b09fefd1799be082e0c140e446a6ee260719333 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Dec 2023 22:47:22 -0500 Subject: [PATCH 06/23] Update configs for active hosts only on ddns update Other changes: - Fixed null property read error on clients (when switching to public access) - Use separate `resolvedAddress` field for resolved IP instead of overwriting address - Reduced ddns log verbosity --- backend/internal/nginx.js | 9 +++++---- backend/lib/ddns_resolver/ddns_resolver.js | 4 +--- backend/lib/ddns_resolver/ddns_updater.js | 6 +++++- backend/lib/utils.js | 3 +++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 3708808eb..84cec0d5a 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -139,12 +139,13 @@ const internalNginx = { */ resolveDDNSAddresses: (host) => { const promises = []; - if (typeof host.access_list !== 'undefined' && typeof host.access_list.clients !== 'undefined') { + if (typeof host.access_list !== 'undefined' && host.access_list && typeof host.access_list.clients !== 'undefined' && host.access_list.clients) { for (const client of host.access_list.clients) { - if (ddnsResolver.requiresResolution(client.address)) { - const p = ddnsResolver.resolveAddress(client.address) + const address = client.address; + if (ddnsResolver.requiresResolution(address)) { + const p = ddnsResolver.resolveAddress(address) .then((resolvedIP) => { - client.address = `${resolvedIP}; # ${client.address}`; + Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP}); return Promise.resolve(); }); promises.push(p); diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 0fa45e0e3..38b6168bc 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -64,15 +64,13 @@ const ddnsResolver = { * @returns {Promise} */ _queryHost: (host) => { - logger.info('Looking up IP for ', host); return utils.execSafe('getent', ['hosts', host]) .then((result) => { if (result.length < 8) { - logger.error('IP lookup returned invalid output: ', result); + logger.error(`IP lookup for ${host} returned invalid output: ${result}`); throw error.ValidationError('Invalid output from getent hosts'); } const out = result.split(/\s+/); - logger.info(`Resolved ${host} to ${out[0]}`); return out[0]; }, (error) => { diff --git a/backend/lib/ddns_resolver/ddns_updater.js b/backend/lib/ddns_resolver/ddns_updater.js index f00e588d2..f67ed1a86 100644 --- a/backend/lib/ddns_resolver/ddns_updater.js +++ b/backend/lib/ddns_resolver/ddns_updater.js @@ -113,7 +113,11 @@ const ddnsUpdater = { for (const row of rows) { if (!updatedRows.has(row.id)) { updatedRows.set(row.id, 1); - proxy_hosts.push(...row.proxy_hosts); + for (const host of row.proxy_hosts) { + if (host.enabled) { + proxy_hosts.push(host); + } + } } } } diff --git a/backend/lib/utils.js b/backend/lib/utils.js index 94ed74cea..26a0ffdc1 100644 --- a/backend/lib/utils.js +++ b/backend/lib/utils.js @@ -128,6 +128,9 @@ module.exports = { */ renderEngine.registerFilter('nginxAccessRule', (v) => { if (typeof v.directive !== 'undefined' && typeof v.address !== 'undefined' && v.directive && v.address) { + if (typeof v.resolvedAddress !== 'undefined' && v.resolvedAddress) { + return `${v.directive} ${v.resolvedAddress}; # ${v.address}`; + } return `${v.directive} ${v.address};`; } return ''; From 3b0ff570d9b948eb3503075cf92437c063e42c3d Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Mon, 11 Dec 2023 22:32:08 -0500 Subject: [PATCH 07/23] doc string update --- backend/internal/nginx.js | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 84cec0d5a..d59f1104f 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -134,6 +134,7 @@ const internalNginx = { /** * Resolves any ddns addresses that need to be resolved for clients in the host's access list. + * Defines a new property 'resolvedAddress' on each client in `host.access_list.clients` that uses a ddns address. * @param {Object} host * @returns {Promise} */ From e3179006d1f186a50794137ece08b7495a28e7a3 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sun, 28 Apr 2024 17:54:21 -0700 Subject: [PATCH 08/23] Add support for '-' in ddns domain names --- backend/schema/endpoints/access-lists.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/schema/endpoints/access-lists.json b/backend/schema/endpoints/access-lists.json index 94585f30b..59aff5c49 100644 --- a/backend/schema/endpoints/access-lists.json +++ b/backend/schema/endpoints/access-lists.json @@ -39,7 +39,7 @@ }, { "type": "string", - "pattern": "^ddns:[\\w\\.]+$" + "pattern": "^ddns:[\\w\\.-]+$" } ] }, From cae8ba9fd43547e0857cc601b84e900080cc0788 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 22 Feb 2025 20:44:21 -0500 Subject: [PATCH 09/23] add ddns field to new json files --- backend/schema/paths/nginx/access-lists/listID/put.json | 4 ++++ backend/schema/paths/nginx/access-lists/post.json | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/backend/schema/paths/nginx/access-lists/listID/put.json b/backend/schema/paths/nginx/access-lists/listID/put.json index 7f887dad6..657694aee 100644 --- a/backend/schema/paths/nginx/access-lists/listID/put.json +++ b/backend/schema/paths/nginx/access-lists/listID/put.json @@ -73,6 +73,10 @@ { "type": "string", "pattern": "^all$" + }, + { + "type": "string", + "pattern": "^ddns:[\\w\\.-]+$" } ] }, diff --git a/backend/schema/paths/nginx/access-lists/post.json b/backend/schema/paths/nginx/access-lists/post.json index 4c5a4edd2..76c5211f9 100644 --- a/backend/schema/paths/nginx/access-lists/post.json +++ b/backend/schema/paths/nginx/access-lists/post.json @@ -62,6 +62,10 @@ { "type": "string", "pattern": "^all$" + }, + { + "type": "string", + "pattern": "^ddns:[\\w\\.-]+$" } ] }, From 00ee20c4f560078495f0281cd18b1bccfa947eff Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 19:08:47 -0500 Subject: [PATCH 10/23] updated ddns resolver to accept any domain/subdomain instead of domains/subdomains prefixed with ddns, update resolution to return ipv4 addresses --- backend/internal/nginx.js | 2 +- backend/lib/ddns_resolver/ddns_resolver.js | 18 +++--------------- backend/lib/ddns_resolver/ddns_updater.js | 2 +- .../schema/components/access-list-object.json | 2 +- .../paths/nginx/access-lists/listID/put.json | 2 +- .../schema/paths/nginx/access-lists/post.json | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 15c0783ff..35a179146 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -143,7 +143,7 @@ const internalNginx = { if (typeof host.access_list !== 'undefined' && host.access_list && typeof host.access_list.clients !== 'undefined' && host.access_list.clients) { for (const client of host.access_list.clients) { const address = client.address; - if (ddnsResolver.requiresResolution(address)) { + if (ddnsResolver.ddnsRegex.test(address)) { const p = ddnsResolver.resolveAddress(address) .then((resolvedIP) => { Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP}); diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 38b6168bc..b1ebbbae4 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -3,17 +3,8 @@ const logger = require('../../logger').ddns; const utils = require('../utils'); const ddnsResolver = { - /** - * Checks whether the address requires resolution (i.e. starts with ddns:) - * @param {String} address - * @returns {boolean} - */ - requiresResolution: (address) => { - if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) { - return true; - } - return false; - }, + /** Pattern to match any valid domain/subdomain */ + ddnsRegex: /'^((?!-)[A-Za-z\d-]{1,63}(? { ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); @@ -64,7 +52,7 @@ const ddnsResolver = { * @returns {Promise} */ _queryHost: (host) => { - return utils.execSafe('getent', ['hosts', host]) + return utils.execSafe('getent', ['ahostsv4', 'hosts', host]) .then((result) => { if (result.length < 8) { logger.error(`IP lookup for ${host} returned invalid output: ${result}`); diff --git a/backend/lib/ddns_resolver/ddns_updater.js b/backend/lib/ddns_resolver/ddns_updater.js index f67ed1a86..0409503dc 100644 --- a/backend/lib/ddns_resolver/ddns_updater.js +++ b/backend/lib/ddns_resolver/ddns_updater.js @@ -63,7 +63,7 @@ const ddnsUpdater = { continue; } for (const client of row.clients) { - if (!ddnsResolver.requiresResolution(client.address)) { + if (!ddnsResolver.ddnsRegex.test(client.address)) { continue; } if (!usedAddresses.has(client.address)) { diff --git a/backend/schema/components/access-list-object.json b/backend/schema/components/access-list-object.json index f465f36c8..1ecbab9a9 100644 --- a/backend/schema/components/access-list-object.json +++ b/backend/schema/components/access-list-object.json @@ -40,7 +40,7 @@ }, { "type": "string", - "pattern": "^ddns:[\\w\\.-]+$" + "pattern": "^((?!-)[A-Za-z\\d-]{1,63}(? Date: Sat, 1 Mar 2025 19:10:36 -0500 Subject: [PATCH 11/23] remove quotes from regex --- backend/lib/ddns_resolver/ddns_resolver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index b1ebbbae4..c7e1af783 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -4,7 +4,7 @@ const utils = require('../utils'); const ddnsResolver = { /** Pattern to match any valid domain/subdomain */ - ddnsRegex: /'^((?!-)[A-Za-z\d-]{1,63}(? Date: Sat, 1 Mar 2025 19:21:42 -0500 Subject: [PATCH 12/23] update comments and documentation, validate output from getent with regex --- backend/lib/ddns_resolver/ddns_resolver.js | 36 ++++++++++------------ 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index c7e1af783..95db5e7fc 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -8,13 +8,13 @@ const ddnsResolver = { /** * Resolves the given address to its IP - * @param {String} address - * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value + * @param {String} domainName domain name of the dynamic DNS record + * @param {boolean} forceUpdate option to force resolution instead of using the cached value */ - resolveAddress: (address, forceUpdate=false) => { - if (!forceUpdate && ddnsResolver._cache.has(address)) { + resolveAddress: (domainName, forceUpdate=false) => { + if (!forceUpdate && ddnsResolver._cache.has(domainName)) { // Check if it is still valid - const value = ddnsResolver._cache.get(address); + const value = ddnsResolver._cache.get(domainName); const ip = value[0]; const lastUpdated = value[1]; const nowSeconds = Date.now(); @@ -23,39 +23,35 @@ const ddnsResolver = { return Promise.resolve(ip); } } - ddnsResolver._cache.delete(address); + ddnsResolver._cache.delete(domainName); // Reach here only if cache value doesn't exist or needs to be updated - let host = address.toLowerCase(); + let host = domainName.toLowerCase(); return ddnsResolver._queryHost(host) .then((resolvedIP) => { - ddnsResolver._cache.set(address, [resolvedIP, Date.now()]); + ddnsResolver._cache.set(domainName, [resolvedIP, Date.now()]); return resolvedIP; }) .catch((/*error*/) => { // return input address in case of failure - return address; + logger.error(`Failed to resolve IP for ${host}`); + return domainName; }); }, - /** Private **/ - // Properties - /** - * cache mapping host to (ip address, last updated time) - */ + /** Cache mapping host to (ip address, last updated time) */ _cache: new Map(), - // Methods /** - * - * @param {String} host - * @returns {Promise} + * Uses execSafe to query the IP address of the given host + * @param {String} host host to query + * @returns {Promise} resolves to the IPV4 address of the host */ _queryHost: (host) => { return utils.execSafe('getent', ['ahostsv4', 'hosts', host]) .then((result) => { - if (result.length < 8) { - logger.error(`IP lookup for ${host} returned invalid output: ${result}`); + if (result.length < 8 || !/^(\d{1,3}\.){3}\d{1,3}$/.test(result)) { + logger.error(`IPV4 lookup for ${host} returned invalid output: ${result}`); throw error.ValidationError('Invalid output from getent hosts'); } const out = result.split(/\s+/); From 72e2e096c1cbac213666ad14a83798a841db9248 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 19:25:13 -0500 Subject: [PATCH 13/23] update access list regex to use \d instead of [0-9] --- backend/schema/components/access-list-object.json | 4 ++-- backend/schema/paths/nginx/access-lists/listID/put.json | 4 ++-- backend/schema/paths/nginx/access-lists/post.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/schema/components/access-list-object.json b/backend/schema/components/access-list-object.json index 1ecbab9a9..91c220e63 100644 --- a/backend/schema/components/access-list-object.json +++ b/backend/schema/components/access-list-object.json @@ -28,11 +28,11 @@ "oneOf": [ { "type": "string", - "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$" + "pattern": "^([\\d]{1,3}\\.){3}[\\d]{1,3}(/([\\d]|[1-2][\\d]|3[0-2]))?$" }, { "type": "string", - "pattern": "^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$" + "pattern": "^s*((([\\dA-Fa-f]{1,4}:){7}([\\dA-Fa-f]{1,4}|:))|(([\\dA-Fa-f]{1,4}:){6}(:[\\dA-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){5}(((:[\\dA-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){4}(((:[\\dA-Fa-f]{1,4}){1,3})|((:[\\dA-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){3}(((:[\\dA-Fa-f]{1,4}){1,4})|((:[\\dA-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){2}(((:[\\dA-Fa-f]{1,4}){1,5})|((:[\\dA-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){1}(((:[\\dA-Fa-f]{1,4}){1,6})|((:[\\dA-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[\\dA-Fa-f]{1,4}){1,7})|((:[\\dA-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([\\d]|[1-9][\\d]|1[0-1][\\d]|12[0-8]))?$" }, { "type": "string", diff --git a/backend/schema/paths/nginx/access-lists/listID/put.json b/backend/schema/paths/nginx/access-lists/listID/put.json index f6e248ba0..e80b1e89b 100644 --- a/backend/schema/paths/nginx/access-lists/listID/put.json +++ b/backend/schema/paths/nginx/access-lists/listID/put.json @@ -64,11 +64,11 @@ "oneOf": [ { "type": "string", - "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$" + "pattern": "^([\\d]{1,3}\\.){3}[\\d]{1,3}(/([\\d]|[1-2][\\d]|3[0-2]))?$" }, { "type": "string", - "pattern": "^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$" + "pattern": "^s*((([\\dA-Fa-f]{1,4}:){7}([\\dA-Fa-f]{1,4}|:))|(([\\dA-Fa-f]{1,4}:){6}(:[\\dA-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){5}(((:[\\dA-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){4}(((:[\\dA-Fa-f]{1,4}){1,3})|((:[\\dA-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){3}(((:[\\dA-Fa-f]{1,4}){1,4})|((:[\\dA-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){2}(((:[\\dA-Fa-f]{1,4}){1,5})|((:[\\dA-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){1}(((:[\\dA-Fa-f]{1,4}){1,6})|((:[\\dA-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[\\dA-Fa-f]{1,4}){1,7})|((:[\\dA-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([\\d]|[1-9][\\d]|1[0-1][\\d]|12[0-8]))?$" }, { "type": "string", diff --git a/backend/schema/paths/nginx/access-lists/post.json b/backend/schema/paths/nginx/access-lists/post.json index b2e40ce34..bee464e12 100644 --- a/backend/schema/paths/nginx/access-lists/post.json +++ b/backend/schema/paths/nginx/access-lists/post.json @@ -53,11 +53,11 @@ "oneOf": [ { "type": "string", - "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$" + "pattern": "^([\\d]{1,3}\\.){3}[\\d]{1,3}(/([\\d]|[1-2][\\d]|3[0-2]))?$" }, { "type": "string", - "pattern": "^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$" + "pattern": "^s*((([\\dA-Fa-f]{1,4}:){7}([\\dA-Fa-f]{1,4}|:))|(([\\dA-Fa-f]{1,4}:){6}(:[\\dA-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){5}(((:[\\dA-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([\\dA-Fa-f]{1,4}:){4}(((:[\\dA-Fa-f]{1,4}){1,3})|((:[\\dA-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){3}(((:[\\dA-Fa-f]{1,4}){1,4})|((:[\\dA-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){2}(((:[\\dA-Fa-f]{1,4}){1,5})|((:[\\dA-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([\\dA-Fa-f]{1,4}:){1}(((:[\\dA-Fa-f]{1,4}){1,6})|((:[\\dA-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[\\dA-Fa-f]{1,4}){1,7})|((:[\\dA-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(/([\\d]|[1-9][\\d]|1[0-1][\\d]|12[0-8]))?$" }, { "type": "string", From 1f6fa75c0b184569fb1fca86895362968d734cba Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 20:14:26 -0500 Subject: [PATCH 14/23] Fix getent params in backend/lib/ddns_resolver/ddns_resolver.js Co-authored-by: Jenson R --- backend/lib/ddns_resolver/ddns_resolver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 95db5e7fc..6759df31b 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -48,7 +48,7 @@ const ddnsResolver = { * @returns {Promise} resolves to the IPV4 address of the host */ _queryHost: (host) => { - return utils.execSafe('getent', ['ahostsv4', 'hosts', host]) + return utils.execSafe('getent', ['ahostsv4', host]) .then((result) => { if (result.length < 8 || !/^(\d{1,3}\.){3}\d{1,3}$/.test(result)) { logger.error(`IPV4 lookup for ${host} returned invalid output: ${result}`); From f736815bd059a82256548acdc1c4b15016f5d31a Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 20:20:18 -0500 Subject: [PATCH 15/23] update command parsing to pull first ipv4 address from result of getent command --- backend/lib/ddns_resolver/ddns_resolver.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 6759df31b..8efab981b 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -50,12 +50,15 @@ const ddnsResolver = { _queryHost: (host) => { return utils.execSafe('getent', ['ahostsv4', host]) .then((result) => { - if (result.length < 8 || !/^(\d{1,3}\.){3}\d{1,3}$/.test(result)) { + const ipv4Regex = /(\d{1,3}\.){3}\d{1,3}/; + const match = result.match(ipv4Regex); + + if (!match) { logger.error(`IPV4 lookup for ${host} returned invalid output: ${result}`); throw error.ValidationError('Invalid output from getent hosts'); } - const out = result.split(/\s+/); - return out[0]; + + return match[0]; }, (error) => { logger.error('Error looking up IP for ' + host + ': ', error); From 98b112a990f3b06dde9e8a9a87b07ad1af894e7d Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 20:52:41 -0500 Subject: [PATCH 16/23] aligned assessments in ddns_resolver.js --- backend/lib/ddns_resolver/ddns_resolver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 8efab981b..26d367a31 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -51,7 +51,7 @@ const ddnsResolver = { return utils.execSafe('getent', ['ahostsv4', host]) .then((result) => { const ipv4Regex = /(\d{1,3}\.){3}\d{1,3}/; - const match = result.match(ipv4Regex); + const match = result.match(ipv4Regex); if (!match) { logger.error(`IPV4 lookup for ${host} returned invalid output: ${result}`); From 5e079a4b137cd72cf2fa94b950c6ea494e3ab6c0 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 21:14:03 -0500 Subject: [PATCH 17/23] return loopback address in case of failure to resolve ip address --- backend/lib/ddns_resolver/ddns_resolver.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 26d367a31..451ae7894 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -34,7 +34,7 @@ const ddnsResolver = { .catch((/*error*/) => { // return input address in case of failure logger.error(`Failed to resolve IP for ${host}`); - return domainName; + return '127.0.0.1'; // in case of failure, return the loopback address }); }, From 6d93f822fd36752a61cdf09fc33b68cefd4fc197 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 21:22:51 -0500 Subject: [PATCH 18/23] avoid adding domains to nginx config instead of resolving to loopback address --- backend/internal/nginx.js | 4 +++- backend/lib/ddns_resolver/ddns_resolver.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 35a179146..0bc911274 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -146,7 +146,9 @@ const internalNginx = { if (ddnsResolver.ddnsRegex.test(address)) { const p = ddnsResolver.resolveAddress(address) .then((resolvedIP) => { - Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP}); + if (resolvedIP !== address) { + Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP}); + } return Promise.resolve(); }); promises.push(p); diff --git a/backend/lib/ddns_resolver/ddns_resolver.js b/backend/lib/ddns_resolver/ddns_resolver.js index 451ae7894..26d367a31 100644 --- a/backend/lib/ddns_resolver/ddns_resolver.js +++ b/backend/lib/ddns_resolver/ddns_resolver.js @@ -34,7 +34,7 @@ const ddnsResolver = { .catch((/*error*/) => { // return input address in case of failure logger.error(`Failed to resolve IP for ${host}`); - return '127.0.0.1'; // in case of failure, return the loopback address + return domainName; }); }, From 6b5dcad0916491e3e3ee6fe7d15cd1c734bb7328 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 21:28:44 -0500 Subject: [PATCH 19/23] remove resolvedAddress if it matches the original address in nginx and ddns updater --- backend/internal/nginx.js | 2 ++ backend/lib/ddns_resolver/ddns_updater.js | 3 +++ 2 files changed, 5 insertions(+) diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 0bc911274..b65cde486 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -148,6 +148,8 @@ const internalNginx = { .then((resolvedIP) => { if (resolvedIP !== address) { Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP}); + } else { + delete client.resolvedAddress; } return Promise.resolve(); }); diff --git a/backend/lib/ddns_resolver/ddns_updater.js b/backend/lib/ddns_resolver/ddns_updater.js index 0409503dc..ac004b27b 100644 --- a/backend/lib/ddns_resolver/ddns_updater.js +++ b/backend/lib/ddns_resolver/ddns_updater.js @@ -92,6 +92,9 @@ const ddnsUpdater = { } const p = ddnsResolver.resolveAddress(address, true) .then((resolvedIP) => { + if (resolvedIP === address) { + updatedAddresses.delete(address); + } if (resolvedIP !== address && resolvedIP !== oldIP) { // Mark this as an updated address updatedAddresses.set(address, rows); From d70661ca94df6a69212b7eff6b8db5cab96f6124 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 23:31:30 -0500 Subject: [PATCH 20/23] resolve ddns after list is created or updated --- backend/internal/access-list.js | 9 +++++++++ backend/lib/ddns_resolver/ddns_updater.js | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/backend/internal/access-list.js b/backend/internal/access-list.js index f6043e18b..461b66c57 100644 --- a/backend/internal/access-list.js +++ b/backend/internal/access-list.js @@ -2,6 +2,7 @@ const _ = require('lodash'); const fs = require('fs'); const batchflow = require('batchflow'); const logger = require('../logger').access; +const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver'); const error = require('../lib/error'); const utils = require('../lib/utils'); const accessListModel = require('../models/access_list'); @@ -97,6 +98,10 @@ const internalAccessList = { .then(() => { return internalAccessList.maskItems(row); }); + }) + .then((result) => { + // Call the DDNS updater after the access list update process is complete + return ddnsResolver.updateDynamicDnsRecords().then(() => result); }); }, @@ -230,6 +235,10 @@ const internalAccessList = { .then(() => { return internalAccessList.maskItems(row); }); + }) + .then((result) => { + // Call the DDNS updater after the access list update process is complete + return ddnsResolver.updateDynamicDnsRecords().then(() => result); }); }, diff --git a/backend/lib/ddns_resolver/ddns_updater.js b/backend/lib/ddns_resolver/ddns_updater.js index ac004b27b..c63fc2f57 100644 --- a/backend/lib/ddns_resolver/ddns_updater.js +++ b/backend/lib/ddns_resolver/ddns_updater.js @@ -9,16 +9,16 @@ const ddnsUpdater = { */ initTimer: () => { ddnsUpdater._initialize(); - ddnsUpdater._interval = setInterval(ddnsUpdater._checkForDDNSUpdates, ddnsUpdater._updateIntervalMs); + ddnsUpdater._interval = setInterval(ddnsUpdater.updateDynamicDnsRecords, ddnsUpdater._updateIntervalMs); logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsUpdater._updateIntervalMs / 1000)}s)`); // Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up - setTimeout(ddnsUpdater._checkForDDNSUpdates, 10 * 1000); + setTimeout(ddnsUpdater.updateDynamicDnsRecords, 10 * 1000); }, /** Private **/ // Properties _initialized: false, - _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var) + _updateIntervalMs: 60 * 60 * 1000, // 1 hr default (overridden with $DDNS_UPDATE_INTERVAL env var) _interval: null, // reference to created interval id _processingDDNSUpdate: false, @@ -45,7 +45,7 @@ const ddnsUpdater = { /** * Triggered by a timer, will check for and update ddns hosts in access list clients */ - _checkForDDNSUpdates: () => { + updateDynamicDnsRecords: () => { logger.info('Checking for DDNS updates...'); if (!ddnsUpdater._processingDDNSUpdate) { ddnsUpdater._processingDDNSUpdate = true; From 450d8d2b48d9e3b672c8c82ca46e3ac14fbc2c3a Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 23:35:11 -0500 Subject: [PATCH 21/23] add note that ddns is supported --- frontend/js/app/nginx/access/form.ejs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/js/app/nginx/access/form.ejs b/frontend/js/app/nginx/access/form.ejs index 79220b14b..ab4cb8aaf 100644 --- a/frontend/js/app/nginx/access/form.ejs +++ b/frontend/js/app/nginx/access/form.ejs @@ -78,6 +78,10 @@ Nginx HTTP Access + or + + DDNS +

From 893d133ca4d5601e9817820ca82414fc8561d954 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sat, 1 Mar 2025 23:36:09 -0500 Subject: [PATCH 22/23] fix alignment whitespace issue --- backend/internal/access-list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/internal/access-list.js b/backend/internal/access-list.js index 461b66c57..e7a685026 100644 --- a/backend/internal/access-list.js +++ b/backend/internal/access-list.js @@ -2,7 +2,7 @@ const _ = require('lodash'); const fs = require('fs'); const batchflow = require('batchflow'); const logger = require('../logger').access; -const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver'); +const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver'); const error = require('../lib/error'); const utils = require('../lib/utils'); const accessListModel = require('../models/access_list'); From f4a4d2322a684c47da27886eecdbd5873421efa4 Mon Sep 17 00:00:00 2001 From: Sylphrena Kleinsasser Date: Sun, 2 Mar 2025 12:47:19 -0500 Subject: [PATCH 23/23] fix invalid function call on creation/update --- backend/internal/access-list.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/internal/access-list.js b/backend/internal/access-list.js index e7a685026..97d0087b4 100644 --- a/backend/internal/access-list.js +++ b/backend/internal/access-list.js @@ -2,7 +2,7 @@ const _ = require('lodash'); const fs = require('fs'); const batchflow = require('batchflow'); const logger = require('../logger').access; -const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver'); +const ddnsUpdater = require('../lib/ddns_resolver/ddns_updater'); const error = require('../lib/error'); const utils = require('../lib/utils'); const accessListModel = require('../models/access_list'); @@ -101,7 +101,7 @@ const internalAccessList = { }) .then((result) => { // Call the DDNS updater after the access list update process is complete - return ddnsResolver.updateDynamicDnsRecords().then(() => result); + return ddnsUpdater.updateDynamicDnsRecords().then(() => result); }); }, @@ -238,7 +238,7 @@ const internalAccessList = { }) .then((result) => { // Call the DDNS updater after the access list update process is complete - return ddnsResolver.updateDynamicDnsRecords().then(() => result); + return ddnsUpdater.updateDynamicDnsRecords().then(() => result); }); },