Merge e3179006d1f186a50794137ece08b7495a28e7a3 into 63d06da8a8591e7a9b2a1873eb91ce1c42b2b0f9

backend/index.js

@@ -9,6 +9,7 @@ async function appStart () {
 	const apiValidator        = require('./lib/validator/api');
 	const internalCertificate = require('./internal/certificate');
 	const internalIpRanges    = require('./internal/ip_ranges');
+	const ddnsUpdater         = require('./lib/ddns_resolver/ddns_updater');
 
 	return migrate.latest()
 		.then(setup)
@@ -20,6 +21,7 @@ async function appStart () {
 
 			internalCertificate.initTimer();
 			internalIpRanges.initTimer();
+			ddnsUpdater.initTimer();
 
 			const server = app.listen(3000, () => {
 				logger.info('Backend PID ' + process.pid + ' listening on port 3000 ...');

backend/internal/nginx.js

@@ -1,9 +1,10 @@
-const _      = require('lodash');
+const _            = require('lodash');
-const fs     = require('fs');
+const fs           = require('fs');
-const logger = require('../logger').nginx;
+const logger       = require('../logger').nginx;
-const config = require('../lib/config');
+const config       = require('../lib/config');
-const utils  = require('../lib/utils');
+const utils        = require('../lib/utils');
-const error  = require('../lib/error');
+const error        = require('../lib/error');
+const ddnsResolver = require('../lib/ddns_resolver/ddns_resolver');
 
 const internalNginx = {
 
@@ -131,6 +132,33 @@ const internalNginx = {
 		return '/data/nginx/' + internalNginx.getFileFriendlyHostType(host_type) + '/' + host_id + '.conf';
 	},
 
+	/**
+	 * Resolves any ddns addresses that need to be resolved for clients in the host's access list.
+	 * Defines a new property 'resolvedAddress' on each client in `host.access_list.clients` that uses a ddns address.
+	 * @param {Object} host 
+	 * @returns {Promise}
+	 */
+	resolveDDNSAddresses: (host) => {
+		const promises = [];
+		if (typeof host.access_list !== 'undefined' && host.access_list && typeof host.access_list.clients !== 'undefined' && host.access_list.clients) {
+			for (const client of host.access_list.clients) {
+				const address = client.address;
+				if (ddnsResolver.requiresResolution(address)) {
+					const p = ddnsResolver.resolveAddress(address)
+						.then((resolvedIP) => {
+							Object.defineProperty(client, 'resolvedAddress', {value: resolvedIP});
+							return Promise.resolve();
+						});
+					promises.push(p);
+				}
+			}
+		}
+		if (promises.length) {
+			return Promise.all(promises);
+		}
+		return Promise.resolve();
+	},
+
 	/**
 	 * Generates custom locations
 	 * @param   {Object}  host
@@ -201,6 +229,12 @@ const internalNginx = {
 				return;
 			}
 
+			// Resolve ddns addresses if needed
+			let resolverPromise = Promise.resolve();
+			if (host_type === 'proxy_host') {
+				resolverPromise = internalNginx.resolveDDNSAddresses(host);
+			}
+
 			let locationsPromise;
 			let origLocations;
 
@@ -215,8 +249,10 @@ const internalNginx = {
 			if (host.locations) {
 				//logger.info ('host.locations = ' + JSON.stringify(host.locations, null, 2));
 				origLocations    = [].concat(host.locations);
-				locationsPromise = internalNginx.renderLocations(host).then((renderedLocations) => {
+				locationsPromise = resolverPromise.then(() => {
-					host.locations = renderedLocations;
+					return internalNginx.renderLocations(host).then((renderedLocations) => {
+						host.locations = renderedLocations;
+					});
 				});
 
 				// Allow someone who is using / custom location path to use it, and skip the default / location
@@ -227,7 +263,7 @@ const internalNginx = {
 				});
 
 			} else {
-				locationsPromise = Promise.resolve();
+				locationsPromise = resolverPromise;
 			}
 
 			// Set the IPv6 setting for the host

backend/lib/ddns_resolver/ddns_resolver.js

@@ -0,0 +1,83 @@
+const error  = require('../error');
+const logger = require('../../logger').ddns;
+const utils  = require('../utils');
+
+const ddnsResolver = {
+	/**
+     * Checks whether the address requires resolution (i.e. starts with ddns:)
+     * @param {String} address 
+     * @returns {boolean}
+     */
+	requiresResolution: (address) => {
+		if (typeof address !== 'undefined' && address && address.toLowerCase().startsWith('ddns:')) {
+			return true;
+		}
+		return false;
+	},
+
+	/**
+     * Resolves the given address to its IP
+     * @param {String} address 
+     * @param {boolean} forceUpdate: whether to force resolution instead of using the cached value
+     */
+	resolveAddress: (address, forceUpdate=false) => {
+		if (!forceUpdate && ddnsResolver._cache.has(address)) {
+			// Check if it is still valid
+			const value       = ddnsResolver._cache.get(address);
+			const ip          = value[0];
+			const lastUpdated = value[1];
+			const nowSeconds  = Date.now();
+			const delta       = nowSeconds - lastUpdated;
+			if (delta < ddnsResolver._updateIntervalMs) {
+				return Promise.resolve(ip);
+			}
+		}
+		ddnsResolver._cache.delete(address);
+		// Reach here only if cache value doesn't exist or needs to be updated 
+		let host = address.toLowerCase();
+		if (host.startsWith('ddns:')) {
+			host = host.substring(5);
+		}
+		return ddnsResolver._queryHost(host)
+			.then((resolvedIP) => {
+				ddnsResolver._cache.set(address, [resolvedIP, Date.now()]);
+				return resolvedIP;
+			})
+			.catch((/*error*/) => {
+				// return input address in case of failure
+				return address;
+			});
+	},
+
+    
+	/** Private **/
+	// Properties
+	/**
+     * cache mapping host to (ip address, last updated time)
+     */
+	_cache: new Map(),
+
+	// Methods
+	/**
+     * 
+     * @param {String} host 
+     * @returns {Promise}
+     */
+	_queryHost: (host) => {
+		return utils.execSafe('getent', ['hosts', host])
+			.then((result) => {
+				if (result.length < 8) {
+					logger.error(`IP lookup for ${host} returned invalid output: ${result}`);
+					throw error.ValidationError('Invalid output from getent hosts');
+				}
+				const out = result.split(/\s+/);
+				return out[0];
+			},
+			(error) => {
+				logger.error('Error looking up IP for ' + host + ': ', error);
+				throw error; 
+			});
+	},
+};
+
+module.exports = ddnsResolver;

backend/lib/ddns_resolver/ddns_updater.js

@@ -0,0 +1,167 @@
+const internalNginx      = require('../../internal/nginx');
+const logger             = require('../../logger').ddns;
+const internalAccessList = require('../../internal/access-list');
+const ddnsResolver       = require('./ddns_resolver');
+
+const ddnsUpdater = {
+	/**
+     * Starts a timer to periodically check for ddns updates
+     */
+	initTimer: () => {
+		ddnsUpdater._initialize();
+		ddnsUpdater._interval = setInterval(ddnsUpdater._checkForDDNSUpdates, ddnsUpdater._updateIntervalMs);
+		logger.info(`DDNS Update Timer initialized (interval: ${Math.floor(ddnsUpdater._updateIntervalMs / 1000)}s)`);
+		// Trigger a run so that initial cache is populated and hosts can be updated - delay by 10s to give server time to boot up
+		setTimeout(ddnsUpdater._checkForDDNSUpdates, 10 * 1000);
+	},
+
+	/** Private **/
+	// Properties
+	_initialized:          false,
+	_updateIntervalMs:     60 * 60 * 1000, // 1 hr default (overriden with $DDNS_UPDATE_INTERVAL env var)
+	_interval:             null, // reference to created interval id
+	_processingDDNSUpdate: false,
+
+	// Methods
+
+	_initialize: () => {
+		if (ddnsUpdater._initialized) {
+			return;
+		}
+		// Init the resolver
+		// Read and set custom update interval from env if needed
+		if (typeof process.env.DDNS_UPDATE_INTERVAL !== 'undefined') {
+			const interval = Number(process.env.DDNS_UPDATE_INTERVAL.toLowerCase());
+			if (!isNaN(interval)) {
+				// Interval value from env is in seconds. Set min to 60s.
+				ddnsUpdater._updateIntervalMs = Math.max(interval * 1000, 60 * 1000);
+			} else {
+				logger.warn(`[DDNS] invalid value for update interval: '${process.env.DDNS_UPDATE_INTERVAL}'`);
+			}
+		}
+		ddnsUpdater._initialized = true;
+	},
+
+	/**
+     * Triggered by a timer, will check for and update ddns hosts in access list clients
+    */
+	_checkForDDNSUpdates: () => {
+		logger.info('Checking for DDNS updates...');
+		if (!ddnsUpdater._processingDDNSUpdate) {
+			ddnsUpdater._processingDDNSUpdate = true;
+            
+			const updatedAddresses = new Map();
+
+			// Get all ddns hostnames in use
+			return ddnsUpdater._getAccessLists()
+				.then((rows) => {
+					// Build map of used addresses that require resolution
+					const usedAddresses = new Map();
+					for (const row of rows) {
+						if (!row.proxy_host_count) {
+							// Ignore rows (access lists) that are not associated to any hosts
+							continue;
+						}
+						for (const client of row.clients) {
+							if (!ddnsResolver.requiresResolution(client.address)) {
+								continue;
+							}
+							if (!usedAddresses.has(client.address)) {
+								usedAddresses.set(client.address, [row]);
+							} else {
+								usedAddresses.get(client.address).push(row);
+							}
+						}
+					}
+					logger.info(`Found ${usedAddresses.size} address(es) in use.`);
+					// Remove unused addresses
+					const addressesToRemove = [];
+					for (const address of ddnsResolver._cache.keys()) {
+						if (!usedAddresses.has(address)) {
+							addressesToRemove.push(address);
+						}
+					}
+					addressesToRemove.forEach((address) => { ddnsResolver._cache.delete(address); });
+
+					const promises = [];
+
+					for (const [address, rows] of usedAddresses) {
+						let oldIP = '';
+						if (ddnsResolver._cache.has(address)) {
+							oldIP = ddnsResolver._cache.get(address)[0];
+						}
+						const p = ddnsResolver.resolveAddress(address, true)
+							.then((resolvedIP) => {
+								if (resolvedIP !== address && resolvedIP !== oldIP) {
+									// Mark this as an updated address
+									updatedAddresses.set(address, rows);
+								}
+							});
+						promises.push(p);
+					}
+
+					if (promises.length) {
+						return Promise.all(promises);
+					}
+					return Promise.resolve();
+				})
+				.then(() => {
+					logger.info(`${updatedAddresses.size} DDNS IP(s) updated.`);
+					const updatedRows = new Map();
+					const proxy_hosts = [];
+					for (const rows of updatedAddresses.values()) {
+						for (const row of rows) {
+							if (!updatedRows.has(row.id)) {
+								updatedRows.set(row.id, 1);
+								for (const host of row.proxy_hosts) {
+									if (host.enabled) {
+										proxy_hosts.push(host);
+									}
+								}
+							}
+						}
+					}
+					if (proxy_hosts.length) {
+						logger.info(`Updating ${proxy_hosts.length} proxy host(s) affected by DDNS changes`);
+						return internalNginx.bulkGenerateConfigs('proxy_host', proxy_hosts)
+							.then(internalNginx.reload);
+					}
+					return Promise.resolve();
+				})
+				.then(() => {
+					logger.info('Finished checking for DDNS updates');
+					ddnsUpdater._processingDDNSUpdate = false;
+				});
+		} else {
+			logger.info('Skipping since previous DDNS update check is in progress');
+		}
+	},
+
+	_getAccessLists: () => {
+		const fakeAccess = {
+			can: (/*role*/) => {
+				return Promise.resolve({
+					permission_visibility: 'all'
+				});
+			}
+		};
+        
+		return internalAccessList.getAll(fakeAccess)
+			.then((rows) => {
+				const promises = [];
+				for (const row of rows) {
+					const p = internalAccessList.get(fakeAccess, {
+						id:     row.id,
+						expand: ['owner', 'items', 'clients', 'proxy_hosts.[certificate,access_list.[clients,items]]']
+					}, true /* <- skip masking */);
+					promises.push(p);
+				}
+				if (promises.length) {
+					return Promise.all(promises);
+				}
+				return Promise.resolve([]);
+			});
+	}
+};
+
+module.exports = ddnsUpdater;

backend/lib/utils.js

@@ -4,6 +4,7 @@ const execFile   = require('child_process').execFile;
 const { Liquid } = require('liquidjs');
 const logger     = require('../logger').global;
 const error      = require('./error');
+const spawn      = require('child_process').spawn;
 
 module.exports = {
 
@@ -26,6 +27,37 @@ module.exports = {
 		return stdout;
 	},
 
+	/**
+	 * Run the given command. Safer than using exec since args are passed as a list instead of in shell mode as a single string.
+	 * @param {string} cmd The command to run
+	 * @param {string} args The args to pass to the command
+	 * @returns Promise that resolves to stdout or an object with error code and stderr if there's an error
+	 */
+	execSafe: (cmd, args) => {
+		return new Promise((resolve, reject) => {
+			let stdout = '';
+			let stderr = '';
+			const proc = spawn(cmd, args);
+			proc.stdout.on('data', (data) => {
+				stdout += data;
+			});
+			proc.stderr.on('data', (data) => {
+				stderr += data;
+			});
+
+			proc.on('close', (exitCode) => {
+				if (!exitCode) {
+					resolve(stdout.trim());
+				} else {
+					reject({
+						exitCode: exitCode,
+						stderr:   stderr
+					});
+				}
+			});
+		});
+	},
+
 	/**
 	 * @param   {String} cmd
 	 * @param   {Array}  args
@@ -96,6 +128,9 @@ module.exports = {
 		 */
 		renderEngine.registerFilter('nginxAccessRule', (v) => {
 			if (typeof v.directive !== 'undefined' && typeof v.address !== 'undefined' && v.directive && v.address) {
+				if (typeof v.resolvedAddress !== 'undefined' && v.resolvedAddress) {
+					return `${v.directive} ${v.resolvedAddress}; # ${v.address}`;
+				}
 				return `${v.directive} ${v.address};`;
 			}
 			return '';

backend/logger.js

@@ -10,5 +10,6 @@ module.exports = {
 	certbot:   new Signale({scope: 'Certbot  '}),
 	import:    new Signale({scope: 'Importer '}),
 	setup:     new Signale({scope: 'Setup    '}),
-	ip_ranges: new Signale({scope: 'IP Ranges'})
+	ip_ranges: new Signale({scope: 'IP Ranges'}),
+	ddns:      new Signale({scope: 'DDNS     '})
 };

backend/schema/endpoints/access-lists.json

@@ -36,6 +36,10 @@
 				{
 					"type": "string",
 					"pattern": "^all$"
+				},
+				{
+					"type": "string",
+					"pattern": "^ddns:[\\w\\.-]+$"
 				}
 			]
 		},