Refactor certbot plugins install

- Added a script to install every single plugin, used in development and debugging
- Improved certbot plugin install commands
- Adjusted some version for plugins to install properly
- It's noted that some plugins require deps that do not match other plugins,
  however these use cases should be extremely rare

backend/internal/certificate.js

@@ -9,10 +9,11 @@ const error            = require('../lib/error');
 const utils            = require('../lib/utils');
 const certificateModel = require('../models/certificate');
 const tokenModel       = require('../models/token');
-const dnsPlugins       = require('../global/certbot-dns-plugins');
+const dnsPlugins       = require('../global/certbot-dns-plugins.json');
 const internalAuditLog = require('./audit-log');
 const internalNginx    = require('./nginx');
 const internalHost     = require('./host');
+const certbot          = require('../lib/certbot');
 const archiver         = require('archiver');
 const path             = require('path');
 const { isArray }      = require('lodash');
@@ -849,26 +850,20 @@ const internalCertificate = {
 
 	/**
 	 * @param   {Object}         certificate          the certificate row
-	 * @param   {String}         dns_provider         the dns provider name (key used in `certbot-dns-plugins.js`)
+	 * @param   {String}         dns_provider         the dns provider name (key used in `certbot-dns-plugins.json`)
 	 * @param   {String | null}  credentials          the content of this providers credentials file
-	 * @param   {String}         propagation_seconds  the cloudflare api token
+	 * @param   {String}         propagation_seconds
 	 * @returns {Promise}
 	 */
-	requestLetsEncryptSslWithDnsChallenge: (certificate) => {
-		const dns_plugin = dnsPlugins[certificate.meta.dns_provider];
-
-		if (!dns_plugin) {
-			throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`);
-		}
-
-		logger.info(`Requesting Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
+	requestLetsEncryptSslWithDnsChallenge: async (certificate) => {
+		await certbot.installPlugin(certificate.meta.dns_provider);
+		const dnsPlugin = dnsPlugins[certificate.meta.dns_provider];
+		logger.info(`Requesting Let'sEncrypt certificates via ${dnsPlugin.name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
 
 		const credentialsLocation = '/etc/letsencrypt/credentials/credentials-' + certificate.id;
 		// Escape single quotes and backslashes
 		const escapedCredentials = certificate.meta.dns_provider_credentials.replaceAll('\'', '\\\'').replaceAll('\\', '\\\\');
 		const credentialsCmd     = 'mkdir -p /etc/letsencrypt/credentials 2> /dev/null; echo \'' + escapedCredentials + '\' > \'' + credentialsLocation + '\' && chmod 600 \'' + credentialsLocation + '\'';
-		// we call `. /opt/certbot/bin/activate` (`.` is alternative to `source` in dash) to access certbot venv
-		const prepareCmd = '. /opt/certbot/bin/activate && pip install --no-cache-dir ' + dns_plugin.package_name + (dns_plugin.version_requirement || '') + ' ' + dns_plugin.dependencies + ' && deactivate';
 
 		// Whether the plugin has a --<name>-credentials argument
 		const hasConfigArg = certificate.meta.dns_provider !== 'route53';
@@ -881,15 +876,15 @@ const internalCertificate = {
 			'--agree-tos ' +
 			'--email "' + certificate.meta.letsencrypt_email + '" ' +
 			'--domains "' + certificate.domain_names.join(',') + '" ' +
-			'--authenticator ' + dns_plugin.full_plugin_name + ' ' +
+			'--authenticator ' + dnsPlugin.full_plugin_name + ' ' +
 			(
 				hasConfigArg
-					? '--' + dns_plugin.full_plugin_name + '-credentials "' + credentialsLocation + '"'
+					? '--' + dnsPlugin.full_plugin_name + '-credentials "' + credentialsLocation + '"'
 					: ''
 			) +
 			(
 				certificate.meta.propagation_seconds !== undefined
-					? ' --' + dns_plugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds
+					? ' --' + dnsPlugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds
 					: ''
 			) +
 			(letsencryptStaging ? ' --staging' : '');
@@ -903,24 +898,19 @@ const internalCertificate = {
 			mainCmd = mainCmd + ' --dns-duckdns-no-txt-restore';
 		}
 
-		logger.info('Command:', `${credentialsCmd} && ${prepareCmd} && ${mainCmd}`);
+		logger.info('Command:', `${credentialsCmd} && && ${mainCmd}`);
 
-		return utils.exec(credentialsCmd)
-			.then(() => {
-				return utils.exec(prepareCmd)
-					.then(() => {
-						return utils.exec(mainCmd)
-							.then(async (result) => {
-								logger.info(result);
-								return result;
-							});
-					});
-			}).catch(async (err) => {
-				// Don't fail if file does not exist
-				const delete_credentialsCmd = `rm -f '${credentialsLocation}' || true`;
-				await utils.exec(delete_credentialsCmd);
-				throw err;
-			});
+		try {
+			await utils.exec(credentialsCmd);
+			const result = await utils.exec(mainCmd);
+			logger.info(result);
+			return result;
+		} catch (err) {
+			// Don't fail if file does not exist
+			const delete_credentialsCmd = `rm -f '${credentialsLocation}' || true`;
+			await utils.exec(delete_credentialsCmd);
+			throw err;
+		}
 	},
 
 
@@ -999,13 +989,13 @@ const internalCertificate = {
 	 * @returns {Promise}
 	 */
 	renewLetsEncryptSslWithDnsChallenge: (certificate) => {
-		const dns_plugin = dnsPlugins[certificate.meta.dns_provider];
+		const dnsPlugin = dnsPlugins[certificate.meta.dns_provider];
 
-		if (!dns_plugin) {
+		if (!dnsPlugin) {
 			throw Error(`Unknown DNS provider '${certificate.meta.dns_provider}'`);
 		}
 
-		logger.info(`Renewing Let'sEncrypt certificates via ${dns_plugin.display_name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
+		logger.info(`Renewing Let'sEncrypt certificates via ${dnsPlugin.name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`);
 
 		let mainCmd = certbotCommand + ' renew --force-renewal ' +
 			'--config "' + letsencryptConfig + '" ' +

backend/lib/certbot.js

@@ -0,0 +1,46 @@
+const dnsPlugins = require('../global/certbot-dns-plugins.json');
+const utils      = require('./utils');
+const error      = require('./error');
+const logger     = require('../logger').certbot;
+
+// const letsencryptStaging = config.useLetsencryptStaging();
+// const letsencryptConfig  = '/etc/letsencrypt.ini';
+// const certbotCommand     = 'certbot';
+
+// const acmeVersion  = '1.32.0';
+const CERTBOT_VERSION_REPLACEMENT = '$(certbot --version | grep -Eo \'[0-9](\\.[0-9]+)+\')';
+
+const certbot = {
+
+	/**
+	 * Installs a cerbot plugin given the key for the object from
+	 * ../global/certbot-dns-plugins.json
+	 *
+	 * @param   {string}  pluginKey
+	 * @returns {Object}
+	 */
+	installPlugin: async function (pluginKey) {
+		if (typeof dnsPlugins[pluginKey] === 'undefined') {
+			// throw Error(`Certbot plugin ${pluginKey} not found`);
+			throw new error.ItemNotFoundError(pluginKey);
+		}
+
+		const plugin = dnsPlugins[pluginKey];
+		logger.start(`Installing ${pluginKey}...`);
+
+		plugin.version      = plugin.version.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT);
+		plugin.dependencies = plugin.dependencies.replace(/{{certbot-version}}/g, CERTBOT_VERSION_REPLACEMENT);
+
+		const cmd = '. /opt/certbot/bin/activate && pip install --no-cache-dir ' + plugin.dependencies + ' ' + plugin.package_name + plugin.version + ' ' + ' && deactivate';
+		return utils.exec(cmd)
+			.then((result) => {
+				logger.complete(`Installed ${pluginKey}`);
+				return result;
+			})
+			.catch((err) => {
+				throw err;
+			});
+	},
+};
+
+module.exports = certbot;

backend/lib/error.js

@@ -82,7 +82,16 @@ module.exports = {
 		this.message  = message;
 		this.public   = false;
 		this.status   = 400;
-	}
+	},
+
+	CommandError: function (stdErr, code, previous) {
+		Error.captureStackTrace(this, this.constructor);
+		this.name     = this.constructor.name;
+		this.previous = previous;
+		this.message  = stdErr;
+		this.code     = code;
+		this.public   = false;
+	},
 };
 
 _.forEach(module.exports, function (error) {

backend/lib/utils.js

@@ -3,23 +3,27 @@ const exec       = require('child_process').exec;
 const execFile   = require('child_process').execFile;
 const { Liquid } = require('liquidjs');
 const logger     = require('../logger').global;
+const error      = require('./error');
 
 module.exports = {
 
-	/**
-	 * @param   {String} cmd
-	 * @returns {Promise}
-	 */
-	exec: function (cmd) {
-		return new Promise((resolve, reject) => {
-			exec(cmd, function (err, stdout, /*stderr*/) {
-				if (err && typeof err === 'object') {
-					reject(err);
+	exec: async function(cmd, options = {}) {
+		logger.debug('CMD:', cmd);
+
+		const { stdout, stderr } = await new Promise((resolve, reject) => {
+			const child = exec(cmd, options, (isError, stdout, stderr) => {
+				if (isError) {
+					reject(new error.CommandError(stderr, isError));
 				} else {
-					resolve(stdout.trim());
+					resolve({ stdout, stderr });
 				}
 			});
+
+			child.on('error', (e) => {
+				reject(new error.CommandError(stderr, 1, e));
+			});
 		});
+		return stdout;
 	},
 
 	/**
@@ -28,7 +32,8 @@ module.exports = {
 	 * @returns {Promise}
 	 */
 	execFile: function (cmd, args) {
-		logger.debug('CMD: ' + cmd + ' ' + (args ? args.join(' ') : ''));
+		// logger.debug('CMD: ' + cmd + ' ' + (args ? args.join(' ') : ''));
+
 		return new Promise((resolve, reject) => {
 			execFile(cmd, args, function (err, stdout, /*stderr*/) {
 				if (err && typeof err === 'object') {

backend/logger.js

@@ -7,6 +7,7 @@ module.exports = {
 	access:    new Signale({scope: 'Access   '}),
 	nginx:     new Signale({scope: 'Nginx    '}),
 	ssl:       new Signale({scope: 'SSL      '}),
+	certbot:   new Signale({scope: 'Certbot  '}),
 	import:    new Signale({scope: 'Importer '}),
 	setup:     new Signale({scope: 'Setup    '}),
 	ip_ranges: new Signale({scope: 'IP Ranges'})

backend/scripts/install-certbot-plugins

@@ -0,0 +1,49 @@
+#!/usr/bin/node
+
+// Usage:
+//   Install all plugins defined in `certbot-dns-plugins.json`:
+//    ./install-certbot-plugins
+//   Install one or more specific plugins:
+//    ./install-certbot-plugins route53 cloudflare
+//
+// Usage with a running docker container:
+//    docker exec npm_core /command/s6-setuidgid 1000:1000 bash -c "/app/scripts/install-certbot-plugins"
+//
+
+const dnsPlugins = require('../global/certbot-dns-plugins.json');
+const certbot    = require('../lib/certbot');
+const logger     = require('../logger').certbot;
+const batchflow  = require('batchflow');
+
+let hasErrors      = false;
+let failingPlugins = [];
+
+let pluginKeys = Object.keys(dnsPlugins);
+if (process.argv.length > 2) {
+	pluginKeys = process.argv.slice(2);
+}
+
+batchflow(pluginKeys).sequential()
+	.each((i, pluginKey, next) => {
+		certbot.installPlugin(pluginKey)
+			.then(() => {
+				next();
+			})
+			.catch((err) => {
+				hasErrors = true;
+				failingPlugins.push(pluginKey);
+				next(err);
+			});
+	})
+	.error((err) => {
+		logger.error(err.message);
+	})
+	.end(() => {
+		if (hasErrors) {
+			logger.error('Some plugins failed to install. Please check the logs above. Failing plugins: ' + '\n - ' + failingPlugins.join('\n - '));
+			process.exit(1);
+		} else {
+			logger.complete('Plugins installed successfully');
+			process.exit(0);
+		}
+	});