Supporting open-appsec

backend/internal/nginx-openappsec.js

@@ -0,0 +1,255 @@
+const _      = require('lodash');
+const fs     = require('fs');
+const logger = require('../logger').nginx;
+const config = require('../lib/config');
+const yaml   = require('js-yaml');
+const path   = require('path');
+const constants = require('../lib/constants');
+
+const internalNginxOpenappsec = {
+
+	// module constants
+	CONFIG_TEMPLATE_FILE_NAME: 'local-policy-open-appsec-enabled-for-proxy-host.yaml',
+	CONFIG_TEMPLATE_DIR: '/app/templates',
+
+	// module variables
+	config: null,
+	configTemplate: null,
+
+	/**
+	 * Generate an open-appsec config file for a proxy host.
+	 *
+	 * @param   {Object} access
+	 * @param   {Object} row
+	 * @param   {Object} data
+	 * @returns {Promise}
+	 */
+	generateConfig: (access, row, data) => {
+		return access.can('settings:update', row.id)
+			.then(() => {
+				if (config.debug()) {
+					logger.info('Generating openappsec config:', JSON.stringify(data, null, 2));
+				}
+
+				const openappsecMode = data.use_openappsec == false ? 'inactive' : data.openappsec_mode;
+
+				const configTemplateFilePath = path.join(internalNginxOpenappsec.CONFIG_TEMPLATE_DIR, internalNginxOpenappsec.CONFIG_TEMPLATE_FILE_NAME)
+				const configFilePath = path.join(constants.APPSEC_EXT_DIR, constants.APPSEC_CONFIG_FILE_NAME);
+
+				let openappsecConfig = yaml.load(fs.readFileSync(configFilePath, 'utf8'));
+				let openappsecConfigTemplate = yaml.load(fs.readFileSync(configTemplateFilePath, 'utf8'));
+
+				internalNginxOpenappsec.config = openappsecConfig;
+				internalNginxOpenappsec.configTemplate = openappsecConfigTemplate;
+
+				const specificRuleName = 'npm-managed-specific-rule-proxyhost-' + row.id;
+				const logTriggerName = 'npm-managed-log-trigger-proxyhost-' + row.id;
+				const practiceName = 'npm-managed-practice-proxyhost-' + row.id;
+				
+				_.remove(openappsecConfig.policies['specific-rules'], rule => rule.name === specificRuleName || rule.name.startsWith(`${specificRuleName}.`));
+
+				data.domain_names.forEach((domain, index) => {
+					let ruleName = index > 0 ? `${specificRuleName}.${index}` : specificRuleName;
+					let specificRuleNode = {
+						host: domain,
+						name: ruleName,
+						triggers: [logTriggerName],
+						mode: openappsecMode,
+						practices: [practiceName]
+					};
+					internalNginxOpenappsec.updateNode('policies', 'specific-rules', specificRuleNode, openappsecMode);
+				});
+
+				internalNginxOpenappsec.updateNode('', 'practices', { name: practiceName, 'web-attacks.override-mode': openappsecMode, 'web-attacks.minimum-confidence': data.minimum_confidence }, openappsecMode);
+				internalNginxOpenappsec.updateNode('', 'log-triggers', { name: logTriggerName }, openappsecMode);
+
+				// remove all openappsec managed location config nodes for a proxy host.			
+				let pattern = new RegExp(`^npm-managed.*-${row.id}-.*`);
+				internalNginxOpenappsec.removeMatchingNodes(openappsecConfig, pattern);
+
+				// for each data.location, create location config nodes
+				data.locations.forEach((location, index) => {
+					let locationSpecificRuleName = 'npm-managed-specific-rule-proxyhost-' + row.id + '-' + index;
+					let locationLogTriggerName = 'npm-managed-log-trigger-proxyhost-' + row.id + '-' + index;
+					let locationPracticeName = 'npm-managed-practice-proxyhost-' + row.id + '-' + index;
+
+					let locationOpenappsecMode = location.use_openappsec == false ? 'inactive' : location.openappsec_mode;
+
+					_.remove(openappsecConfig.policies['specific-rules'], rule => rule.name === locationSpecificRuleName || rule.name.startsWith(`${locationSpecificRuleName}.`));
+
+					data.domain_names.forEach((domain, index) => {
+						let locationUrl = domain + location.path;
+						let ruleName = index > 0 ? `${locationSpecificRuleName}.${index}` : locationSpecificRuleName;
+
+						let domainSpecificRuleNode = {
+							host: locationUrl,
+							name: ruleName,
+							triggers: [locationLogTriggerName],
+							mode: locationOpenappsecMode,
+							practices: [locationPracticeName]
+						};
+						internalNginxOpenappsec.updateNode('policies', 'specific-rules', domainSpecificRuleNode, locationOpenappsecMode, 'location', openappsecMode);
+					});
+
+					internalNginxOpenappsec.updateNode('', 'practices', { name: locationPracticeName, 'web-attacks.override-mode': locationOpenappsecMode, 'web-attacks.minimum-confidence': location.minimum_confidence }, locationOpenappsecMode, 'location', openappsecMode);
+					internalNginxOpenappsec.updateNode('', 'log-triggers', { name: locationLogTriggerName }, locationOpenappsecMode, 'location', openappsecMode);
+				});
+
+				fs.writeFileSync(configFilePath, yaml.dump(openappsecConfig));
+			},
+				(err) => {
+					logger.error('Error generating openappsec config:', err);
+					return Promise.reject(err);
+				});
+	},
+
+	/** 
+	 * Remove all openappsec managed config nodes for a proxy host.
+	 *  
+	 * @param {Object} access
+	 * @param {Object} row
+	 * @returns {Promise}
+	 * 
+	 */
+	deleteConfig: (access, row) => {
+		return access.can('settings:update', row.id)
+			.then(() => {
+				const configFilePath = path.join(constants.APPSEC_EXT_DIR, constants.APPSEC_CONFIG_FILE_NAME);
+				let openappsecConfig = yaml.load(fs.readFileSync(configFilePath, 'utf8'));
+
+				// remove all openappsec managed location config nodes for a proxy host.			
+				let pattern = new RegExp(`^npm-managed.*-${row.id}`);
+				internalNginxOpenappsec.removeMatchingNodes(openappsecConfig, pattern);
+				fs.writeFileSync(configFilePath, yaml.dump(openappsecConfig));
+			})
+			.catch(err => {
+				logger.error('Error deleting openappsec config:', err);
+				return Promise.reject(err);
+			});
+	},
+	
+	/**
+	 * Update a node in the openappsec config.
+	 * 		- if the node does not exist, create it.
+	 * 		- if the node exists, update it.
+	 * 		- if openappsecMode is 'inactive', delete the node.
+	 * 
+	 * @param {String} parentNodePath - path to the parent node. e.g. 'policies'.
+	 * @param {String} nodeName - name of the node. e.g. 'specific-rules', 'practices', 'log-triggers'.
+	 * @param {Object} nodeItemProperties
+	 * @param {String} openappsecMode
+	 * @param {String} nodeType - 'host' or 'location'
+	 * @param {String} hostAppsecMode - to check if the host of a location is inactive.
+	 */
+	updateNode: function (parentNodePath, nodeName, nodeItemProperties, openappsecMode, nodeType = 'host', hostAppsecMode = '') {
+		// if no parent node path is specified, use the root of the config object.
+		const parent = parentNodePath ? _.get(this.config, parentNodePath, this.config) : this.config;
+
+		if (!parent) {
+			console.log('parent is not defined');
+			return;
+		}
+
+		let nodeItems = _.find(parent[nodeName], { name: nodeItemProperties.name });
+		if (openappsecMode == 'inactive' && nodeItems) {
+			_.remove(parent[nodeName], { name: nodeItemProperties.name });
+		}
+
+		if (openappsecMode !== 'inactive' || nodeType === 'location' && hostAppsecMode !== 'inactive') {
+			if (!nodeItems) {
+				// create the node from the template if it does not exist.
+				let templateSearchPath = parentNodePath ? `${parentNodePath}.${nodeName}[0]` : `${nodeName}[0]`;
+				nodeItems = _.cloneDeep(_.get(this.configTemplate, templateSearchPath));
+
+				// update the node with the nodeItemProperties. if the nodeType is 'location' and the openappsecMode is 'inactive', only update the name, host, and the (inactive) mode.
+				if (nodeType === 'location' && openappsecMode === 'inactive') {
+					nodeItemProperties = _.pick(nodeItemProperties, ['name', 'host', 'triggers', 'practices', 'mode', 'web-attacks.override-mode']);
+				}
+
+				Object.keys(nodeItemProperties).forEach(key => {
+					_.set(nodeItems, key, nodeItemProperties[key]);
+				});
+				parent[nodeName] = parent[nodeName] || [];
+				parent[nodeName].push(nodeItems);
+			} else {
+				// update the node if it exists.
+				Object.keys(nodeItemProperties).forEach(key => {
+					_.set(nodeItems, key, nodeItemProperties[key]);
+				});
+			}
+		}
+	},
+
+	/**
+	 * Recursively removes nodes from a JavaScript object based on a pattern.
+	*
+	* @param {Object|Array} obj - The object or array to remove nodes from.
+	* @param {RegExp} pattern - The pattern to match against node names.
+	*/
+	removeMatchingNodes: function (obj, pattern) {
+		_.forEach(obj, (value, key) => {
+			if (_.isPlainObject(value)) {
+				if (pattern.test(key)) {
+					delete obj[key];
+				} else {
+					this.removeMatchingNodes(value, pattern);
+				}
+			} else if (_.isArray(value)) {
+				_.remove(value, function (item) {
+					return _.isPlainObject(item) && pattern.test(item.name);
+				});
+				value.forEach(item => {
+					if (_.isPlainObject(item)) {
+						this.removeMatchingNodes(item, pattern);
+					}
+				});
+			}
+		});
+	},
+
+	/**
+	 * Get the openappsec mode, use_openappsec and minimum_confidence for a proxy host.
+	 * 
+	 * @param {Object} openappsecConfig - openappsec config object
+	 * @param {Number} rowId - proxy host id
+	 * @returns {Object} { mode, use_openappsec, minimum_confidence }
+	 */
+	getOpenappsecFields: (openappsecConfig, rowId) => {
+		const specificRuleName = 'npm-managed-specific-rule-proxyhost-' + rowId;
+
+		const specificRule = _.find(openappsecConfig?.policies['specific-rules'], { name: specificRuleName });
+		const mode = specificRule?.mode || 'inactive';
+		const use_openappsec = mode !== 'inactive' && mode !== undefined;
+
+		const practiceName = 'npm-managed-practice-proxyhost-' + rowId;
+		const practice = _.find(openappsecConfig?.practices, { name: practiceName });
+		const minimum_confidence = practice?.['web-attacks']['minimum-confidence'] || 'high';
+
+		return { mode, use_openappsec, minimum_confidence };
+	},
+
+	/**
+	 * get the openappsec config file path.
+	 */
+	getConfigFilePath: () => {
+		const configFilePath = path.join(constants.APPSEC_EXT_DIR, constants.APPSEC_CONFIG_FILE_NAME);
+		return configFilePath;
+	},
+
+	/**
+	 * A simple wrapper around unlinkSync that writes to the logger
+	 *
+	 * @param   {String}  filename
+	 */
+	deleteFile: (filename) => {
+		logger.debug('Deleting file: ' + filename);
+		try {
+			fs.unlinkSync(filename);
+		} catch (err) {
+			logger.debug('Could not delete file:', JSON.stringify(err, null, 2));
+		}
+	}
+
+};
+
+module.exports = internalNginxOpenappsec;

backend/internal/openappsec-log.js

@@ -0,0 +1,89 @@
+const fs = require('fs');
+const path = require('path');
+const util = require('util');
+const error = require('../lib/error');
+const { APPSEC_LOG_DIR } = require('../lib/constants');
+
+const internalOpenappsecLog = {
+
+	/**
+	 * All logs
+	 *
+	 * @param   {Access}  access
+	 * @param   {Array}   [expand]
+	 * @param   {String}  [search_query]
+	 * @returns {Promise}
+	 */
+	getAll: (access, expand, search_query) => {
+		return access.can('auditlog:list')
+			.then(() => {
+
+				const directoryPath = APPSEC_LOG_DIR;
+			
+				const readdir = util.promisify(fs.readdir);
+				const readFile = util.promisify(fs.readFile);
+
+				async function listLogFiles(dir) {
+					const files = await readdir(dir);
+					const logFiles = files.filter(file => path.extname(file).startsWith('.log'));
+
+					const sortedLogFiles = logFiles.sort((a, b) => {
+						const baseA = path.basename(a, path.extname(a));
+						const baseB = path.basename(b, path.extname(b));
+
+						if (baseA < baseB) return -1;
+						if (baseA > baseB) return 1;
+
+						return path.extname(a).localeCompare(path.extname(b));
+					});
+
+					const groupedFiles = sortedLogFiles.reduce((groups, file) => {
+						const fileName = path.basename(file, path.extname(file));
+						if (!groups[fileName]) {
+							groups[fileName] = [];
+						}
+						groups[fileName].push(file);
+						return groups;
+					}, {});
+
+					const wrappedObjects = [];
+
+					for (const [groupName, files] of Object.entries(groupedFiles)) {
+						for (const file of files) {
+							try {
+								const content = await readFile(path.join(dir, file), 'utf8');
+								const lines = content.split('\n');
+								for (const line of lines) {
+									try {
+										const json = JSON.parse(line);
+										const wrappedObject = {
+											source: groupName,
+											meta: json,
+											serviceName: json.eventSource.serviceName,
+											eventPriority: json.eventPriority,
+											eventSeverity: json.eventSeverity,
+											eventLevel: json.eventLevel,
+											eventTime: json.eventTime,
+              								eventName: json.eventName
+										};
+										wrappedObjects.push(wrappedObject);
+									} catch (err) {
+										// Ignore lines that don't contain JSON data
+									}
+								}
+							} catch (err) {
+								console.error(`Failed to read file ${file}: ${err.message}`);
+							}
+						}
+					}
+					wrappedObjects.sort((a, b) => new Date(b.eventTime) - new Date(a.eventTime));
+					return wrappedObjects;
+				}
+
+				let groupedFiles = listLogFiles(directoryPath).catch(console.error);
+				return groupedFiles;
+			});
+	}
+};
+
+module.exports = internalOpenappsecLog;

backend/internal/setting-openappsec.js

@@ -0,0 +1,52 @@
+const fs            = require('fs');
+const error         = require('../lib/error');
+const path          = require('path');
+
+const constants = require('../lib/constants');
+
+const internalOpenappsecSetting = {
+	configFilePath: path.join(constants.APPSEC_EXT_DIR, constants.APPSEC_CONFIG_FILE_NAME),
+
+	/**
+	 * @param  {Access}   access
+	 * @return {Promise}
+	 */
+	getLocalPolicy: (access) => {
+		return access.can('settings:list')
+			.then(() => {
+				try {
+					const filePath = internalOpenappsecSetting.configFilePath
+					if (!fs.existsSync(filePath)) {
+						return;
+					}
+					const fileContent = fs.readFileSync(filePath, 'utf8');
+					const jsonStr = JSON.stringify(fileContent);
+					return jsonStr;
+
+				} catch (err) {
+					console.error(err);
+				}
+			});
+	},
+
+	/**
+	 * @param  {Access}   access
+	 * @param  {Object}   data
+	 * @return {Promise}
+	 */
+	updateLocalPolicy: (access, data) => {
+		return access.can('settings:list')
+			.then(() => {
+				const filePath = internalOpenappsecSetting.configFilePath
+				const yamlStr = data.local_policy;
+				fs.writeFileSync(filePath, yamlStr, {encoding: 'utf8'});
+				return true;
+			})
+			.catch((err) => {
+				console.error(err);
+				throw new error.ConfigurationError(err.message);
+			});
+	}
+};
+
+module.exports = internalOpenappsecSetting;

backend/lib/constants.js

@@ -0,0 +1,5 @@
+module.exports = {
+	APPSEC_CONFIG_FILE_NAME: 'local_policy.yaml',
+	APPSEC_EXT_DIR: '/ext/appsec',
+	APPSEC_LOG_DIR: '/ext/appsec-logs',
+};

backend/routes/api/openappsec-log.js

@@ -0,0 +1,35 @@
+const express          = require('express');
+const jwtdecode        = require('../../lib/express/jwt-decode');
+const internalOpenappsecLog = require('../../internal/openappsec-log');
+
+let router = express.Router({
+	caseSensitive: true,
+	strict:        true,
+	mergeParams:   true
+});
+
+/**
+ * /api/openappsec-log
+ */
+router
+	.route('/')
+	.options((req, res) => {
+		res.sendStatus(204);
+	})
+	.all(jwtdecode())
+
+	/**
+	 * GET /api/openappsec-log
+	 *
+	 * Retrieve all logs
+	 */
+	.get((req, res, next) => {
+		return internalOpenappsecLog.getAll(res.locals.access)
+			.then((policy) => {
+				res.status(200)
+					.send(policy);
+			})
+			.catch(next);
+	});
+
+module.exports = router;

backend/routes/api/openappsec-settings.js

@@ -0,0 +1,49 @@
+const express         = require('express');
+const jwtdecode       = require('../../lib/express/jwt-decode');
+const internalOpenappsecSetting = require('../../internal/setting-openappsec');
+
+let router = express.Router({
+	caseSensitive: true,
+	strict:        true,
+	mergeParams:   true
+});
+
+/**
+ * /api/openappsec-settings
+ */
+router
+	.route('/')
+	.options((req, res) => {
+		res.sendStatus(204);
+	})
+	.all(jwtdecode())
+
+	/**
+	 * GET /api/openappsec-settings
+	 * 
+	 * Retrieve the open-appsec local policy.
+	 */
+	.get((req, res, next) => {
+		return internalOpenappsecSetting.getLocalPolicy(res.locals.access)
+			.then((policy) => {
+				res.status(200)
+					.send(policy);
+			})
+			.catch(next);
+	})
+
+	/**
+	 * PUT /api/openappsec-settings
+	 *
+	 * Update the open-appsec local policy.
+	 */
+	.put((req, res, next) => {
+		return internalOpenappsecSetting.updateLocalPolicy(res.locals.access, req.body)
+			.then((result) => {
+				res.status(200)
+					.send(result);
+			})
+			.catch(next);
+	});
+
+module.exports = router;

backend/templates/local-policy-open-appsec-enabled-for-proxy-host.yaml

@@ -0,0 +1,121 @@
+# This example is for NPM Proxy Host with open-appsec enabled,
+# Enforcement Mode set to Prevent/Learn, hostname web.server.com/example
+
+policies:
+  default:
+    triggers:
+    - appsec-default-log-trigger
+    mode: inactive
+    practices:
+    - webapp-default-practice
+    custom-response: appsec-default-web-user-response
+  specific-rules:
+  - host: web.server.com/example
+    # as set in "Edit Proxy Host" in "Domain Names" field
+    # IMPORTANT LIMITATION: Currently open-appsec declarative with CRD version 1.0 only supports single host entry per specific rule
+    # This will be resolved with new CRDs 2.0
+    name: npm-managed-specific-rule-proxyhost-1
+    # This “name” key will be the actual reference to a specific Reverse Proxy object defined in NPM
+    triggers:
+    - npm-managed-log-trigger-proxyhost-1
+    mode: prevent-learn 
+    practices:
+    - npm-managed-practice-proxyhost-1
+
+practices:
+  - name: webapp-default-practice
+    web-attacks:
+      max-body-size-kb: 1000000
+      max-header-size-bytes: 102400
+      max-object-depth: 40
+      max-url-size-bytes: 32768
+      minimum-confidence: high
+      override-mode: inactive
+      protections:
+        csrf-protection: inactive
+        error-disclosure: inactive
+        non-valid-http-methods: false
+        open-redirect: inactive
+    anti-bot:
+      injected-URIs: []
+      validated-URIs: []
+      override-mode: inactive
+    snort-signatures:
+      configmap: []
+      override-mode: inactive
+    openapi-schema-validation:
+      configmap: []
+      override-mode: inactive
+
+  - name: npm-managed-practice-proxyhost-1
+    web-attacks:
+      max-body-size-kb: 1000000
+      max-header-size-bytes: 102400
+      max-object-depth: 40
+      max-url-size-bytes: 32768
+      minimum-confidence: high
+      override-mode: inactive
+      protections:
+        csrf-protection: inactive
+        error-disclosure: inactive
+        non-valid-http-methods: false
+        open-redirect: inactive
+    anti-bot:
+      injected-URIs: []
+      validated-URIs: []
+      override-mode: inactive
+    snort-signatures:
+      configmap: []
+      override-mode: inactive
+    openapi-schema-validation:
+      configmap: []
+      override-mode: inactive
+
+log-triggers:
+  - name: appsec-default-log-trigger
+    access-control-logging:
+      allow-events: false
+      drop-events: true
+    additional-suspicious-events-logging:
+      enabled: true
+      minimum-severity: high
+      response-body: false
+    appsec-logging:
+      all-web-requests: false
+      detect-events: true
+      prevent-events: true
+    extended-logging:
+      http-headers: false
+      request-body: false
+      url-path: false
+      url-query: false
+    log-destination:
+      cloud: false
+      stdout:
+        format: json
+  - name: npm-managed-log-trigger-proxyhost-1
+    access-control-logging:
+      allow-events: false
+      drop-events: true
+    additional-suspicious-events-logging:
+      enabled: true
+      minimum-severity: high
+      response-body: false
+    appsec-logging:
+      all-web-requests: false
+      detect-events: true
+      prevent-events: true
+    extended-logging:
+      http-headers: false
+      request-body: false
+      url-path: false
+      url-query: false
+    log-destination:
+      cloud: false
+      stdout:
+        format: json
+
+custom-responses:
+  - name: appsec-default-web-user-response
+    mode: response-code-only
+    http-response-code: 403

backend/templates/openappsec.conf

@@ -0,0 +1,16 @@
+policies:
+  default:
+    triggers:
+    - appsec-default-log-trigger
+    mode: inactive
+    practices:
+    - webapp-default-practice
+    custom-response: appsec-default-web-user-response
+  specific-rules:
+  - host: {{ domain_names.first }}
+    triggers:
+    - appsec-default-log-trigger
+    mode: {{openappsec_mode}}
+    practices:
+    - webapp-default-practice
+    custom-response: appsec-default-web-user-response