From 351ba8dacd55f70bf0797212bfa4f9650b92a8b1 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Mon, 14 Oct 2024 22:48:36 +1000 Subject: [PATCH 01/10] More tests for certificates, fixed schema problems --- backend/internal/certificate.js | 11 +++-- .../schema/components/certificate-object.json | 28 ++++++++--- .../certificates/certID/upload/post.json | 19 ++++++++ test/cypress/e2e/api/Certificates.cy.js | 47 +++++++++++++++++++ test/package.json | 2 +- test/yarn.lock | 8 ++-- 6 files changed, 98 insertions(+), 17 deletions(-) diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index 9bdfe695..68dd55e8 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -3,27 +3,28 @@ const fs = require('fs'); const https = require('https'); const tempWrite = require('temp-write'); const moment = require('moment'); +const archiver = require('archiver'); +const path = require('path'); +const { isArray } = require('lodash'); const logger = require('../logger').ssl; const config = require('../lib/config'); const error = require('../lib/error'); const utils = require('../lib/utils'); +const certbot = require('../lib/certbot'); const certificateModel = require('../models/certificate'); const tokenModel = require('../models/token'); const dnsPlugins = require('../global/certbot-dns-plugins.json'); const internalAuditLog = require('./audit-log'); const internalNginx = require('./nginx'); const internalHost = require('./host'); -const certbot = require('../lib/certbot'); -const archiver = require('archiver'); -const path = require('path'); -const { isArray } = require('lodash'); + const letsencryptStaging = config.useLetsencryptStaging(); const letsencryptConfig = '/etc/letsencrypt.ini'; const certbotCommand = 'certbot'; function omissions() { - return ['is_deleted']; + return ['is_deleted', 'owner.is_deleted']; } const internalCertificate = { diff --git a/backend/schema/components/certificate-object.json b/backend/schema/components/certificate-object.json index 04cd8980..a4e8e1fc 100644 --- a/backend/schema/components/certificate-object.json +++ b/backend/schema/components/certificate-object.json @@ -24,13 +24,23 @@ "description": "Nice Name for the custom certificate" }, "domain_names": { - "$ref": "../common.json#/properties/domain_names" + "description": "Domain Names separated by a comma", + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[^&| @!#%^();:/\\\\}{=+?<>,~`'\"]+$" + } }, "expires_on": { "description": "Date and time of expiration", "readOnly": true, "type": "string" }, + "owner": { + "$ref": "./user-object.json" + }, "meta": { "type": "object", "additionalProperties": false, @@ -51,12 +61,16 @@ "type": "string" }, "propagation_seconds": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - } - ] + "type": "integer", + "minimum": 0 + }, + "certificate": { + "type": "string", + "minLength": 1 + }, + "certificate_key": { + "type": "string", + "minLength": 1 } } } diff --git a/backend/schema/paths/nginx/certificates/certID/upload/post.json b/backend/schema/paths/nginx/certificates/certID/upload/post.json index e9274856..f38b8102 100644 --- a/backend/schema/paths/nginx/certificates/certID/upload/post.json +++ b/backend/schema/paths/nginx/certificates/certID/upload/post.json @@ -55,6 +55,25 @@ "certificate_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1n9j9C5Bes1nd\nqACDckERauxXVNKCnUlUM1buGBx1xc+j2e2Ar23wUJJuWBY18VfT8yqfqVDktO2w\nrbmvZvLuPmXePOKbIKS+XXh+2NG9L5bDG9rwGFCRXnbQj+GWCdMfzx14+CR1IHge\nYz6Cv/Si2/LJPCh/CoBfM4hUQJON3lxAWrWBpdbZnKYMrxuPBRfW9OuzTbCVXToQ\noxRAHiOR9081Xn1WeoKr7kVBIa5UphlvWXa12w1YmUwJu7YndnJGIavLWeNCVc7Z\nEo+nS8Wr/4QWicatIWZXpVaEOPhRoeplQDxNWg5b/Q26rYoVd7PrCmRs7sVcH79X\nzGONeH1PAgMBAAECggEAANb3Wtwl07pCjRrMvc7WbC0xYIn82yu8/g2qtjkYUJcU\nia5lQbYN7RGCS85Oc/tkq48xQEG5JQWNH8b918jDEMTrFab0aUEyYcru1q9L8PL6\nYHaNgZSrMrDcHcS8h0QOXNRJT5jeGkiHJaTR0irvB526tqF3knbK9yW22KTfycUe\na0Z9voKn5xRk1DCbHi/nk2EpT7xnjeQeLFaTIRXbS68omkr4YGhwWm5OizoyEGZu\nW0Zum5BkQyMr6kor3wdxOTG97ske2rcyvvHi+ErnwL0xBv0qY0Dhe8DpuXpDezqw\no72yY8h31Fu84i7sAj24YuE5Df8DozItFXQpkgbQ6QKBgQDPrufhvIFm2S/MzBdW\nH8JxY7CJlJPyxOvc1NIl9RczQGAQR90kx52cgIcuIGEG6/wJ/xnGfMmW40F0DnQ+\nN+oLgB9SFxeLkRb7s9Z/8N3uIN8JJFYcerEOiRQeN2BXEEWJ7bUThNtsVrAcKoUh\nELsDmnHW/3V+GKwhd0vpk842+wKBgQDf4PGLG9PTE5tlAoyHFodJRd2RhTJQkwsU\nMDNjLJ+KecLv+Nl+QiJhoflG1ccqtSFlBSCG067CDQ5LV0xm3mLJ7pfJoMgjcq31\nqjEmX4Ls91GuVOPtbwst3yFKjsHaSoKB5fBvWRcKFpBUezM7Qcw2JP3+dQT+bQIq\ncMTkRWDSvQKBgQDOdCQFDjxg/lR7NQOZ1PaZe61aBz5P3pxNqa7ClvMaOsuEQ7w9\nvMYcdtRq8TsjA2JImbSI0TIg8gb2FQxPcYwTJKl+FICOeIwtaSg5hTtJZpnxX5LO\nutTaC0DZjNkTk5RdOdWA8tihyUdGqKoxJY2TVmwGe2rUEDjFB++J4inkEwKBgB6V\ng0nmtkxanFrzOzFlMXwgEEHF+Xaqb9QFNa/xs6XeNnREAapO7JV75Cr6H2hFMFe1\nmJjyqCgYUoCWX3iaHtLJRnEkBtNY4kzyQB6m46LtsnnnXO/dwKA2oDyoPfFNRoDq\nYatEd3JIXNU9s2T/+x7WdOBjKhh72dTkbPFmTPDdAoGAU6rlPBevqOFdObYxdPq8\nEQWu44xqky3Mf5sBpOwtu6rqCYuziLiN7K4sjN5GD5mb1cEU+oS92ZiNcUQ7MFXk\n8yTYZ7U0VcXyAcpYreWwE8thmb0BohJBr+Mp3wLTx32x0HKdO6vpUa0d35LUTUmM\nRrKmPK/msHKK/sVHiL+NFqo=\n-----END PRIVATE KEY-----\n" } } + }, + "schema": { + "type": "object", + "additionalProperties": false, + "required": ["certificate", "certificate_key"], + "properties": { + "certificate": { + "type": "string", + "minLength": 1 + }, + "certificate_key": { + "type": "string", + "minLength": 1 + }, + "intermediate_certificate": { + "type": "string", + "minLength": 1 + } + } } } } diff --git a/test/cypress/e2e/api/Certificates.cy.js b/test/cypress/e2e/api/Certificates.cy.js index 043680f3..687d09e5 100644 --- a/test/cypress/e2e/api/Certificates.cy.js +++ b/test/cypress/e2e/api/Certificates.cy.js @@ -2,6 +2,7 @@ describe('Certificates endpoints', () => { let token; + let certID; before(() => { cy.getToken().then((tok) => { @@ -24,6 +25,52 @@ describe('Certificates endpoints', () => { }); }); + it('Custom certificate lifecycle', function() { + cy.task('backendApiPost', { + token: token, + path: '/api/nginx/certificates', + data: { + provider: "other", + nice_name: "Test Certificate", + }, + }).then((data) => { + cy.validateSwaggerSchema('post', 201, '/nginx/certificates', data); + expect(data).to.have.property('id'); + certID = data.id; + + cy.task('backendApiPostFiles', { + token: token, + path: `/api/nginx/certificates/${certID}/upload`, + files: { + certificate: 'test.example.com.pem', + certificate_key: 'test.example.com-key.pem', + }, + }).then((data) => { + cy.validateSwaggerSchema('post', 201, '/nginx/certificates/upload', data); + expect(data).to.have.property('certificate'); + expect(data).to.have.property('certificate_key'); + + cy.task('backendApiDelete', { + token: token, + path: `/api/nginx/certificates/${certID}` + }).then((data) => { + cy.validateSwaggerSchema('delete', 200, '/nginx/certificates/{certID}', data); + expect(data).to.be.equal(true); + }); + }); + }); + }); + + it('Should be able to get all certs', function() { + cy.task('backendApiGet', { + token: token, + path: '/api/nginx/certificates?expand=owner' + }).then((data) => { + cy.validateSwaggerSchema('get', 200, '/nginx/certificates', data); + expect(data.length).to.be.greaterThan(0); + }); + }); + it('Request Certificate - CVE-2024-46256/CVE-2024-46257', function() { cy.task('backendApiPost', { token: token, diff --git a/test/package.json b/test/package.json index 99215363..a1c68908 100644 --- a/test/package.json +++ b/test/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "dependencies": { - "@jc21/cypress-swagger-validation": "^0.2.7", + "@jc21/cypress-swagger-validation": "^0.2.8", "axios": "^1.7.7", "cypress": "^13.15.0", "cypress-multi-reporters": "^1.6.4", diff --git a/test/yarn.lock b/test/yarn.lock index b12b4247..38d85e0a 100644 --- a/test/yarn.lock +++ b/test/yarn.lock @@ -167,10 +167,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.3.1.tgz#c72a5c76a9fbaf3488e231b13dc52c0da7bab42a" integrity sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA== -"@jc21/cypress-swagger-validation@^0.2.7": - version "0.2.7" - resolved "https://registry.yarnpkg.com/@jc21/cypress-swagger-validation/-/cypress-swagger-validation-0.2.7.tgz#64642b12d98b884df8c30b72852162941285d2af" - integrity sha512-4EQ0gfigRwVVl3DnVYbR48/EKGnn7oH5YYdMzf6zqypO+bqYvDHu9kgk/WqkGlT/aauGQ7e0YGMo8ZvR7mL0Ng== +"@jc21/cypress-swagger-validation@^0.2.8": + version "0.2.8" + resolved "https://registry.yarnpkg.com/@jc21/cypress-swagger-validation/-/cypress-swagger-validation-0.2.8.tgz#8ab059bd41e3ee100a1998a1484b9e5a2e9a4224" + integrity sha512-9fiZIHj3//bJjC5YUMOc42RnoEUeeokVn6xtMnP52XIZ/ryWQ9PIyFdlOAH8q/LW/uPxozJo2+hdB6ou4iurag== dependencies: "@apidevtools/swagger-parser" "^10.1.0" ajv "^8.17.1" From f48e1b46a8979b58d05c953f0893f140717c5fb1 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Tue, 15 Oct 2024 23:54:39 +1000 Subject: [PATCH 02/10] Updated swagger cypress package, which works with proxies --- Jenkinsfile | 10 +++++++ docker/docker-compose.dev.yml | 4 +-- test/cypress/config/ci.js | 2 +- test/cypress/config/dev.js | 22 --------------- test/cypress/e2e/api/Certificates.cy.js | 34 ++++++++++++----------- test/cypress/e2e/api/Health.cy.js | 4 +-- test/cypress/e2e/api/Hosts.cy.js | 2 +- test/cypress/e2e/api/Users.cy.js | 2 +- test/cypress/plugins/backendApi/client.js | 2 +- test/cypress/plugins/index.js | 2 +- test/package.json | 6 ++-- test/yarn.lock | 31 ++++++++------------- 12 files changed, 52 insertions(+), 69 deletions(-) delete mode 100644 test/cypress/config/dev.js diff --git a/Jenkinsfile b/Jenkinsfile index 75757946..9b29ee97 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -127,6 +127,11 @@ pipeline { junit 'test/results/junit/*' sh 'docker-compose down --remove-orphans --volumes -t 30 || true' } + unstable { + dir(path: 'testing/results') { + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/*', excludes: '**/*.xml') + } + } } } stage('Test Mysql') { @@ -155,6 +160,11 @@ pipeline { junit 'test/results/junit/*' sh 'docker-compose down --remove-orphans --volumes -t 30 || true' } + unstable { + dir(path: 'testing/results') { + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/*', excludes: '**/*.xml') + } + } } } stage('MultiArch Build') { diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 4cdb1e91..e7f9bcba 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -1,7 +1,7 @@ # WARNING: This is a DEVELOPMENT docker-compose file, it should not be used for production. services: - npm: + fullstack: image: nginxproxymanager:dev container_name: npm_core build: @@ -67,7 +67,7 @@ services: URL: "http://npm:81/api/schema" PORT: '80' depends_on: - - npm + - fullstack squid: image: ubuntu/squid diff --git a/test/cypress/config/ci.js b/test/cypress/config/ci.js index 873330dc..dc968dbd 100644 --- a/test/cypress/config/ci.js +++ b/test/cypress/config/ci.js @@ -15,7 +15,7 @@ module.exports = defineConfig({ return require("../plugins/index.js")(on, config); }, env: { - swaggerBase: 'http://fullstack:81/api/schema', + swaggerBase: '{{baseUrl}}/api/schema?ts=' + Date.now(), }, baseUrl: 'http://fullstack:81', } diff --git a/test/cypress/config/dev.js b/test/cypress/config/dev.js deleted file mode 100644 index f3c9f6d2..00000000 --- a/test/cypress/config/dev.js +++ /dev/null @@ -1,22 +0,0 @@ -const { defineConfig } = require('cypress'); - -module.exports = defineConfig({ - requestTimeout: 30000, - defaultCommandTimeout: 20000, - reporter: 'cypress-multi-reporters', - reporterOptions: { - configFile: 'multi-reporter.json' - }, - video: false, - videosFolder: 'results/videos', - screenshotsFolder: 'results/screenshots', - e2e: { - setupNodeEvents(on, config) { - return require("../plugins/index.js")(on, config); - }, - env: { - swaggerBase: 'http://npm:81/api/schema', - }, - baseUrl: 'http://npm:81', - } -}); diff --git a/test/cypress/e2e/api/Certificates.cy.js b/test/cypress/e2e/api/Certificates.cy.js index 687d09e5..1e8a6fed 100644 --- a/test/cypress/e2e/api/Certificates.cy.js +++ b/test/cypress/e2e/api/Certificates.cy.js @@ -1,4 +1,4 @@ -/// +/// describe('Certificates endpoints', () => { let token; @@ -26,6 +26,7 @@ describe('Certificates endpoints', () => { }); it('Custom certificate lifecycle', function() { + // Create custom cert cy.task('backendApiPost', { token: token, path: '/api/nginx/certificates', @@ -38,6 +39,7 @@ describe('Certificates endpoints', () => { expect(data).to.have.property('id'); certID = data.id; + // Upload files cy.task('backendApiPostFiles', { token: token, path: `/api/nginx/certificates/${certID}/upload`, @@ -46,31 +48,31 @@ describe('Certificates endpoints', () => { certificate_key: 'test.example.com-key.pem', }, }).then((data) => { - cy.validateSwaggerSchema('post', 201, '/nginx/certificates/upload', data); + cy.validateSwaggerSchema('post', 200, '/nginx/certificates/{certID}/upload', data); expect(data).to.have.property('certificate'); expect(data).to.have.property('certificate_key'); - cy.task('backendApiDelete', { + // Get all certs + cy.task('backendApiGet', { token: token, - path: `/api/nginx/certificates/${certID}` + path: '/api/nginx/certificates?expand=owner' }).then((data) => { - cy.validateSwaggerSchema('delete', 200, '/nginx/certificates/{certID}', data); - expect(data).to.be.equal(true); + cy.validateSwaggerSchema('get', 200, '/nginx/certificates', data); + expect(data.length).to.be.greaterThan(0); + + // Delete cert + cy.task('backendApiDelete', { + token: token, + path: `/api/nginx/certificates/${certID}` + }).then((data) => { + cy.validateSwaggerSchema('delete', 200, '/nginx/certificates/{certID}', data); + expect(data).to.be.equal(true); + }); }); }); }); }); - it('Should be able to get all certs', function() { - cy.task('backendApiGet', { - token: token, - path: '/api/nginx/certificates?expand=owner' - }).then((data) => { - cy.validateSwaggerSchema('get', 200, '/nginx/certificates', data); - expect(data.length).to.be.greaterThan(0); - }); - }); - it('Request Certificate - CVE-2024-46256/CVE-2024-46257', function() { cy.task('backendApiPost', { token: token, diff --git a/test/cypress/e2e/api/Health.cy.js b/test/cypress/e2e/api/Health.cy.js index f765c99b..49881e97 100644 --- a/test/cypress/e2e/api/Health.cy.js +++ b/test/cypress/e2e/api/Health.cy.js @@ -1,4 +1,4 @@ -/// +/// describe('Basic API checks', () => { it('Should return a valid health payload', function () { @@ -12,7 +12,7 @@ describe('Basic API checks', () => { it('Should return a valid schema payload', function () { cy.task('backendApiGet', { - path: '/api/schema', + path: '/api/schema?ts=' + Date.now(), }).then((data) => { expect(data.openapi).to.be.equal('3.1.0'); }); diff --git a/test/cypress/e2e/api/Hosts.cy.js b/test/cypress/e2e/api/Hosts.cy.js index 62b9581b..75d732c7 100644 --- a/test/cypress/e2e/api/Hosts.cy.js +++ b/test/cypress/e2e/api/Hosts.cy.js @@ -1,4 +1,4 @@ -/// +/// describe('Hosts endpoints', () => { let token; diff --git a/test/cypress/e2e/api/Users.cy.js b/test/cypress/e2e/api/Users.cy.js index 43303d43..06b18317 100644 --- a/test/cypress/e2e/api/Users.cy.js +++ b/test/cypress/e2e/api/Users.cy.js @@ -1,4 +1,4 @@ -/// +/// describe('Users endpoints', () => { let token; diff --git a/test/cypress/plugins/backendApi/client.js b/test/cypress/plugins/backendApi/client.js index e7c0c439..ba28e3ae 100644 --- a/test/cypress/plugins/backendApi/client.js +++ b/test/cypress/plugins/backendApi/client.js @@ -80,7 +80,7 @@ BackendApi.prototype._handleError = function(err, resolve, reject, returnOnError * @returns {Promise} */ BackendApi.prototype.request = function (method, path, returnOnError, data) { - logger(method.toUpperCase(), this.config.baseUrl + path); + logger(method.toUpperCase(), path); const options = this._prepareOptions(returnOnError); return new Promise((resolve, reject) => { diff --git a/test/cypress/plugins/index.js b/test/cypress/plugins/index.js index 8cf6eef6..2f3e3c5a 100644 --- a/test/cypress/plugins/index.js +++ b/test/cypress/plugins/index.js @@ -1,4 +1,4 @@ -const {SwaggerValidation} = require('@jc21/cypress-swagger-validation'); +const { SwaggerValidation } = require('@jc21/cypress-swagger-validation'); module.exports = (on, config) => { // Replace swaggerBase config var wildcard diff --git a/test/package.json b/test/package.json index a1c68908..b52ecfbd 100644 --- a/test/package.json +++ b/test/package.json @@ -4,7 +4,7 @@ "description": "", "main": "index.js", "dependencies": { - "@jc21/cypress-swagger-validation": "^0.2.8", + "@jc21/cypress-swagger-validation": "^0.3.1", "axios": "^1.7.7", "cypress": "^13.15.0", "cypress-multi-reporters": "^1.6.4", @@ -19,8 +19,8 @@ "mocha-junit-reporter": "^2.2.1" }, "scripts": { - "cypress": "HTTP_PROXY=127.0.0.1:8128 HTTPS_PROXY=127.0.0.1:8128 cypress open --config-file=cypress/config/dev.js", - "cypress:headless": "HTTP_PROXY=127.0.0.1:8128 HTTPS_PROXY=127.0.0.1:8128 cypress run --config-file=cypress/config/dev.js" + "cypress": "HTTP_PROXY=127.0.0.1:8128 HTTPS_PROXY=127.0.0.1:8128 cypress open --config-file=cypress/config/ci.js", + "cypress:headless": "HTTP_PROXY=127.0.0.1:8128 HTTPS_PROXY=127.0.0.1:8128 cypress run --config-file=cypress/config/ci.js" }, "author": "", "license": "ISC" diff --git a/test/yarn.lock b/test/yarn.lock index 38d85e0a..4fa9e51f 100644 --- a/test/yarn.lock +++ b/test/yarn.lock @@ -11,14 +11,13 @@ call-me-maybe "^1.0.1" js-yaml "^3.13.1" -"@apidevtools/json-schema-ref-parser@9.0.9": - version "9.0.9" - resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz#d720f9256e3609621280584f2b47ae165359268b" - integrity sha512-GBD2Le9w2+lVFoc4vswGI/TjkNIZSVp7+9xPf+X3uidBfWnAeUWmquteSyt0+VCrhNMWj/FTABISQrD3Z/YA+w== +"@apidevtools/json-schema-ref-parser@^11.7.2": + version "11.7.2" + resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.7.2.tgz#cdf3e0aded21492364a70e193b45b7cf4177f031" + integrity sha512-4gY54eEGEstClvEkGnwVkTkrx0sqwemEFG5OSRRn3tD91XH0+Q8XIkYIfo7IwEWPpJZwILb9GUXeShtplRc/eA== dependencies: "@jsdevtools/ono" "^7.1.3" - "@types/json-schema" "^7.0.6" - call-me-maybe "^1.0.1" + "@types/json-schema" "^7.0.15" js-yaml "^4.1.0" "@apidevtools/openapi-schemas@^2.1.0": @@ -167,15 +166,16 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.3.1.tgz#c72a5c76a9fbaf3488e231b13dc52c0da7bab42a" integrity sha512-JBxkERygn7Bv/GbN5Rv8Ul6LVknS+5Bp6RgDC/O8gEBU/yeH5Ui5C/OlWrTb6qct7LjjfT6Re2NxB0ln0yYybA== -"@jc21/cypress-swagger-validation@^0.2.8": - version "0.2.8" - resolved "https://registry.yarnpkg.com/@jc21/cypress-swagger-validation/-/cypress-swagger-validation-0.2.8.tgz#8ab059bd41e3ee100a1998a1484b9e5a2e9a4224" - integrity sha512-9fiZIHj3//bJjC5YUMOc42RnoEUeeokVn6xtMnP52XIZ/ryWQ9PIyFdlOAH8q/LW/uPxozJo2+hdB6ou4iurag== +"@jc21/cypress-swagger-validation@^0.3.1": + version "0.3.1" + resolved "https://registry.yarnpkg.com/@jc21/cypress-swagger-validation/-/cypress-swagger-validation-0.3.1.tgz#1cdd49850a20f876ed62149623f99988264751be" + integrity sha512-Vdt1gLfj8p0tJhA42Cfn43XBbsKocNfVCEVSwkn7RmZgWUyRKjqhBBRTVa9cKZTozyg8Co/yhBMsNyjmHFVXtQ== dependencies: + "@apidevtools/json-schema-ref-parser" "^11.7.2" "@apidevtools/swagger-parser" "^10.1.0" ajv "^8.17.1" + axios "^1.7.7" json-schema "^0.4.0" - json-schema-ref-parser "^9.0.9" jsonpath "^1.1.1" lodash "^4.17.21" openapi-types "^12.1.3" @@ -196,7 +196,7 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50" integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== -"@types/json-schema@^7.0.15", "@types/json-schema@^7.0.6": +"@types/json-schema@^7.0.15": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -1468,13 +1468,6 @@ json-buffer@3.0.1: resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== -json-schema-ref-parser@^9.0.9: - version "9.0.9" - resolved "https://registry.yarnpkg.com/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz#66ea538e7450b12af342fa3d5b8458bc1e1e013f" - integrity sha512-qcP2lmGy+JUoQJ4DOQeLaZDqH9qSkeGCK3suKWxJXS82dg728Mn3j97azDMaOUmJAN4uCq91LdPx4K7E8F1a7Q== - dependencies: - "@apidevtools/json-schema-ref-parser" "9.0.9" - json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" From 929ac3bd7cbe4f11e7ec49744c251cf9dc0aadad Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Wed, 16 Oct 2024 11:06:29 +1000 Subject: [PATCH 03/10] Adds env var to set certbot acme server this is required for test suite to use dns certbot request without talking to live or staging letsencrypt servers or production level dns providers. This is a backwards port from the v3 branch and opens the door for a full certificate cypress test --- backend/internal/certificate.js | 4 +- backend/lib/config.js | 10 ++++ docker/Dockerfile | 4 ++ docker/dev/Dockerfile | 24 +++++---- docker/docker-compose.ci.yml | 3 ++ docker/docker-compose.dev.yml | 86 +++++++++++++++++++++++++++++++++ scripts/.common.sh | 10 ++++ scripts/cypress-dev | 13 +++++ scripts/start-dev | 37 +++++++++++++- 9 files changed, 180 insertions(+), 11 deletions(-) create mode 100755 scripts/cypress-dev diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index 68dd55e8..aea741c7 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -20,6 +20,7 @@ const internalHost = require('./host'); const letsencryptStaging = config.useLetsencryptStaging(); +const letsencryptServer = config.useLetsencryptServer(); const letsencryptConfig = '/etc/letsencrypt.ini'; const certbotCommand = 'certbot'; @@ -838,7 +839,8 @@ const internalCertificate = { '--email "' + certificate.meta.letsencrypt_email + '" ' + '--preferred-challenges "dns,http" ' + '--domains "' + certificate.domain_names.join(',') + '" ' + - (letsencryptStaging ? '--staging' : ''); + (letsencryptStaging ? '--staging' : '') + + (letsencryptServer !== null ? `--server '${letsencryptServer}'` : ''); logger.info('Command:', cmd); diff --git a/backend/lib/config.js b/backend/lib/config.js index b42f9e3c..f7fbdca6 100644 --- a/backend/lib/config.js +++ b/backend/lib/config.js @@ -180,5 +180,15 @@ module.exports = { */ useLetsencryptStaging: function () { return !!process.env.LE_STAGING; + }, + + /** + * @returns {string|null} + */ + useLetsencryptServer: function () { + if (process.env.LE_SERVER) { + return process.env.LE_SERVER; + } + return null; } }; diff --git a/docker/Dockerfile b/docker/Dockerfile index 799ee2a6..0603e2de 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,6 +3,8 @@ # This file assumes that the frontend has been built using ./scripts/frontend-build +FROM nginxproxymanager/testca AS testca +FROM letsencrypt/pebble AS pebbleca FROM nginxproxymanager/nginx-full:certbot-node ARG TARGETPLATFORM @@ -45,6 +47,8 @@ RUN yarn install \ # add late to limit cache-busting by modifications COPY docker/rootfs / +COPY --from=pebbleca /test/certs/pebble.minica.pem /etc/ssl/certs/pebble.minica.pem +COPY --from=testca /home/step/certs/root_ca.crt /etc/ssl/certs/NginxProxyManager.crt # Remove frontend service not required for prod, dev nginx config as well RUN rm -rf /etc/s6-overlay/s6-rc.d/user/contents.d/frontend /etc/nginx/conf.d/dev.conf \ diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 3c21849c..bb4ac6d4 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,7 +1,10 @@ +FROM nginxproxymanager/testca AS testca +FROM letsencrypt/pebble AS pebbleca FROM nginxproxymanager/nginx-full:certbot-node LABEL maintainer="Jamie Curnow " -# See: https://github.com/just-containers/s6-overlay/blob/master/README.md +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + ENV SUPPRESS_NO_CONFIG_WARNING=1 \ S6_BEHAVIOUR_IF_STAGE2_FAILS=1 \ S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 \ @@ -17,17 +20,20 @@ RUN echo "fs.file-max = 65535" > /etc/sysctl.conf \ && rm -rf /var/lib/apt/lists/* # Task -RUN cd /usr \ - && curl -sL https://taskfile.dev/install.sh | sh \ - && cd /root +WORKDIR /usr +RUN curl -sL https://taskfile.dev/install.sh | sh +WORKDIR /root COPY rootfs / -RUN rm -f /etc/nginx/conf.d/production.conf -RUN chmod 644 /etc/logrotate.d/nginx-proxy-manager - -# s6 overlay COPY scripts/install-s6 /tmp/install-s6 -RUN /tmp/install-s6 "${TARGETPLATFORM}" && rm -f /tmp/install-s6 +RUN rm -f /etc/nginx/conf.d/production.conf \ + && chmod 644 /etc/logrotate.d/nginx-proxy-manager \ + && /tmp/install-s6 "${TARGETPLATFORM}" \ + && rm -f /tmp/install-s6 + +# Certs for testing purposes +COPY --from=pebbleca /test/certs/pebble.minica.pem /etc/ssl/certs/pebble.minica.pem +COPY --from=testca /home/step/certs/root_ca.crt /etc/ssl/certs/NginxProxyManager.crt EXPOSE 80 81 443 ENTRYPOINT [ "/init" ] diff --git a/docker/docker-compose.ci.yml b/docker/docker-compose.ci.yml index e13e8bf2..bb68858f 100644 --- a/docker/docker-compose.ci.yml +++ b/docker/docker-compose.ci.yml @@ -9,6 +9,9 @@ services: environment: DEBUG: 'true' FORCE_COLOR: 1 + # Required for DNS Certificate provisioning in CI + LE_SERVER: 'https://ca.internal/acme/acme/directory' + REQUESTS_CA_BUNDLE: '/etc/ssl/certs/NginxProxyManager.crt' volumes: - 'npm_data_ci:/data' - 'npm_le_ci:/etc/letsencrypt' diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index e7f9bcba..2bfa2b79 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -33,12 +33,20 @@ services: DB_MYSQL_NAME: 'npm' # DB_SQLITE_FILE: "/data/database.sqlite" # DISABLE_IPV6: "true" + # Required for DNS Certificate provisioning testing: + LE_SERVER: 'https://ca.internal/acme/acme/directory' + REQUESTS_CA_BUNDLE: '/etc/ssl/certs/NginxProxyManager.crt' volumes: - npm_data:/data - le_data:/etc/letsencrypt + - './dev/resolv.conf:/etc/resolv.conf:ro' - ../backend:/app - ../frontend:/app/frontend - ../global:/app/global + healthcheck: + test: ["CMD", "/usr/bin/check-health"] + interval: 10s + timeout: 3s depends_on: - db working_dir: /app @@ -58,6 +66,23 @@ services: volumes: - db_data:/var/lib/mysql + stepca: + image: jc21/testca + volumes: + - './dev/resolv.conf:/etc/resolv.conf:ro' + - '/etc/localtime:/etc/localtime:ro' + networks: + nginx_proxy_manager: + aliases: + - ca.internal + + dnsrouter: + image: jc21/dnsrouter + volumes: + - ./dev/dnsrouter-config.json.tmp:/dnsrouter-config.json:ro + networks: + - nginx_proxy_manager + swagger: image: swaggerapi/swagger-ui:latest container_name: npm_swagger @@ -74,12 +99,71 @@ services: container_name: npm_squid volumes: - './dev/squid.conf:/etc/squid/squid.conf:ro' + - './dev/resolv.conf:/etc/resolv.conf:ro' - '/etc/localtime:/etc/localtime:ro' networks: - nginx_proxy_manager ports: - 8128:3128 + pdns: + image: pschiffe/pdns-mysql + volumes: + - '/etc/localtime:/etc/localtime:ro' + environment: + PDNS_master: 'yes' + PDNS_api: 'yes' + PDNS_api_key: 'npm' + PDNS_webserver: 'yes' + PDNS_webserver_address: '0.0.0.0' + PDNS_webserver_password: 'npm' + PDNS_webserver-allow-from: '127.0.0.0/8,192.0.0.0/8,10.0.0.0/8,172.0.0.0/8' + PDNS_version_string: 'anonymous' + PDNS_default_ttl: 1500 + PDNS_allow_axfr_ips: '127.0.0.0/8,192.0.0.0/8,10.0.0.0/8,172.0.0.0/8' + PDNS_gmysql_host: pdns-db + PDNS_gmysql_port: 3306 + PDNS_gmysql_user: pdns + PDNS_gmysql_password: pdns + PDNS_gmysql_dbname: pdns + depends_on: + - pdns-db + networks: + nginx_proxy_manager: + aliases: + - ns1.pdns + - ns2.pdns + + pdns-db: + image: mariadb + environment: + MYSQL_ROOT_PASSWORD: 'pdns' + MYSQL_DATABASE: 'pdns' + MYSQL_USER: 'pdns' + MYSQL_PASSWORD: 'pdns' + volumes: + - 'pdns_mysql:/var/lib/mysql' + - '/etc/localtime:/etc/localtime:ro' + - './dev/pdns-db.sql:/docker-entrypoint-initdb.d/01_init.sql:ro' + networks: + - nginx_proxy_manager + + cypress: + image: "npm_dev_cypress" + build: + context: ../ + dockerfile: test/cypress/Dockerfile + environment: + HTTP_PROXY: 'squid:3128' + HTTPS_PROXY: 'squid:3128' + volumes: + - '../test/results:/results' + - './dev/resolv.conf:/etc/resolv.conf:ro' + - '/etc/localtime:/etc/localtime:ro' + command: cypress run --browser chrome --config-file=cypress/config/ci.js + networks: + - nginx_proxy_manager + volumes: npm_data: name: npm_core_data @@ -87,6 +171,8 @@ volumes: name: npm_le_data db_data: name: npm_db_data + pdns_mysql: + name: npm_pdns_mysql networks: nginx_proxy_manager: diff --git a/scripts/.common.sh b/scripts/.common.sh index 3cea0916..4111db3b 100644 --- a/scripts/.common.sh +++ b/scripts/.common.sh @@ -15,3 +15,13 @@ COMPOSE_PROJECT_NAME="npmdev" COMPOSE_FILE="docker/docker-compose.dev.yml" export COMPOSE_FILE COMPOSE_PROJECT_NAME + +# $1: container_name +get_container_ip () { + local container_name=$1 + local container + local ip + container=$(docker-compose ps --all -q "${container_name}" | tail -n1) + ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") + echo "$ip" +} diff --git a/scripts/cypress-dev b/scripts/cypress-dev new file mode 100755 index 00000000..a0c64ad9 --- /dev/null +++ b/scripts/cypress-dev @@ -0,0 +1,13 @@ +#!/bin/bash -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$DIR/.common.sh" + +# Ensure docker-compose exists +if hash docker-compose 2>/dev/null; then + cd "${DIR}/.." + rm -rf "$DIR/../test/results" + docker-compose up --build cypress +else + echo -e "${RED}❯ docker-compose command is not available${RESET}" +fi diff --git a/scripts/start-dev b/scripts/start-dev index f064a4bd..9da6ed45 100755 --- a/scripts/start-dev +++ b/scripts/start-dev @@ -7,8 +7,43 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if hash docker-compose 2>/dev/null; then cd "${DIR}/.." echo -e "${BLUE}❯ ${CYAN}Starting Dev Stack ...${RESET}" + echo -e "${BLUE}❯ $(docker-compose config)${RESET}" - docker-compose up -d --remove-orphans --force-recreate --build + # Bring up a stack, in steps so we can inject IPs everywhere + docker-compose up -d pdns pdns-db + PDNS_IP=$(get_container_ip "pdns") + echo -e "${BLUE}❯ ${YELLOW}PDNS IP is ${PDNS_IP}${RESET}" + + # adjust the dnsrouter config + LOCAL_DNSROUTER_CONFIG="$DIR/../docker/dev/dnsrouter-config.json" + rm -rf "$LOCAL_DNSROUTER_CONFIG.tmp" + # IMPORTANT: changes to dnsrouter-config.json will affect this line: + jq --arg a "$PDNS_IP" '.servers[0].upstreams[1].upstream = $a' "$LOCAL_DNSROUTER_CONFIG" > "$LOCAL_DNSROUTER_CONFIG.tmp" + + docker-compose up -d dnsrouter + DNSROUTER_IP=$(get_container_ip "dnsrouter") + echo -e "${BLUE}❯ ${YELLOW}DNS Router IP is ${DNSROUTER_IP}" + + if [ "${DNSROUTER_IP:-}" = "" ]; then + echo -e "${RED}❯ ERROR: DNS Router IP is not set${RESET}" + exit 1 + fi + + # mount the resolver + LOCAL_RESOLVE="$DIR/../docker/dev/resolv.conf" + rm -rf "${LOCAL_RESOLVE}" + printf "nameserver %s\noptions ndots:0" "${DNSROUTER_IP}" > "${LOCAL_RESOLVE}" + + # bring up all remaining containers, except cypress! + docker-compose up -d --remove-orphans stepca squid + docker-compose pull db + docker-compose up -d --remove-orphans --pull=never fullstack + docker-compose up -d --remove-orphans swagger + + # docker-compose up -d --remove-orphans --force-recreate --build + + # wait for main container to be healthy + bash "$DIR/wait-healthy" "$(docker-compose ps --all -q fullstack)" 120 echo "" echo -e "${CYAN}Admin UI: http://127.0.0.1:3081${RESET}" From 5bdc05878f60a92de4d90a325d8724d0fb521248 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Wed, 16 Oct 2024 11:22:24 +1000 Subject: [PATCH 04/10] Fix issues with certbot command when using LE_SERVER --- backend/internal/certificate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index aea741c7..901128d9 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -839,8 +839,8 @@ const internalCertificate = { '--email "' + certificate.meta.letsencrypt_email + '" ' + '--preferred-challenges "dns,http" ' + '--domains "' + certificate.domain_names.join(',') + '" ' + - (letsencryptStaging ? '--staging' : '') + - (letsencryptServer !== null ? `--server '${letsencryptServer}'` : ''); + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); logger.info('Command:', cmd); From fe2d8895d68b6f9f96ea0fdba6e34e63a16e24c9 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Wed, 16 Oct 2024 14:53:57 +1000 Subject: [PATCH 05/10] Cypress test for http and dns cert provision --- backend/internal/certificate.js | 69 ++++++++++--------- .../schema/components/certificate-object.json | 27 ++++---- test/cypress/e2e/api/FullCertProvision.cy.js | 61 ++++++++++++++++ test/cypress/plugins/backendApi/client.js | 2 +- 4 files changed, 114 insertions(+), 45 deletions(-) create mode 100644 test/cypress/e2e/api/FullCertProvision.cy.js diff --git a/backend/internal/certificate.js b/backend/internal/certificate.js index 901128d9..34b8fdf5 100644 --- a/backend/internal/certificate.js +++ b/backend/internal/certificate.js @@ -209,6 +209,7 @@ const internalCertificate = { .patchAndFetchById(certificate.id, { expires_on: moment(cert_info.dates.to, 'X').format('YYYY-MM-DD HH:mm:ss') }) + .then(utils.omitRow(omissions())) .then((saved_row) => { // Add cert data for audit log saved_row.meta = _.assign({}, saved_row.meta, { @@ -732,29 +733,29 @@ const internalCertificate = { return utils.exec('openssl x509 -in ' + certificate_file + ' -subject -noout') .then((result) => { + // Examples: + // subject=CN = *.jc21.com // subject=CN = something.example.com const regex = /(?:subject=)?[^=]+=\s+(\S+)/gim; const match = regex.exec(result); - - if (typeof match[1] === 'undefined') { - throw new error.ValidationError('Could not determine subject from certificate: ' + result); + if (match && typeof match[1] !== 'undefined') { + certData['cn'] = match[1]; } - - certData['cn'] = match[1]; }) .then(() => { return utils.exec('openssl x509 -in ' + certificate_file + ' -issuer -noout'); }) + .then((result) => { + // Examples: // issuer=C = US, O = Let's Encrypt, CN = Let's Encrypt Authority X3 + // issuer=C = US, O = Let's Encrypt, CN = E5 + // issuer=O = NginxProxyManager, CN = NginxProxyManager Intermediate CA","O = NginxProxyManager, CN = NginxProxyManager Intermediate CA const regex = /^(?:issuer=)?(.*)$/gim; const match = regex.exec(result); - - if (typeof match[1] === 'undefined') { - throw new error.ValidationError('Could not determine issuer from certificate: ' + result); + if (match && typeof match[1] !== 'undefined') { + certData['issuer'] = match[1]; } - - certData['issuer'] = match[1]; }) .then(() => { return utils.exec('openssl x509 -in ' + certificate_file + ' -dates -noout'); @@ -829,16 +830,16 @@ const internalCertificate = { requestLetsEncryptSsl: (certificate) => { logger.info('Requesting Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', ')); - const cmd = certbotCommand + ' certonly ' + - '--config "' + letsencryptConfig + '" ' + + const cmd = `${certbotCommand} certonly ` + + `--config '${letsencryptConfig}' ` + '--work-dir "/tmp/letsencrypt-lib" ' + '--logs-dir "/tmp/letsencrypt-log" ' + - '--cert-name "npm-' + certificate.id + '" ' + + `--cert-name "npm-${certificate.id}" ` + '--agree-tos ' + '--authenticator webroot ' + - '--email "' + certificate.meta.letsencrypt_email + '" ' + + `--email '${certificate.meta.letsencrypt_email}' ` + '--preferred-challenges "dns,http" ' + - '--domains "' + certificate.domain_names.join(',') + '" ' + + `--domains "${certificate.domain_names.join(',')}" ` + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); @@ -871,25 +872,26 @@ const internalCertificate = { const hasConfigArg = certificate.meta.dns_provider !== 'route53'; let mainCmd = certbotCommand + ' certonly ' + - '--config "' + letsencryptConfig + '" ' + + `--config '${letsencryptConfig}' ` + '--work-dir "/tmp/letsencrypt-lib" ' + '--logs-dir "/tmp/letsencrypt-log" ' + - '--cert-name "npm-' + certificate.id + '" ' + + `--cert-name 'npm-${certificate.id}' ` + '--agree-tos ' + - '--email "' + certificate.meta.letsencrypt_email + '" ' + - '--domains "' + certificate.domain_names.join(',') + '" ' + - '--authenticator ' + dnsPlugin.full_plugin_name + ' ' + + `--email '${certificate.meta.letsencrypt_email}' ` + + `--domains '${certificate.domain_names.join(',')}' ` + + `--authenticator '${dnsPlugin.full_plugin_name}' ` + ( hasConfigArg - ? '--' + dnsPlugin.full_plugin_name + '-credentials "' + credentialsLocation + '"' + ? `--${dnsPlugin.full_plugin_name}-credentials '${credentialsLocation}' ` : '' ) + ( certificate.meta.propagation_seconds !== undefined - ? ' --' + dnsPlugin.full_plugin_name + '-propagation-seconds ' + certificate.meta.propagation_seconds + ? `--${dnsPlugin.full_plugin_name}-propagation-seconds '${certificate.meta.propagation_seconds}' ` : '' ) + - (letsencryptStaging ? ' --staging' : ''); + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); // Prepend the path to the credentials file as an environment variable if (certificate.meta.dns_provider === 'route53') { @@ -966,14 +968,15 @@ const internalCertificate = { logger.info('Renewing Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', ')); const cmd = certbotCommand + ' renew --force-renewal ' + - '--config "' + letsencryptConfig + '" ' + + `--config '${letsencryptConfig}' ` + '--work-dir "/tmp/letsencrypt-lib" ' + '--logs-dir "/tmp/letsencrypt-log" ' + - '--cert-name "npm-' + certificate.id + '" ' + + `--cert-name 'npm-${certificate.id}' ` + '--preferred-challenges "dns,http" ' + '--no-random-sleep-on-renew ' + '--disable-hook-validation ' + - (letsencryptStaging ? '--staging' : ''); + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); logger.info('Command:', cmd); @@ -998,13 +1001,14 @@ const internalCertificate = { logger.info(`Renewing Let'sEncrypt certificates via ${dnsPlugin.name} for Cert #${certificate.id}: ${certificate.domain_names.join(', ')}`); let mainCmd = certbotCommand + ' renew --force-renewal ' + - '--config "' + letsencryptConfig + '" ' + + `--config "${letsencryptConfig}" ` + '--work-dir "/tmp/letsencrypt-lib" ' + '--logs-dir "/tmp/letsencrypt-log" ' + - '--cert-name "npm-' + certificate.id + '" ' + + `--cert-name 'npm-${certificate.id}' ` + '--disable-hook-validation ' + '--no-random-sleep-on-renew ' + - (letsencryptStaging ? ' --staging' : ''); + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); // Prepend the path to the credentials file as an environment variable if (certificate.meta.dns_provider === 'route53') { @@ -1030,12 +1034,13 @@ const internalCertificate = { logger.info('Revoking Let\'sEncrypt certificates for Cert #' + certificate.id + ': ' + certificate.domain_names.join(', ')); const mainCmd = certbotCommand + ' revoke ' + - '--config "' + letsencryptConfig + '" ' + + `--config '${letsencryptConfig}' ` + '--work-dir "/tmp/letsencrypt-lib" ' + '--logs-dir "/tmp/letsencrypt-log" ' + - '--cert-path "/etc/letsencrypt/live/npm-' + certificate.id + '/fullchain.pem" ' + + `--cert-path '/etc/letsencrypt/live/npm-${certificate.id}/fullchain.pem' ` + '--delete-after-revoke ' + - (letsencryptStaging ? '--staging' : ''); + (letsencryptServer !== null ? `--server '${letsencryptServer}' ` : '') + + (letsencryptStaging && letsencryptServer === null ? '--staging ' : ''); // Don't fail command if file does not exist const delete_credentialsCmd = `rm -f '/etc/letsencrypt/credentials/credentials-${certificate.id}' || true`; diff --git a/backend/schema/components/certificate-object.json b/backend/schema/components/certificate-object.json index a4e8e1fc..b75dcf61 100644 --- a/backend/schema/components/certificate-object.json +++ b/backend/schema/components/certificate-object.json @@ -45,11 +45,13 @@ "type": "object", "additionalProperties": false, "properties": { - "letsencrypt_email": { - "type": "string" + "certificate": { + "type": "string", + "minLength": 1 }, - "letsencrypt_agree": { - "type": "boolean" + "certificate_key": { + "type": "string", + "minLength": 1 }, "dns_challenge": { "type": "boolean" @@ -60,17 +62,18 @@ "dns_provider_credentials": { "type": "string" }, + "letsencrypt_agree": { + "type": "boolean" + }, + "letsencrypt_certificate": { + "type": "object" + }, + "letsencrypt_email": { + "type": "string" + }, "propagation_seconds": { "type": "integer", "minimum": 0 - }, - "certificate": { - "type": "string", - "minLength": 1 - }, - "certificate_key": { - "type": "string", - "minLength": 1 } } } diff --git a/test/cypress/e2e/api/FullCertProvision.cy.js b/test/cypress/e2e/api/FullCertProvision.cy.js new file mode 100644 index 00000000..93cacce9 --- /dev/null +++ b/test/cypress/e2e/api/FullCertProvision.cy.js @@ -0,0 +1,61 @@ +/// + +describe('Full Certificate Provisions', () => { + let token; + + before(() => { + cy.getToken().then((tok) => { + token = tok; + }); + }); + + it.only('Should be able to create new http certificate', function() { + cy.task('backendApiPost', { + token: token, + path: '/api/nginx/certificates', + data: { + domain_names: [ + 'website1.example.com' + ], + meta: { + letsencrypt_email: 'admin@example.com', + letsencrypt_agree: true, + dns_challenge: false + }, + provider: 'letsencrypt' + } + }).then((data) => { + cy.validateSwaggerSchema('post', 201, '/nginx/certificates', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.greaterThan(0); + expect(data.provider).to.be.equal('letsencrypt'); + }); + }); + + it('Should be able to create new DNS certificate with Powerdns', function() { + cy.task('backendApiPost', { + token: token, + path: '/api/certificates', + data: { + domain_names: [ + 'website2.example.com' + ], + meta: { + letsencrypt_email: "admin@example.com", + dns_challenge: true, + dns_provider: 'powerdns', + dns_provider_credentials: 'dns_powerdns_api_url = http://ns1.pdns:8081\r\ndns_powerdns_api_key = npm', + letsencrypt_agree: true + }, + provider: 'letsencrypt' + } + }).then((data) => { + cy.validateSwaggerSchema('post', 201, '/nginx/certificates', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.greaterThan(0); + expect(data.provider).to.be.equal('letsencrypt'); + expect(data.meta.dns_provider).to.be.equal('powerdns'); + }); + }); + +}); diff --git a/test/cypress/plugins/backendApi/client.js b/test/cypress/plugins/backendApi/client.js index ba28e3ae..c10653e4 100644 --- a/test/cypress/plugins/backendApi/client.js +++ b/test/cypress/plugins/backendApi/client.js @@ -7,7 +7,7 @@ const BackendApi = function(config, token) { this.axios = axios.create({ baseURL: config.baseUrl, - timeout: 5000, + timeout: 60000, }); }; From d96a3987c0d6c679059af47bec33f8d94f7e4d78 Mon Sep 17 00:00:00 2001 From: Nephiel Date: Wed, 16 Oct 2024 19:04:50 +0000 Subject: [PATCH 06/10] Fix forward_scheme validation in Redirection Host Should solve error `data/forward_scheme must be equal to one of the allowed values` when configuring a Redirection Host with scheme set to `auto`. #4074 --- backend/schema/components/redirection-host-object.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/schema/components/redirection-host-object.json b/backend/schema/components/redirection-host-object.json index cc4dbdd2..e7a495fd 100644 --- a/backend/schema/components/redirection-host-object.json +++ b/backend/schema/components/redirection-host-object.json @@ -28,7 +28,7 @@ }, "forward_scheme": { "type": "string", - "enum": ["http", "https"] + "enum": ["auto", "http", "https"] }, "forward_domain_name": { "description": "Domain Name", From fa2c814fcb76b6a11b979dc23a817853a459644d Mon Sep 17 00:00:00 2001 From: Nephiel Date: Wed, 16 Oct 2024 19:09:14 +0000 Subject: [PATCH 07/10] Fix schema validation in Default Site Should solve error `data/value must match exactly one schema in oneOf` when setting the Default Site to 404 or 444. #4074 --- backend/schema/components/setting-object.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/schema/components/setting-object.json b/backend/schema/components/setting-object.json index e0877726..65ec2a08 100644 --- a/backend/schema/components/setting-object.json +++ b/backend/schema/components/setting-object.json @@ -25,7 +25,7 @@ "value": { "description": "Value in almost any form", "example": "congratulations", - "oneOf": [ + "anyOf": [ { "type": "string", "minLength": 1 From edbed1af90ab86658a9fc4a17ef571797484edcc Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Thu, 17 Oct 2024 08:48:47 +1000 Subject: [PATCH 08/10] Adds tests for settings endpoints and reenables dns cert test and fixes problems with schema --- backend/internal/nginx.js | 4 +- .../schema/components/proxy-host-object.json | 10 +- backend/schema/components/setting-object.json | 5 +- .../paths/nginx/dead-hosts/hostID/put.json | 4 +- .../schema/paths/nginx/dead-hosts/post.json | 4 +- .../paths/nginx/proxy-hosts/hostID/put.json | 4 +- .../schema/paths/nginx/proxy-hosts/post.json | 4 +- .../nginx/redirection-hosts/hostID/put.json | 4 +- .../paths/nginx/redirection-hosts/post.json | 4 +- .../paths/nginx/streams/streamID/put.json | 4 +- .../schema/paths/settings/settingID/put.json | 18 ++- test/cypress/e2e/api/FullCertProvision.cy.js | 7 +- .../e2e/api/{Hosts.cy.js => ProxyHosts.cy.js} | 2 +- test/cypress/e2e/api/Settings.cy.js | 124 ++++++++++++++++++ test/cypress/plugins/backendApi/client.js | 2 +- 15 files changed, 160 insertions(+), 40 deletions(-) rename test/cypress/e2e/api/{Hosts.cy.js => ProxyHosts.cy.js} (96%) create mode 100644 test/cypress/e2e/api/Settings.cy.js diff --git a/backend/internal/nginx.js b/backend/internal/nginx.js index 77933e73..5f802c00 100644 --- a/backend/internal/nginx.js +++ b/backend/internal/nginx.js @@ -181,7 +181,9 @@ const internalNginx = { * @param {Object} host * @returns {Promise} */ - generateConfig: (host_type, host) => { + generateConfig: (host_type, host_row) => { + // Prevent modifying the original object: + let host = JSON.parse(JSON.stringify(host_row)); const nice_host_type = internalNginx.getFileFriendlyHostType(host_type); if (config.debug()) { diff --git a/backend/schema/components/proxy-host-object.json b/backend/schema/components/proxy-host-object.json index a64a58c8..5098802b 100644 --- a/backend/schema/components/proxy-host-object.json +++ b/backend/schema/components/proxy-host-object.json @@ -23,9 +23,7 @@ "locations", "hsts_enabled", "hsts_subdomains", - "certificate", - "use_default_location", - "ipv6" + "certificate" ], "additionalProperties": false, "properties": { @@ -151,12 +149,6 @@ "$ref": "./access-list-object.json" } ] - }, - "use_default_location": { - "type": "boolean" - }, - "ipv6": { - "type": "boolean" } } } diff --git a/backend/schema/components/setting-object.json b/backend/schema/components/setting-object.json index 65ec2a08..b9c6a103 100644 --- a/backend/schema/components/setting-object.json +++ b/backend/schema/components/setting-object.json @@ -46,7 +46,10 @@ }, "meta": { "description": "Extra metadata", - "example": {}, + "example": { + "redirect": "http://example.com", + "html": "

404

" + }, "type": "object" } } diff --git a/backend/schema/paths/nginx/dead-hosts/hostID/put.json b/backend/schema/paths/nginx/dead-hosts/hostID/put.json index 6a0a57e3..f9505ed4 100644 --- a/backend/schema/paths/nginx/dead-hosts/hostID/put.json +++ b/backend/schema/paths/nginx/dead-hosts/hostID/put.json @@ -94,9 +94,7 @@ "avatar": "", "roles": ["admin"] }, - "certificate": null, - "use_default_location": true, - "ipv6": true + "certificate": null } } }, diff --git a/backend/schema/paths/nginx/dead-hosts/post.json b/backend/schema/paths/nginx/dead-hosts/post.json index 59313506..c8bbb693 100644 --- a/backend/schema/paths/nginx/dead-hosts/post.json +++ b/backend/schema/paths/nginx/dead-hosts/post.json @@ -79,9 +79,7 @@ "nickname": "Admin", "avatar": "", "roles": ["admin"] - }, - "use_default_location": true, - "ipv6": true + } } } }, diff --git a/backend/schema/paths/nginx/proxy-hosts/hostID/put.json b/backend/schema/paths/nginx/proxy-hosts/hostID/put.json index af73905d..5cab6e75 100644 --- a/backend/schema/paths/nginx/proxy-hosts/hostID/put.json +++ b/backend/schema/paths/nginx/proxy-hosts/hostID/put.json @@ -129,9 +129,7 @@ "roles": ["admin"] }, "certificate": null, - "access_list": null, - "use_default_location": true, - "ipv6": true + "access_list": null } } }, diff --git a/backend/schema/paths/nginx/proxy-hosts/post.json b/backend/schema/paths/nginx/proxy-hosts/post.json index 13f64161..85455fb6 100644 --- a/backend/schema/paths/nginx/proxy-hosts/post.json +++ b/backend/schema/paths/nginx/proxy-hosts/post.json @@ -114,9 +114,7 @@ "avatar": "//www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?default=mm", "roles": ["admin"] }, - "access_list": null, - "use_default_location": true, - "ipv6": true + "access_list": null } } }, diff --git a/backend/schema/paths/nginx/redirection-hosts/hostID/put.json b/backend/schema/paths/nginx/redirection-hosts/hostID/put.json index 870f16fc..fd97cbfa 100644 --- a/backend/schema/paths/nginx/redirection-hosts/hostID/put.json +++ b/backend/schema/paths/nginx/redirection-hosts/hostID/put.json @@ -114,9 +114,7 @@ "avatar": "", "roles": ["admin"] }, - "certificate": null, - "use_default_location": true, - "ipv6": true + "certificate": null } } }, diff --git a/backend/schema/paths/nginx/redirection-hosts/post.json b/backend/schema/paths/nginx/redirection-hosts/post.json index 3a9a05fe..5bfde2c3 100644 --- a/backend/schema/paths/nginx/redirection-hosts/post.json +++ b/backend/schema/paths/nginx/redirection-hosts/post.json @@ -99,9 +99,7 @@ "nickname": "Admin", "avatar": "", "roles": ["admin"] - }, - "use_default_location": true, - "ipv6": true + } } } }, diff --git a/backend/schema/paths/nginx/streams/streamID/put.json b/backend/schema/paths/nginx/streams/streamID/put.json index f3ef54d4..fbfdc901 100644 --- a/backend/schema/paths/nginx/streams/streamID/put.json +++ b/backend/schema/paths/nginx/streams/streamID/put.json @@ -129,9 +129,7 @@ "roles": ["admin"] }, "certificate": null, - "access_list": null, - "use_default_location": true, - "ipv6": true + "access_list": null } } }, diff --git a/backend/schema/paths/settings/settingID/put.json b/backend/schema/paths/settings/settingID/put.json index 5888ec05..4ca62429 100644 --- a/backend/schema/paths/settings/settingID/put.json +++ b/backend/schema/paths/settings/settingID/put.json @@ -13,7 +13,8 @@ "name": "settingID", "schema": { "type": "string", - "minLength": 1 + "minLength": 1, + "enum": ["default-site"] }, "required": true, "description": "Setting ID", @@ -31,10 +32,21 @@ "minProperties": 1, "properties": { "value": { - "$ref": "../../../components/setting-object.json#/properties/value" + "type": "string", + "minLength": 1, + "enum": ["congratulations", "404", "444", "redirect", "html"] }, "meta": { - "$ref": "../../../components/setting-object.json#/properties/meta" + "type": "object", + "additionalProperties": false, + "properties": { + "redirect": { + "type": "string" + }, + "html": { + "type": "string" + } + } } } } diff --git a/test/cypress/e2e/api/FullCertProvision.cy.js b/test/cypress/e2e/api/FullCertProvision.cy.js index 93cacce9..5ca5692c 100644 --- a/test/cypress/e2e/api/FullCertProvision.cy.js +++ b/test/cypress/e2e/api/FullCertProvision.cy.js @@ -9,7 +9,7 @@ describe('Full Certificate Provisions', () => { }); }); - it.only('Should be able to create new http certificate', function() { + it('Should be able to create new http certificate', function() { cy.task('backendApiPost', { token: token, path: '/api/nginx/certificates', @@ -35,7 +35,7 @@ describe('Full Certificate Provisions', () => { it('Should be able to create new DNS certificate with Powerdns', function() { cy.task('backendApiPost', { token: token, - path: '/api/certificates', + path: '/api/nginx/certificates', data: { domain_names: [ 'website2.example.com' @@ -45,7 +45,8 @@ describe('Full Certificate Provisions', () => { dns_challenge: true, dns_provider: 'powerdns', dns_provider_credentials: 'dns_powerdns_api_url = http://ns1.pdns:8081\r\ndns_powerdns_api_key = npm', - letsencrypt_agree: true + letsencrypt_agree: true, + propagation_seconds: 5, }, provider: 'letsencrypt' } diff --git a/test/cypress/e2e/api/Hosts.cy.js b/test/cypress/e2e/api/ProxyHosts.cy.js similarity index 96% rename from test/cypress/e2e/api/Hosts.cy.js rename to test/cypress/e2e/api/ProxyHosts.cy.js index 75d732c7..5bc64580 100644 --- a/test/cypress/e2e/api/Hosts.cy.js +++ b/test/cypress/e2e/api/ProxyHosts.cy.js @@ -1,6 +1,6 @@ /// -describe('Hosts endpoints', () => { +describe('Proxy Hosts endpoints', () => { let token; before(() => { diff --git a/test/cypress/e2e/api/Settings.cy.js b/test/cypress/e2e/api/Settings.cy.js new file mode 100644 index 00000000..6942760c --- /dev/null +++ b/test/cypress/e2e/api/Settings.cy.js @@ -0,0 +1,124 @@ +/// + +describe('Settings endpoints', () => { + let token; + + before(() => { + cy.getToken().then((tok) => { + token = tok; + }); + }); + + it('Get all settings', function() { + cy.task('backendApiGet', { + token: token, + path: '/api/settings', + }).then((data) => { + cy.validateSwaggerSchema('get', 200, '/settings', data); + expect(data.length).to.be.greaterThan(0); + }); + }); + + it('Get default-site setting', function() { + cy.task('backendApiGet', { + token: token, + path: '/api/settings/default-site', + }).then((data) => { + cy.validateSwaggerSchema('get', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + }); + }); + + it('Default Site congratulations', function() { + cy.task('backendApiPut', { + token: token, + path: '/api/settings/default-site', + data: { + value: 'congratulations', + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + expect(data).to.have.property('value'); + expect(data.value).to.be.equal('congratulations'); + }); + }); + + it('Default Site 404', function() { + cy.task('backendApiPut', { + token: token, + path: '/api/settings/default-site', + data: { + value: '404', + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + expect(data).to.have.property('value'); + expect(data.value).to.be.equal('404'); + }); + }); + + it('Default Site 444', function() { + cy.task('backendApiPut', { + token: token, + path: '/api/settings/default-site', + data: { + value: '444', + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + expect(data).to.have.property('value'); + expect(data.value).to.be.equal('444'); + }); + }); + + it('Default Site redirect', function() { + cy.task('backendApiPut', { + token: token, + path: '/api/settings/default-site', + data: { + value: 'redirect', + meta: { + redirect: 'https://www.google.com', + }, + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + expect(data).to.have.property('value'); + expect(data.value).to.be.equal('redirect'); + expect(data).to.have.property('meta'); + expect(data.meta).to.have.property('redirect'); + expect(data.meta.redirect).to.be.equal('https://www.google.com'); + }); + }); + + it('Default Site html', function() { + cy.task('backendApiPut', { + token: token, + path: '/api/settings/default-site', + data: { + value: 'html', + meta: { + html: '

hello world

' + }, + }, + }).then((data) => { + cy.validateSwaggerSchema('put', 200, '/settings/{settingID}', data); + expect(data).to.have.property('id'); + expect(data.id).to.be.equal('default-site'); + expect(data).to.have.property('value'); + expect(data.value).to.be.equal('html'); + expect(data).to.have.property('meta'); + expect(data.meta).to.have.property('html'); + expect(data.meta.html).to.be.equal('

hello world

'); + }); + }); +}); diff --git a/test/cypress/plugins/backendApi/client.js b/test/cypress/plugins/backendApi/client.js index c10653e4..6f5f7661 100644 --- a/test/cypress/plugins/backendApi/client.js +++ b/test/cypress/plugins/backendApi/client.js @@ -7,7 +7,7 @@ const BackendApi = function(config, token) { this.axios = axios.create({ baseURL: config.baseUrl, - timeout: 60000, + timeout: 90000, }); }; From 2f9e062718d0d66fcff0e240ac47533bb1fcbda4 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Thu, 17 Oct 2024 09:05:25 +1000 Subject: [PATCH 09/10] bump version --- .version | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.version b/.version index d8b69897..3cf561c0 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.12.0 +2.12.1 diff --git a/README.md b/README.md index 2d1b8da5..9ac6a6c8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@



- + From d499e2bfef7c79981ad46fa5946e743d7380eb65 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Thu, 17 Oct 2024 10:00:12 +1000 Subject: [PATCH 10/10] Push PR and github branch builds to separate docker image --- Jenkinsfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9b29ee97..302e05b2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -43,7 +43,7 @@ pipeline { steps { script { // Defaults to the Branch name, which is applies to all branches AND pr's - buildxPushTags = "-t docker.io/jc21/${IMAGE}:github-${BRANCH_LOWER}" + buildxPushTags = "-t docker.io/nginxproxymanager/${IMAGE}-dev:${BRANCH_LOWER}" } } } @@ -203,7 +203,13 @@ pipeline { } steps { script { - npmGithubPrComment("Docker Image for build ${BUILD_NUMBER} is available on [DockerHub](https://cloud.docker.com/repository/docker/jc21/${IMAGE}) as `jc21/${IMAGE}:github-${BRANCH_LOWER}`\n\n**Note:** ensure you backup your NPM instance before testing this PR image! Especially if this PR contains database changes.", true) + npmGithubPrComment("""Docker Image for build ${BUILD_NUMBER} is available on +[DockerHub](https://cloud.docker.com/repository/docker/nginxproxymanager/${IMAGE}-dev) +as `nginxproxymanager/${IMAGE}-dev:${BRANCH_LOWER}` + +**Note:** ensure you backup your NPM instance before testing this image! Especially if there are database changes +**Note:** this is a different docker image namespace than the official image +""", true) } } }