From 48df0eeb0a104fdec6dd18df3a6d00243450adc7 Mon Sep 17 00:00:00 2001 From: Jamie Curnow Date: Mon, 14 Jun 2021 21:21:06 +1000 Subject: [PATCH] Buildx improvements --- .dockerignore | 7 ++++ Jenkinsfile | 48 +++++------------------- backend/go.sum | 12 ------ docker/Dockerfile | 73 ++++++++++++++---------------------- docker/docker-compose.ci.yml | 2 +- scripts/buildx | 5 ++- scripts/docker-gobuild | 50 ++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 98 deletions(-) create mode 100644 .dockerignore create mode 100755 scripts/docker-gobuild diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b8f844a6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Ignore everything +* + +# Only allow the following for docker build: +!backend/ +!docker/ +!scripts/ diff --git a/Jenkinsfile b/Jenkinsfile index 8ba4ed19..80a39e6d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ pipeline { agent { - label 'docker-multiarch' + label 'taurus' } options { buildDiscarder(logRotator(numToKeepStr: '5')) @@ -71,18 +71,16 @@ pipeline { steps { withCredentials([usernamePassword(credentialsId: 'oss-index-token', passwordVariable: 'NANCY_TOKEN', usernameVariable: 'NANCY_USER')]) { sh '''docker build --pull --no-cache --squash --compress \\ - -t ${IMAGE}:ci-${BUILD_NUMBER} \\ + -t "${IMAGE}:${BRANCH_LOWER}-ci-${BUILD_NUMBER}" \\ -f docker/Dockerfile \\ - --build-arg TARGETPLATFORM=linux/amd64 \\ - --build-arg BUILDPLATFORM=linux/amd64 \\ + --build-arg BUILD_COMMIT="${BUILD_COMMIT:-dev}" \\ --build-arg BUILD_DATE="$(date '+%Y-%m-%d %T %Z')" \\ --build-arg BUILD_VERSION="${BUILD_VERSION}" \\ - --build-arg BUILD_COMMIT="${BUILD_COMMIT}" \\ - --build-arg SENTRY_DSN="${SENTRY_DSN:-}" \\ - --build-arg GOPROXY="${GOPROXY:-}" \\ --build-arg GOPRIVATE="${GOPRIVATE:-}" \\ - --build-arg NANCY_USER="${NANCY_USER}" \\ - --build-arg NANCY_TOKEN="${NANCY_TOKEN}" \\ + --build-arg GOPROXY="${GOPROXY:-}" \\ + --build-arg NANCY_TOKEN="${NANCY_TOKEN:-}" \\ + --build-arg NANCY_USER="${NANCY_USER:-}" \\ + --build-arg SENTRY_DSN="${SENTRY_DSN:-}" \\ . ''' } @@ -151,9 +149,9 @@ pipeline { withCredentials([string(credentialsId: 'npm-sentry-dsn', variable: 'SENTRY_DSN')]) { withCredentials([usernamePassword(credentialsId: 'jc21-dockerhub', passwordVariable: 'dpass', usernameVariable: 'duser')]) { // Docker Login - sh "docker login -u '${duser}' -p '${dpass}'" + sh 'docker login -u "${duser}" -p "${dpass}"' // Buildx with push from cache - sh "./scripts/buildx --push ${BUILDX_PUSH_TAGS}" + sh './scripts/buildx --push ${BUILDX_PUSH_TAGS}' // sh './scripts/buildx -o type=local,dest=docker-build' } } @@ -206,34 +204,6 @@ pipeline { } } } - /* - stage('Artifacts') { - when { - allOf { - not { - equals expected: 'UNSTABLE', actual: currentBuild.result - } - } - } - steps { - sh 'mkdir -p artifacts' - // Multiarch builds - dir(path: 'docker-build/linux_amd64/app') { - sh 'zip -qr ../../../artifacts/linux_amd64.zip *' - } - dir(path: 'docker-build/linux_arm64/app') { - sh 'zip -qr ../../../artifacts/linux_arm64.zip *' - } - dir(path: 'docker-build/linux_arm_v7/app') { - sh 'zip -qr ../../../artifacts/linux_arm_v7.zip *' - } - // Archive them - dir(path: 'artifacts') { - archiveArtifacts artifacts: '** /*' - } - } - } - */ } post { always { diff --git a/backend/go.sum b/backend/go.sum index 9b1d959d..e6ab9ab9 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -69,18 +69,6 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/jc21/jsref v0.0.0-20210608013137-43b07c7d31bd h1:Ag/L5Yc9BeBbi4i8bNAev8Ejtu/jq8Qk/xK+HDHnWNc= -github.com/jc21/jsref v0.0.0-20210608013137-43b07c7d31bd/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= -github.com/jc21/jsref v0.0.0-20210608014024-8bda7cb41eef h1:1jF5nv8PmgH2txfWGmsPium0Hj9PEnGkb96tkZ+4uDU= -github.com/jc21/jsref v0.0.0-20210608014024-8bda7cb41eef/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= -github.com/jc21/jsref v0.0.0-20210608014914-2edd4dea9791 h1:s0hsMFnTiGGytgwDbHo20OvmJj2/+FFMZvLpRNexnvk= -github.com/jc21/jsref v0.0.0-20210608014914-2edd4dea9791/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= -github.com/jc21/jsref v0.0.0-20210608023003-123d7fb98643 h1:ZpDTP4ow7hZMx0ORi06jnLP4ZDGQVa6SayH+5rWWlYg= -github.com/jc21/jsref v0.0.0-20210608023003-123d7fb98643/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= -github.com/jc21/jsref v0.0.0-20210608023437-810a57e5f736 h1:1nZYRLsHvECy8rbOLkqRBK45Y6zKQ5ZRuGPMQalPWVc= -github.com/jc21/jsref v0.0.0-20210608023437-810a57e5f736/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= -github.com/jc21/jsref v0.0.0-20210608024103-9eaa65f76123 h1:pb24Ybg78OdqO4GHh0xcwlVPWKlDYX/ZVnf+wq8D9To= -github.com/jc21/jsref v0.0.0-20210608024103-9eaa65f76123/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= github.com/jc21/jsref v0.0.0-20210608024405-a97debfc4760 h1:7wxq2DIgtO36KLrFz1RldysO0WVvcYsD49G9tyAs01k= github.com/jc21/jsref v0.0.0-20210608024405-a97debfc4760/go.mod h1:yIq2t51OJgVsdRlPY68NAnyVdBH0kYXxDTFtUxOap80= github.com/jmoiron/sqlx v1.3.3 h1:j82X0bf7oQ27XeqxicSZsTU5suPwKElg3oyxNn43iTk= diff --git a/docker/Dockerfile b/docker/Dockerfile index 5ccdc947..f1e428ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,59 +11,40 @@ FROM jc21/nginx-full:github-acme.sh-golang AS gobuild SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ARG GOPROXY +ARG BUILD_COMMIT +ARG BUILD_VERSION ARG GOPRIVATE - -ENV GOPROXY=$GOPROXY \ - GOPRIVATE=$GOPRIVATE \ - GO111MODULE=on \ - CGO_ENABLED=1 - -# Nancy -RUN go get github.com/sonatype-nexus-community/nancy -RUN mkdir -p /workspace -WORKDIR /workspace -COPY backend/go.mod backend/go.sum backend/.nancy-ignore ./ -RUN go mod download - +ARG GOPROXY ARG NANCY_TOKEN ARG NANCY_USER -RUN go list -json -m all | nancy sleuth --quiet --username "${NANCY_USER}" --token "${NANCY_TOKEN}" -RUN rm -rf /workspace +ARG SENTRY_DSN +ARG SKIP_TESTS + +ENV BUILD_COMMIT="${BUILD_COMMIT:-dev}" \ + BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \ + CGO_ENABLED=1 \ + GO111MODULE=on \ + GOPRIVATE="${GOPRIVATE:-}" \ + GOPROXY="${GOPROXY:-}" \ + NANCY_TOKEN="${NANCY_TOKEN:-}" \ + NANCY_USER="${NANCY_USER:-}" \ + SENTRY_DSN="${SENTRY_DSN:-}" \ + SKIP_TESTS="${SKIP_TESTS:-}" # Code +RUN mkdir -p /app WORKDIR /app COPY . . -WORKDIR /app/backend - -# Build -RUN go mod download -RUN echo "Testing and compiling project" \ - && [ -z "$(go tool fix -diff ./internal)" ] - -# Disabled as CI has issues at the moment -#RUN if [ "$TARGETPLATFORM" == "" ] || [ "$TARGETPLATFORM" == "linux/amd64" ]; then golangci-lint -v run ./...; fi - -RUN richgo test -cover -v ./internal/... -RUN richgo test -bench=. ./internal/... - -ARG BUILD_VERSION -ARG BUILD_COMMIT -ARG SENTRY_DSN -RUN go build \ - -ldflags "-w -s -X main.commit=${BUILD_COMMIT} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \ - -o ../dist/bin/server \ - -v ./cmd/server +RUN ./scripts/docker-gobuild #=============== # Final image #=============== -FROM jc21/nginx-full:github-acme.sh +FROM jc21/nginx-full:github-acme.sh AS final COPY --from=gobuild /app/dist /app COPY --from=gobuild /app/backend/migrations /app/migrations -# COPY frontend/build /app/frontend ENV SUPPRESS_NO_CONFIG_WARNING=1 ENV S6_FIX_ATTRS_HIDDEN=1 @@ -83,25 +64,29 @@ RUN rm -rf /etc/services.d/frontend /etc/nginx/conf.d/dev.conf VOLUME /data CMD [ "/init" ] +# TODO: remove healthchecks HEALTHCHECK --interval=15s --timeout=3s CMD curl -f http://127.0.0.1:81/api || exit 1 ARG NOW ARG BUILD_VERSION ARG BUILD_COMMIT ARG BUILD_DATE -ENV NPM_BUILD_VERSION="${BUILD_VERSION}" NPM_BUILD_COMMIT="${BUILD_COMMIT}" NPM_BUILD_DATE="${BUILD_DATE}" + ENV DATABASE_URL="sqlite:////data/nginxproxymanager.db" \ DBMATE_MIGRATIONS_DIR="/app/migrations" \ + DBMATE_NO_DUMP_SCHEMA="1" \ DBMATE_SCHEMA_FILE="/data/schema.sql" \ - DBMATE_NO_DUMP_SCHEMA="1" + NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \ + NPM_BUILD_COMMIT="${BUILD_COMMIT:-dev}" \ + NPM_BUILD_DATE="${BUILD_DATE:-}" LABEL org.label-schema.schema-version="1.0" \ org.label-schema.license="MIT" \ org.label-schema.name="nginx-proxy-manager" \ org.label-schema.description="Nginx Host Management and Proxy" \ - org.label-schema.build-date="$NOW" \ - org.label-schema.version="$BUILD_VERSION" \ + org.label-schema.build-date="${NOW:-}" \ + org.label-schema.version="${BUILD_VERSION:-0.0.0}" \ org.label-schema.url="https://nginxproxymanager.com" \ org.label-schema.vcs-url="https://github.com/jc21/nginx-proxy-manager.git" \ - org.label-schema.vcs-ref="$BUILD_COMMIT" \ - org.label-schema.cmd="docker run --rm -ti jc21/nginx-proxy-manager:$BUILD_VERSION" + org.label-schema.vcs-ref="${BUILD_COMMIT:-dev}" \ + org.label-schema.cmd="docker run --rm -ti jc21/nginx-proxy-manager:${BUILD_VERSION:-0.0.0}" diff --git a/docker/docker-compose.ci.yml b/docker/docker-compose.ci.yml index 9412f302..f9b60714 100644 --- a/docker/docker-compose.ci.yml +++ b/docker/docker-compose.ci.yml @@ -3,7 +3,7 @@ version: "3" services: fullstack: - image: ${IMAGE}:ci-${BUILD_NUMBER} + image: ${IMAGE}:${BRANCH_LOWER}-ci-${BUILD_NUMBER} environment: - LOG_LEVEL=debug volumes: diff --git a/scripts/buildx b/scripts/buildx index 2738798c..10fcf62d 100755 --- a/scripts/buildx +++ b/scripts/buildx @@ -17,12 +17,13 @@ docker buildx create --name "${BUILDX_NAME:-npm}" || echo docker buildx use "${BUILDX_NAME:-npm}" docker buildx build \ - --build-arg BUILD_VERSION="${BUILD_VERSION:-dev}" \ --build-arg BUILD_COMMIT="${BUILD_COMMIT:-notset}" \ --build-arg BUILD_DATE="$(date '+%Y-%m-%d %T %Z')" \ + --build-arg BUILD_VERSION="${BUILD_VERSION:-dev}" \ --build-arg NOW="$(date --rfc-3339=s)" \ - --build-arg GOPROXY="${GOPROXY:-}" \ + --build-arg SKIP_TESTS=1 \ --build-arg GOPRIVATE="${GOPRIVATE:-}" \ + --build-arg GOPROXY="${GOPROXY:-}" \ --build-arg SENTRY_DSN="${SENTRY_DSN:-}" \ --platform linux/amd64,linux/arm64,linux/arm/7 \ --progress plain \ diff --git a/scripts/docker-gobuild b/scripts/docker-gobuild new file mode 100755 index 00000000..a34a58cf --- /dev/null +++ b/scripts/docker-gobuild @@ -0,0 +1,50 @@ +#!/bin/bash -e + +# This script is run as part of the Dockerfile +# It will conduct golang testing and vuln lookups +# unless SKIP_TESTS=1 is defined + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$DIR/.common.sh" + +echo -e "${BLUE}❯ ${CYAN}docker-gobuild${RESET}" +echo -e " ${YELLOW}BUILD_COMMIT: ${BUILD_COMMIT:-not set}${RESET}" +echo -e " ${YELLOW}BUILD_VERSION: ${BUILD_VERSION:-not set}${RESET}" +echo -e " ${YELLOW}CGO_ENABLED: ${CGO_ENABLED:-not set}${RESET}" +echo -e " ${YELLOW}GOPROXY: ${GOPROXY:-not set}${RESET}" +echo -e " ${YELLOW}GOPRIVATE: ${GOPRIVATE:-not set}${RESET}" +echo -e " ${YELLOW}GO111MODULE: ${GO111MODULE:-not set}${RESET}" +echo -e " ${YELLOW}SKIP_TESTS: ${SKIP_TESTS:-not set}${RESET}" + +echo -e "${BLUE}❯ ${CYAN}Downloading backend go modules${RESET}" +cd /app/backend +go mod download + +# Testing and vulnerability lookup +if ! [ "${SKIP_TESTS:-}" = "1" ]; then + mkdir -p /workspace + echo -e "${BLUE}❯ ${CYAN}Nancy setup${RESET}" + cd /workspace + go get github.com/sonatype-nexus-community/nancy + cp /app/backend/go.mod /app/backend/go.sum /app/backend/.nancy-ignore . + go mod download + + echo -e "${BLUE}❯ ${CYAN}Nancy testing${RESET}" + go list -json -m all | nancy sleuth --quiet --username "${NANCY_USER}" --token "${NANCY_TOKEN:-}" + rm -rf /workspace + + echo -e "${BLUE}❯ ${CYAN}Testing backend code${RESET}" + cd /app/backend + [ -z "$(go tool fix -diff ./internal)" ] + richgo test -cover -v ./internal/... + richgo test -bench=. ./internal/... + golangci-lint -v run ./... +fi + +echo -e "${BLUE}❯ ${CYAN}Building backend binary${RESET}" +go build \ + -ldflags "-w -s -X main.commit=${BUILD_COMMIT} -X main.version=${BUILD_VERSION} -X main.sentryDSN=${SENTRY_DSN:-}" \ + -o ../dist/bin/server \ + -v ./cmd/server + +echo -e "${BLUE}❯ ${CYAN}docker-gobuild ${GREEN}completed${RESET}"