build everything inside dockerfile/add some scripts

Signed-off-by: Zoey <zoey@z0ey.de>
This commit is contained in:
Zoey
2023-03-16 21:52:41 +01:00
parent 437144211a
commit 4af50b7ef5
10 changed files with 307 additions and 206 deletions

View File

@@ -10,43 +10,25 @@ on:
paths:
- .github/workflows/docker.yml
- Dockerfile
- frontend/**
- backend/**
- global/**
- rootfs/**
pull_request:
paths:
- .github/workflows/docker.yml
- Dockerfile
- frontend/**
- backend/**
- global/**
- rootfs/**
workflow_dispatch:
jobs:
backend-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 19
- name: Test Backend
run: |
sudo pip install certbot
sudo mkdir -p /usr/local/nginx/conf/conf.d/include /data/tls/certbot /tmp/acme-challenge
sudo touch /usr/local/nginx/conf/conf.d/include/ip_ranges.conf
sudo cp rootfs/etc/tls/certbot.ini /data/tls/certbot/config.ini
mv global backend
cd backend
npm install --package-lock=false --force
sudo nginx
NODE_ENV=production sudo -E timeout 30 node --abort_on_uncaught_exception --max_old_space_size=250 index.js || if [ "$?" == "124" ]; then exit 0; else exit 1; fi
- name: Kill workflow
if: failure()
run: |
curl -X POST https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel --header "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Load frontend from cache
uses: actions/cache/restore@v3
with:
path: frontend/dist
key: frontend
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
@@ -56,6 +38,7 @@ jobs:
with:
driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=-1
- name: Login to DockerHub
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
@@ -71,13 +54,43 @@ jobs:
password: ${{ github.token }}
- name: Build
uses: docker/build-push-action@v4
if: ${{ github.event_name != 'pull_request' }}
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64 #,linux/amd64/v2,linux/amd64/v3,linux/amd64/v4 #,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6
push: ${{ github.ref == 'refs/heads/develop' }}
tags: "ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }}\n${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }}\n \n"
push: ${{ github.event_name != 'pull_request' }}
tags: |
${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }}
ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }}
build-args: |
"BUILD=${{ github.event.repository.name }}"
- name: show version
if: ${{ github.event_name != 'pull_request' }}
run: |
docker run --rm --entrypoint nginx ${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }} -V
docker run --rm --entrypoint nginx ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ github.ref_name }} -V
- name: Set PR-Number (PR)
if: ${{ github.event_name == 'pull_request' }}
id: pr
run: echo "pr=$(echo pr-${{ github.ref_name }} | sed "s|refs/pull/:||g" | sed "s|/merge||g")" >> $GITHUB_OUTPUT
- name: Build (PR)
uses: docker/build-push-action@v4
if: ${{ github.event_name == 'pull_request' }}
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64 #,linux/amd64/v2,linux/amd64/v3,linux/amd64/v4 #,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6
push: ${{ github.event_name == 'pull_request' }}
tags: ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }}
build-args: |
"BUILD=${{ github.event.repository.name }}"
- name: show version (PR)
if: ${{ github.event_name == 'pull_request' }}
run: docker run --rm --entrypoint nginx ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }} -V
- name: add comment (PR)
uses: mshick/add-pr-comment@v2
if: ${{ github.event_name == 'pull_request' }}
with:
message: "The Docker Image can now be found here: `ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }}`"
repo-token: ${{ github.token }}

View File

@@ -1,35 +0,0 @@
name: Build frontend
on:
push:
branches:
- develop
paths:
- .github/workflows/frontend.yml
- frontend/**
- global/**
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 19
- name: Prepare frontend
run: |
export NODE_OPTIONS=--openssl-legacy-provider
npm install --global yarn
cd frontend
sed -i "s|\"0.0.0\"|\""$(cat ../global/.version)"\"|g" package.json
yarn --no-lockfile install
yarn --no-lockfile build
mkdir dist/.well-known
cp ../security.txt dist/.well-known
curl https://api.github.com/repos/${{ github.repository }}/actions/caches?key=frontend -X DELETE --header "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"
- name: Cache frontend
uses: actions/cache/save@v3
with:
path: frontend/dist
key: frontend

View File

@@ -1,87 +0,0 @@
name: Build PR
on:
pull_request:
paths:
- .github/workflows/pull-request
- Dockerfile
- frontend/**
- backend/**
- global/**
- rootfs/**
jobs:
backend-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 19
- name: Test Backend
run: |
sudo pip install certbot
sudo mkdir -p /usr/local/nginx/conf/conf.d/include /data/tls/certbot /tmp/acme-challenge
sudo touch /usr/local/nginx/conf/conf.d/include/ip_ranges.conf
sudo cp rootfs/etc/tls/certbot.ini /data/tls/certbot/config.ini
mv global backend
cd backend
npm install --package-lock=false --force
sudo nginx
NODE_ENV=production sudo -E timeout 30 node --abort_on_uncaught_exception --max_old_space_size=250 index.js || if [ "$?" == "124" ]; then exit 0; else exit 1; fi
- name: Kill workflow
if: failure()
run: |
curl -X POST https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/cancel --header "authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 19
- name: Prepare frontend
run: |
export NODE_OPTIONS=--openssl-legacy-provider
npm install --global yarn
cd frontend
sed -i "s|\"0.0.0\"|\""$(cat ../global/.version)"\"|g" package.json
yarn --no-lockfile install
yarn --no-lockfile build
mkdir dist/.well-known
cp ../security.txt dist/.well-known
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: arm64 #all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=-1
- name: Convert Username
id: un
run: echo "un=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ steps.un.outputs.un }}
password: ${{ github.token }}
- name: Set PR-Number
id: pr
run: echo "pr=$(echo pr-${{ github.ref_name }} | sed "s|refs/pull/:||g" | sed "s|/merge||g")" >> $GITHUB_OUTPUT
- name: Build
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
platforms: linux/amd64,linux/arm64 #,linux/amd64/v2,linux/amd64/v3,linux/amd64/v4 #,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6
push: ${{ github.event_name == 'pull_request' }}
tags: ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }}
- name: show version
run: docker run --rm --entrypoint nginx ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }} -V
- name: add comment
uses: mshick/add-pr-comment@v2
with:
message: "The Docker Image can now be found here: `ghcr.io/${{ steps.un.outputs.un }}/${{ github.event.repository.name }}:${{ steps.pr.outputs.pr }}`"
repo-token: ${{ github.token }}

View File

@@ -1,26 +1,59 @@
FROM --platform="$BUILDPLATFORM" alpine:3.17.2 as frontend
COPY global /build/global
COPY frontend /build/frontend
RUN apk upgrade --no-cache && \
apk add --no-cache ca-certificates tzdata \
nodejs yarn git build-base python3
ARG NODE_ENV=production \
NODE_OPTIONS=--openssl-legacy-provider
RUN cd /build/frontend && \
sed -i "s|\"0.0.0\"|\""$(cat ../global/.version)"\"|g" package.json && \
yarn --no-lockfile install && \
yarn --no-lockfile build
COPY security.txt /build/frontend/dist/.well-known/security.txt
FROM --platform="$BUILDPLATFORM" alpine:3.17.2 as backend
COPY backend /build/backend
COPY global /build/backend/global
RUN apk upgrade --no-cache && \
apk add --no-cache ca-certificates tzdata \
nodejs-current yarn && \
wget https://gobinaries.com/tj/node-prune -O - | sh
ARG NODE_ENV=production \
TARGETARCH
RUN cd /build/backend && \
sed -i "s|\"0.0.0\"|\""$(cat global/.version)"\"|g" package.json && \
if [ "$TARGETARCH" = "amd64" ]; then \
npm_config_target_platform=linux npm_config_target_arch=x64 yarn install --no-lockfile; \
elif [ "$TARGETARCH" = "arm64" ]; then \
npm_config_target_platform=linux npm_config_target_arch=arm64 yarn install --no-lockfile; \
fi && \
node-prune
FROM zoeyvid/nginx-quic:87
COPY rootfs /
COPY backend /app
COPY global /app/global
COPY frontend/dist /app/frontend
ENV NODE_ENV=production \
DB_SQLITE_FILE=/data/database.sqlite
WORKDIR /app
RUN apk upgrade --no-cache && \
apk add --no-cache ca-certificates tzdata \
nodejs-current \
openssl apache2-utils \
coreutils grep jq curl \
npm build-base libffi-dev && \
# Build Backend
sed -i "s|\"0.0.0\"|\""$(cat global/.version)"\"|g" package.json && \
npm install --package-lock=false --force && \
build-base libffi-dev && \
# Install Certbot
pip install --no-cache-dir certbot && \
# Clean
apk del --no-cache npm build-base libffi-dev
apk del --no-cache build-base libffi-dev
COPY rootfs /
COPY --from=backend /build/backend /app
COPY --from=frontend /build/frontend/dist /app/frontend
RUN ln -s /app/password-reset.js /usr/local/bin/password-reset.js && \
ln -s /app/sqlite-vaccum.js /usr/local/bin/sqlite-vaccum.js
ENV NODE_ENV=production \
DB_SQLITE_FILE=/data/database.sqlite
WORKDIR /app
ENTRYPOINT ["start.sh"]
HEALTHCHECK CMD check-health.sh

View File

@@ -61,6 +61,9 @@ so that the barrier for entry here is low.
- HTTP/2 always enabled
- HTTP/2 upload fixed
- Infinite upload size allowed
- Auto database vacuum (only sqlite) (FULLCLEAN=true)
- Auto certbot old certs clean (FULLCLEAN=true)
- Passwort reset (only sqlite) (`docker exec -it nginx-proxy-manager password-reset.js USER_EMAIL PASSWORD`)
## Soon
- more

59
backend/password-reset.js Normal file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/node
// based on: https://github.com/jlesage/docker-nginx-proxy-manager/blob/796734a3f9a87e0b1561b47fd418f82216359634/rootfs/opt/nginx-proxy-manager/bin/reset-password
const fs = require('fs');
const bcrypt = require('bcrypt');
const sqlite3 = require('sqlite3');
function usage() {
console.log(`usage: node ${process.argv[1]} USER_EMAIL PASSWORD
Reset password of a Nginx Proxy Manager user.
Arguments:
USER_EMAIL Email address of the user to reset the password.
PASSWORD Optional new password of the user. If not set, password
is set to 'changeme'.
`);
process.exit(1);
}
const args = process.argv.slice(2);
const USER_EMAIL = args[0];
if (!USER_EMAIL) {
console.error('ERROR: User email address must be set.');
usage();
}
const PASSWORD = args[1];
if (!PASSWORD) {
console.error('ERROR: Password must be set.');
usage();
}
if (fs.existsSync(process.env.DB_SQLITE_FILE)) {
bcrypt.hash(PASSWORD, 13, (err, PASSWORD_HASH) => {
if (err) {
console.error(err);
process.exit(1);
}
const db = new sqlite3.Database(process.env.DB_SQLITE_FILE);
db.run(
`UPDATE auth SET secret = ? WHERE EXISTS
(SELECT * FROM user WHERE user.id = auth.user_id AND user.email = ?)`,
[PASSWORD_HASH, USER_EMAIL],
function (err) {
if (err) {
console.error(err);
process.exit(1);
}
console.log(`Password for user ${USER_EMAIL} has been reset.`);
process.exit(0);
}
);
});
}

22
backend/sqlite-vaccum.js Normal file
View File

@@ -0,0 +1,22 @@
#!/usr/bin/node
const fs = require('fs');
const sqlite3 = require('sqlite3');
if (fs.existsSync(process.env.DB_SQLITE_FILE)) {
const db = new sqlite3.Database(process.env.DB_SQLITE_FILE, sqlite3.OPEN_READWRITE, (err) => {
if (err) {
console.error(err.message);
} else {
db.run('VACUUM;', [], (err) => {
if (err) {
console.error(err.message);
}
db.close((err) => {
if (err) {
console.error(err.message);
}
});
});
}
});
}

View File

@@ -0,0 +1,90 @@
#!/bin/sh
# based on https://github.com/jlesage/docker-nginx-proxy-manager/blob/796734a/rootfs/opt/nginx-proxy-manager/bin/lecleaner
BASE="/data/tls/certbot"
live_dir="$BASE/live"
archive_dir="$BASE/archive"
csr_dir="$BASE/csr"
key_dir="$BASE/keys"
# Set of certificate paths actively used.
in_use=""
keep_count=0
delete_count=0
error_count=0
remove_file() {
f="$1"
if rm -f "$f"; then
return 0
else
echo "ERROR: Could not remove $f." >&2
return 1
fi
}
# Build the set of certificates in use.
for domain_dir in "$live_dir"/*; do
[ -e "$domain_dir" ] || continue
if [ ! -d "$domain_dir" ]; then
continue
fi
for certlink in "$domain_dir"/*; do
[ -e "$certlink" ] || continue
if [ ! -L "$certlink" ]; then
continue
fi
target=$(readlink -f "$certlink")
in_use="$in_use $target"
done
done
echo "----------------------------------------------------------"
echo "Let's Encrypt certificates cleanup - $(date +"%Y/%m/%d %H:%M:%S")"
echo "----------------------------------------------------------"
# Remove all unused certificates from the archive directory.
for domain_dir in "$archive_dir"/*; do
[ -e "$domain_dir" ] || continue
if [ ! -d "$domain_dir" ]; then
continue
fi
for certfile in "$domain_dir"/*; do
[ -e "$certlink" ] || continue
if echo "$in_use" | grep -q "$certfile"; then
echo "Keeping $certfile."
keep_count=$((keep_count+1))
else
echo "Deleting $certfile."
if remove_file "$certfile"; then
delete_count=$((delete_count+1))
else
error_count=$((error_count+1))
fi
fi
done
done
# Remove all files from the csr and key directories.
for dir in "$csr_dir" "$key_dir"; do
for file in "$dir"/*; do
[ -e "$file" ] || continue
if [ ! -f "$file" ]; then
continue
fi
echo "Deleting $file."
if remove_file "$file"; then
delete_count=$((delete_count+1))
else
error_count=$((error_count+1))
fi
done
done
echo "$keep_count file(s) kept."
echo "$delete_count file(s) deleted."
if [ "$error_count" -gt 0 ]; then
echo "$error_count file(s) failed to be deleted."
fi

View File

@@ -174,12 +174,15 @@ if [ -n "$FULLCLEAN" ]; then
fi
if [ "$FULLCLEAN" = true ]; then
find /data/tls/certbot/csr -mtime +90 -name "*.pem" -delete
find /data/tls/certbot/keys -mtime +90 -name "*.pem" -delete
find /data/tls/certbot/archive -mtime +90 -name "*.pem" -delete
if [ "$PHP81" != true ] && [ "$PHP82" != true ]; then
rm -vrf /data/php
fi
if [ -f "$DB_SQLITE_FILE" ]; then
sqlite-vaccum.js || exit 1
fi
certbot-cleaner.sh
fi
find /data/nginx -type f -name '*.conf' -exec sed -i "s|listen 80 http2|listen 80|g" {} \; || sleep inf
@@ -313,7 +316,7 @@ else
fi
fi
ns="$(< /etc/resolv.conf grep -P "^nameserver [0-9\[\].:]+$" | sed "s|nameserver ||g" | tr "\n" " " | sed "s/\(.*\) /\1/" | head -1)" || sleep inf
ns="$(< /etc/resolv.conf grep -P "^nameserver ((?:[0-9.]+)|(?:\[[0-9a-fA-F:]+\]))$" | sed "s|nameserver ||g" | tr "\n" " " | sed "s/\(.*\) /\1/" | head -1)" || sleep inf
export ns
sed -i "s|resolver localhost;|resolver $ns;|g" /usr/local/nginx/conf/nginx.conf || sleep inf
echo "using this nameservers: \"$ns\"" || sleep inf