add SKIP_IP_RANGES/improve crowdsec docs/dep updates

Signed-off-by: Zoey <zoey@z0ey.de>
This commit is contained in:
Zoey
2024-02-04 14:08:13 +01:00
parent a779df8d1b
commit e9421dd5f9
10 changed files with 76 additions and 31 deletions

View File

@@ -31,8 +31,9 @@ RUN apk add --no-cache ca-certificates nodejs-current yarn && \
FROM --platform="$BUILDPLATFORM" alpine:3.19.1 as crowdsec
SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
ARG CSNB_VER=v1.0.6-rc5
ARG CSNB_VER=v1.0.7
WORKDIR /src
RUN apk add --no-cache ca-certificates git build-base && \
@@ -46,15 +47,17 @@ RUN apk add --no-cache ca-certificates git build-base && \
sed -i "s|ENABLED=.*|ENABLED=false|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
sed -i "s|API_URL=.*|API_URL=http://127.0.0.1:8080|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
sed -i "s|BAN_TEMPLATE_PATH=.*|BAN_TEMPLATE_PATH=/data/etc/crowdsec/ban.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
sed -i "s|CAPTCHA_TEMPLATE_PATH=.*|CAPTCHA_TEMPLATE_PATH=/data/etc/crowdsec/captcha.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf
sed -i "s|CAPTCHA_TEMPLATE_PATH=.*|CAPTCHA_TEMPLATE_PATH=/data/etc/crowdsec/captcha.html|g" /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
echo "APPSEC_URL=http://127.0.0.1:7422" | tee -a /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf && \
echo "APPSEC_FAILURE_ACTION=deny" | tee -a /src/crowdsec-nginx-bouncer/lua-mod/config_example.conf
FROM zoeyvid/nginx-quic:252
FROM zoeyvid/nginx-quic:256
SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
ARG CRS_VER=v4.0/dev
COPY rootfs /
COPY --from=zoeyvid/certbot-docker:21 /usr/local /usr/local
COPY --from=zoeyvid/certbot-docker:24 /usr/local /usr/local
COPY --from=zoeyvid/curl-quic:370 /usr/local/bin/curl /usr/local/bin/curl
RUN apk add --no-cache ca-certificates tzdata tini \
@@ -116,6 +119,7 @@ ENV PUID=0 \
NGINX_LOG_NOT_FOUND=false \
CLEAN=true \
FULLCLEAN=false \
SKIP_IP_RANGES=false \
LOGROTATE=false \
LOGROTATIONS=3 \
GOA=false \

View File

@@ -18,9 +18,9 @@ running at home or otherwise, including free TLS, without having to know too muc
**Note: If you don't use network mode host, which I don't recommend, don't forget to enable IPv6 in Docker, see [here](https://github.com/nextcloud/all-in-one/blob/main/docker-ipv6-support.md), you only need to edit the daemon.json and restart docker, if you use the bridge network, otherwise please enable IPv6 in your custom docker network!** <br>
**Note: Don't forget to open Port 80 (tcp) and 443 (tcp AND udp, http3/quic needs udp) in your firewall (because of network mode host, you also need to open this ports in ufw, if you use ufw).** <br>
**Note: ModSecurity overblocking (403 Error)? Please see `/opt/npm/etc/modsecurity`, if you also use CRS please see [here](https://coreruleset.org/docs/concepts/false_positives_tuning).** <br>
**Note: Internal/LAN Instance? Please disable `must-staple` in `/opt/npm/tls/certbot/config.ini`.** <br>
**Note: Internal/LAN Instance? Please disable `must-staple` in `/opt/npm/tls/certbot/config.ini` before creating your certificates.** <br>
**Note: Other Databases like MariaDB may work, but are unsupported.** <br>
**Note: access.log, logrotate and goaccess are NOT enabled by default bceuase of GDPR.** <br>
**Note: access.log/stream.log, logrotate and goaccess are NOT enabled by default bceuase of GDPR, you can enable them in the compose.yaml.** <br>
## Project Goal
@@ -104,12 +104,39 @@ so that the barrier for entry here is low.
# Crowdsec
1. Install crowdsec using this compose file: https://github.com/ZoeyVid/NPMplus/blob/develop/compose.crowdsec.yaml
2. make sure to use `network_mode: host` in your compose file
3. run `docker exec crowdsec cscli bouncers add npmplus -o raw` and save the output
4. open `/opt/npm/etc/crowdsec/crowdsec.conf`
5. set `ENABLED` to `true`
6. use the output of step 3 as `API_KEY`
7. make sure `API_URL` is set to `http://127.0.0.1:8080`
2. open `/opt/crowdsec/conf/acquis.d/appsec.yaml` and fill it with:
```yaml
listen_addr: 127.0.0.1:7422
appsec_config: crowdsecurity/virtual-patching
name: myAppSecComponent
source: appsec
labels:
type: appsec
```
3. open `/opt/crowdsec/conf/acquis.d/npmplus.yaml` and fill it with:
```yaml
filenames:
- /opt/npm/nginx/access.log
labels:
type: npmplus
---
source: docker
container_name:
- npmplus
labels:
type: npmplus
---
source: docker
container_name:
- npmplus
labels:
type: modsecurity
```
4. make sure to use `network_mode: host` in your compose file
5. run `docker exec crowdsec cscli bouncers add npmplus -o raw` and save the output
6. open `/opt/npm/etc/crowdsec/crowdsec.conf`
7. set `ENABLED` to `true`
8. use the output of step 5 as `API_KEY`
9. save the file
10. restart the npm

View File

@@ -20,8 +20,10 @@ const internalIpRanges = {
iteration_count: 0,
initTimer: () => {
logger.info('IP Ranges Renewal Timer initialized');
internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
if (process.env.SKIP_IP_RANGES === false) {
logger.info('IP Ranges Renewal Timer initialized');
internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
}
},
fetchUrl: (url) => {
@@ -47,7 +49,7 @@ const internalIpRanges = {
* Triggered at startup and then later by a timer, this will fetch the ip ranges from services and apply them to nginx.
*/
fetch: () => {
if (!internalIpRanges.interval_processing) {
if (!internalIpRanges.interval_processing && process.env.SKIP_IP_RANGES === false) {
internalIpRanges.interval_processing = true;
logger.info('Fetching IP Ranges from online services...');

View File

@@ -5,9 +5,13 @@ services:
restart: always
network_mode: bridge
ports:
- "127.0.0.1:7422:7422"
- "127.0.0.1:8080:8080"
environment:
- "TZ=Europe/Berlin"
- "COLLECTIONS=ZoeyVid/npmplus crowdsecurity/appsec-virtual-patching"
volumes:
- "/opt/crowdsec/conf:/etc/crowdsec"
- "/opt/crowdsec/data:/var/lib/crowdsec/data"
- "/opt/npm/nginx:/opt/npm/nginx:ro"
- "/var/run/docker.sock:/var/run/docker.sock:ro"

View File

@@ -33,7 +33,8 @@ services:
# - "NGINX_LOG_NOT_FOUND=true" # Allow logging of 404 errors, default false
# - "CLEAN=false" # Clean folders, default true
# - "FULLCLEAN=true" # Clean unused config folders, default false
# - "LOGROTATE=true" # Enables writing http access logs to /opt/npm/nginx/access.log and daily logrotation, default false
# - "SKIP_IP_RANGES=true" # Skip feteching/whitelisting ip ranges from aws and cloudflare, default false
# - "LOGROTATE=true" # Enables writing http access logs to /opt/npm/nginx/access.log, stream access logs to /opt/npm/nginx/stream.log and enables daily logrotation, default false
# - "LOGROTATIONS=7" # Set how often the access.log should be rotated until it is deleted, default 3
# - "GOA=true" # Enables goaccess, overrides LOGROTATE, default false --- if you download the GeoLite2-Country.mmdb, GeoLite2-City.mmdb AND GeoLite2-ASN.mmdb file from MaxMind and place them in /opt/npm/etc/goaccess/geoip it will automatically enable GeoIP in goaccess after restarting NPMplus (no need to change GOACLA below), you may also use the compose.geoip.yaml
# - "GOACLA=--agent-list --real-os --double-decode --anonymize-ip --anonymize-level=2 --keep-last=7 --with-output-resolver --no-query-string" # Arguments that should be passed to goaccess, default: https://github.com/ZoeyVid/NPMplus/blob/develop/rootfs/usr/local/bin/launch.sh#L50 and: --agent-list --real-os --double-decode --anonymize-ip --anonymize-level=1 --keep-last=30 --with-output-resolver --no-query-string

View File

@@ -8,7 +8,7 @@
"babel-core": "6.26.3",
"babel-loader": "8.3.0",
"babel-preset-env": "1.7.0",
"backbone": "1.5.0",
"backbone": "1.6.0",
"backbone.marionette": "4.1.3",
"copy-webpack-plugin": "5.1.2",
"css-loader": "5.2.7",

View File

@@ -1,4 +1,4 @@
/data/nginx/access.log {
/data/nginx/*.log {
daily
rotate 3
missingok

File diff suppressed because one or more lines are too long

View File

@@ -146,6 +146,11 @@ if ! echo "$FULLCLEAN" | grep -q "^true$\|^false$"; then
sleep inf
fi
if ! echo "$SKIP_IP_RANGES" | grep -q "^true$\|^false$"; then
echo "SKIP_IP_RANGES needs to be true or false."
sleep inf
fi
if ! echo "$LOGROTATE" | grep -q "^true$\|^false$"; then
echo "LOGROTATE needs to be true or false."
sleep inf
@@ -356,7 +361,10 @@ if [ "$LOGROTATE" = "true" ]; then
sed -i "s|rotate [0-9]\+|rotate $LOGROTATIONS|g" /etc/logrotate
elif [ "$FULLCLEAN" = "true" ]; then
rm -vrf /data/etc/logrotate.status \
/data/nginx/access.log.*
/data/nginx/access.log \
/data/nginx/access.log.* \
/data/nginx/stream.log \
/data/nginx/stream.log.*
fi
mkdir -p /tmp/acme-challenge \
@@ -662,12 +670,6 @@ sed -i "s|#\?ssl_certificate_key .*|ssl_certificate_key $DEFAULT_KEY;|g" /usr/lo
if [ -n "$DEFAULT_CHAIN" ]; then sed -i "s|#\?ssl_trusted_certificate .*|ssl_trusted_certificate $DEFAULT_CHAIN;|g" /usr/local/nginx/conf/conf.d/include/goaccess-no-server-name.conf; fi
if [ "$DISABLE_IPV6" = "true" ]; then
sed -i "s|#\?resolver .*|resolver local=on valid=10s ipv6=off;|g" /usr/local/nginx/conf/nginx.conf
else
sed -i "s|#\?resolver .*|resolver local=on valid=10s;|g" /usr/local/nginx/conf/nginx.conf
fi
sed -i "s|48693|$NIBEP|g" /app/index.js
sed -i "s|48693|$NIBEP|g" /usr/local/nginx/conf/conf.d/npm.conf
@@ -732,9 +734,11 @@ else
fi
if [ "$LOGROTATE" = "true" ]; then
sed -i "s|access_log.*|access_log /data/nginx/access.log log;|g" /usr/local/nginx/conf/nginx.conf
sed -i "s|access_log off; # http|access_log /data/nginx/access.log log;|g" /usr/local/nginx/conf/nginx.conf
sed -i "s|access_log off; # stream|access_log /data/nginx/stream.log proxy;|g" /usr/local/nginx/conf/nginx.conf
else
sed -i "s|access_log.*|access_log off;|g" /usr/local/nginx/conf/nginx.conf
sed -i "s|access_log /data/nginx/access.log log;|access_log off; # http|g" /usr/local/nginx/conf/nginx.conf
sed -i "s|access_log /data/nginx/stream.log proxy;|access_log off; # stream|g" /usr/local/nginx/conf/nginx.conf
fi
if [ ! -s /data/nginx/default.conf ]; then

View File

@@ -16,7 +16,7 @@ events {
http {
log_format log '[$time_local] $host $remote_addr $request_time "$request" $status $body_bytes_sent $bytes_sent $http_referer $http_user_agent';
access_log off;
access_log off; # http
log_not_found off;
include mime.types;
@@ -49,7 +49,7 @@ http {
quic_retry on;
ssl_dyn_rec_enable on;
#resolver ;
resolver local=on valid=10s ipv6=off;
fastcgi_index index.php;
index index.php index.html;
@@ -139,6 +139,9 @@ http {
}
stream {
log_format proxy '$remote_addr [$time_local] $protocol $status $bytes_sent $bytes_received $session_time "$upstream_addr" "$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
access_log off; # stream
# Custom
include /data/nginx/custom/stream_top.conf;