diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 0875c68f..6701dc99 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -22,24 +22,25 @@ func main() { config.Init(&version, &commit, &sentryDSN) appstate := state.NewState() - setting.ApplySettings() - database.CheckSetup() + database.Migrate(func() { + setting.ApplySettings() + database.CheckSetup() + go worker.StartCertificateWorker(appstate) - go worker.StartCertificateWorker(appstate) + api.StartServer() + irqchan := make(chan os.Signal, 1) + signal.Notify(irqchan, syscall.SIGINT, syscall.SIGTERM) - api.StartServer() - irqchan := make(chan os.Signal, 1) - signal.Notify(irqchan, syscall.SIGINT, syscall.SIGTERM) - - for irq := range irqchan { - if irq == syscall.SIGINT || irq == syscall.SIGTERM { - logger.Info("Got ", irq, " shutting server down ...") - // Close db - err := database.GetInstance().Close() - if err != nil { - logger.Error("DatabaseCloseError", err) + for irq := range irqchan { + if irq == syscall.SIGINT || irq == syscall.SIGTERM { + logger.Info("Got ", irq, " shutting server down ...") + // Close db + err := database.GetInstance().Close() + if err != nil { + logger.Error("DatabaseCloseError", err) + } + break } - break } - } + }) } diff --git a/backend/migrations/20201013035318_initial_schema.sql b/backend/internal/database/migrations/20201013035318_initial_schema.sql similarity index 100% rename from backend/migrations/20201013035318_initial_schema.sql rename to backend/internal/database/migrations/20201013035318_initial_schema.sql diff --git a/backend/migrations/20201013035839_initial_data.sql b/backend/internal/database/migrations/20201013035839_initial_data.sql similarity index 100% rename from backend/migrations/20201013035839_initial_data.sql rename to backend/internal/database/migrations/20201013035839_initial_data.sql diff --git a/backend/internal/database/migrator.go b/backend/internal/database/migrator.go new file mode 100644 index 00000000..2f0e8582 --- /dev/null +++ b/backend/internal/database/migrator.go @@ -0,0 +1,205 @@ +package database + +import ( + "database/sql" + "embed" + "fmt" + "io/fs" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "npm/internal/logger" + "npm/internal/util" + + "github.com/jmoiron/sqlx" +) + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +// MigrationConfiguration options for the migrator. +type MigrationConfiguration struct { + Table string `json:"table"` + mux sync.Mutex +} + +// Default migrator configuration +var mConfiguration = MigrationConfiguration{ + Table: "migration", +} + +// ConfigureMigrator and will return error if missing required fields. +func ConfigureMigrator(c *MigrationConfiguration) error { + // ensure updates to the config are atomic + mConfiguration.mux.Lock() + defer mConfiguration.mux.Unlock() + if c == nil { + return fmt.Errorf("a non nil Configuration is mandatory") + } + if strings.TrimSpace(c.Table) != "" { + mConfiguration.Table = c.Table + } + mConfiguration.Table = c.Table + return nil +} + +type afterMigrationComplete func() + +// Migrate will perform the migration from start to finish +func Migrate(followup afterMigrationComplete) bool { + logger.Info("Migration: Started") + + // Try to connect to the database sleeping for 15 seconds in between + var db *sqlx.DB + for { + db = GetInstance() + if db == nil { + logger.Warn("Database is unavailable for migration, retrying in 15 seconds") + time.Sleep(15 * time.Second) + } else { + break + } + } + + // Check for migration table existence + if !tableExists(db, mConfiguration.Table) { + err := createMigrationTable(db) + if err != nil { + logger.Error("MigratorError", err) + return false + } + logger.Info("Migration: Migration Table created") + } + + // DO MIGRATION + migrationCount, migrateErr := performFileMigrations(db) + if migrateErr != nil { + logger.Error("MigratorError", migrateErr) + } + + if migrateErr == nil { + logger.Info("Migration: Completed %v migration files", migrationCount) + followup() + return true + } + return false +} + +// createMigrationTable performs a query to create the migration table +// with the name specified in the configuration +func createMigrationTable(db *sqlx.DB) error { + logger.Info("Migration: Creating Migration Table: %v", mConfiguration.Table) + // nolint:lll + query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%v` (filename TEXT PRIMARY KEY, migrated_on INTEGER NOT NULL DEFAULT 0)", mConfiguration.Table) + _, err := db.Exec(query) + return err +} + +// tableExists will check the database for the existence of the specified table. +func tableExists(db *sqlx.DB, tableName string) bool { + query := `SELECT name FROM sqlite_master WHERE type='table' AND name = $1` + + row := db.QueryRowx(query, tableName) + if row == nil { + logger.Error("MigratorError", fmt.Errorf("Cannot check if table exists, no row returned: %v", tableName)) + return false + } + + var exists *bool + if err := row.Scan(&exists); err != nil { + if err == sql.ErrNoRows { + return false + } + logger.Error("MigratorError", err) + return false + } + return *exists +} + +// performFileMigrations will perform the actual migration, +// importing files and updating the database with the rows imported. +func performFileMigrations(db *sqlx.DB) (int, error) { + var importedCount = 0 + + // Grab a list of previously ran migrations from the database: + previousMigrations, prevErr := getPreviousMigrations(db) + if prevErr != nil { + return importedCount, prevErr + } + + // List up the ".sql" files on disk + err := fs.WalkDir(migrationFiles, ".", func(file string, d fs.DirEntry, err error) error { + if !d.IsDir() { + shortFile := filepath.Base(file) + + // Check if this file already exists in the previous migrations + // and if so, ignore it + if util.SliceContainsItem(previousMigrations, shortFile) { + return nil + } + + logger.Info("Migration: Importing %v", shortFile) + + sqlContents, ioErr := migrationFiles.ReadFile(path.Clean(file)) + if ioErr != nil { + return ioErr + } + + sqlString := string(sqlContents) + + tx := db.MustBegin() + if _, execErr := tx.Exec(sqlString); execErr != nil { + return execErr + } + if commitErr := tx.Commit(); commitErr != nil { + return commitErr + } + if markErr := markMigrationSuccessful(db, shortFile); markErr != nil { + return markErr + } + + importedCount++ + } + return nil + }) + + return importedCount, err +} + +// getPreviousMigrations will query the migration table for names +// of migrations we can ignore because they should have already +// been imported +func getPreviousMigrations(db *sqlx.DB) ([]string, error) { + var existingMigrations []string + // nolint:gosec + query := fmt.Sprintf("SELECT filename FROM `%v` ORDER BY filename", mConfiguration.Table) + rows, err := db.Queryx(query) + if err != nil { + if err == sql.ErrNoRows { + return existingMigrations, nil + } + return existingMigrations, err + } + + for rows.Next() { + var filename *string + err := rows.Scan(&filename) + if err != nil { + return existingMigrations, err + } + existingMigrations = append(existingMigrations, *filename) + } + + return existingMigrations, nil +} + +// markMigrationSuccessful will add a row to the migration table +func markMigrationSuccessful(db *sqlx.DB, filename string) error { + // nolint:gosec + query := fmt.Sprintf("INSERT INTO `%v` (filename) VALUES ($1)", mConfiguration.Table) + _, err := db.Exec(query, filename) + return err +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 43cb4f82..e3e5e2c8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,6 @@ RUN mkdir -p /dist \ FROM jc21/nginx-full:github-acme.sh AS final COPY --from=gobuild /dist/server /app/bin/server -COPY backend/migrations /app/migrations ENV SUPPRESS_NO_CONFIG_WARNING=1 ENV S6_FIX_ATTRS_HIDDEN=1 @@ -66,11 +65,7 @@ ARG BUILD_VERSION ARG BUILD_COMMIT ARG BUILD_DATE -ENV DATABASE_URL="sqlite:////data/nginxproxymanager.db" \ - DBMATE_MIGRATIONS_DIR="/app/migrations" \ - DBMATE_NO_DUMP_SCHEMA="1" \ - DBMATE_SCHEMA_FILE="/data/schema.sql" \ - NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \ +ENV NPM_BUILD_VERSION="${BUILD_VERSION:-0.0.0}" \ NPM_BUILD_COMMIT="${BUILD_COMMIT:-dev}" \ NPM_BUILD_DATE="${BUILD_DATE:-}" diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index f868356a..8aa1aa8f 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -10,10 +10,7 @@ ENV GOPROXY=$GOPROXY \ GOPRIVATE=$GOPRIVATE \ S6_LOGGING=0 \ SUPPRESS_NO_CONFIG_WARNING=1 \ - S6_FIX_ATTRS_HIDDEN=1 \ - DATABASE_URL="sqlite:////data/nginxproxymanager.db" \ - DBMATE_MIGRATIONS_DIR="/app/backend/migrations" \ - DBMATE_SCHEMA_FILE="/data/schema.sql" + S6_FIX_ATTRS_HIDDEN=1 RUN echo "fs.file-max = 65535" > /etc/sysctl.conf diff --git a/docker/rootfs/etc/cont-init.d/30-dbmate b/docker/rootfs/etc/cont-init.d/30-dbmate deleted file mode 100755 index bf68358b..00000000 --- a/docker/rootfs/etc/cont-init.d/30-dbmate +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/with-contenv bash - -CYAN='\E[1;36m' -YELLOW='\E[1;33m' -MAGENTA='\E[1;35m' -RESET='\E[0m' - -if [ "$LOG_LEVEL" == "debug" ]; then - echo -e "${MAGENTA}[DEBUG] ${CYAN}DATABASE_URL=${YELLOW}${DATABASE_URL}${RESET}" -fi - -# Firstly create the sqlite database if it doesn't already exist -# and run any migrations required -echo -e "${YELLOW}Running dbmate migrations ...${RESET}" -s6-setuidgid npmuser /bin/dbmate up || exit 1 -echo -e "${GREEN}Completed dbmate migrations!${RESET}"