Convert db backend to use Gorm, with basis for support

for Mysql and Postgres in addition to existing Sqlite
This commit is contained in:
Jamie Curnow
2023-05-26 11:04:43 +10:00
parent b4e5b8b6db
commit 29990110b1
93 changed files with 1215 additions and 3075 deletions

View File

@ -1,25 +0,0 @@
package host
import (
"npm/internal/entity"
)
var filterMapFunctions = make(map[string]entity.FilterMapFunction)
// getFilterMapFunctions is a map of functions that should be executed
// during the filtering process, if a field is defined here then the value in
// the filter will be given to the defined function and it will return a new
// value for use in the sql query.
func getFilterMapFunctions() map[string]entity.FilterMapFunction {
// if len(filterMapFunctions) == 0 {
// TODO: See internal/model/file_item.go:620 for an example
// }
return filterMapFunctions
}
// GetFilterSchema returns filter schema
func GetFilterSchema() string {
var m Model
return entity.GetFilterSchema(m)
}

View File

@ -1,181 +1,40 @@
package host
import (
"database/sql"
"fmt"
"npm/internal/database"
"npm/internal/entity"
"npm/internal/errors"
"npm/internal/logger"
"npm/internal/model"
"github.com/rotisserie/eris"
)
// GetByID finds a Host by ID
func GetByID(id int) (Model, error) {
func GetByID(id uint) (Model, error) {
var m Model
err := m.LoadByID(id)
return m, err
}
// create will create a Host from this model
func create(host *Model) (int, error) {
if host.ID != 0 {
return 0, eris.New("Cannot create host when model already has an ID")
}
host.Touch(true)
db := database.GetInstance()
// nolint: gosec
result, err := db.NamedExec(`INSERT INTO `+tableName+` (
created_on,
modified_on,
user_id,
type,
nginx_template_id,
listen_interface,
domain_names,
upstream_id,
proxy_scheme,
proxy_host,
proxy_port,
certificate_id,
access_list_id,
ssl_forced,
caching_enabled,
block_exploits,
allow_websocket_upgrade,
http2_support,
hsts_enabled,
hsts_subdomains,
paths,
advanced_config,
status,
error_message,
is_disabled,
is_deleted
) VALUES (
:created_on,
:modified_on,
:user_id,
:type,
:nginx_template_id,
:listen_interface,
:domain_names,
:upstream_id,
:proxy_scheme,
:proxy_host,
:proxy_port,
:certificate_id,
:access_list_id,
:ssl_forced,
:caching_enabled,
:block_exploits,
:allow_websocket_upgrade,
:http2_support,
:hsts_enabled,
:hsts_subdomains,
:paths,
:advanced_config,
:status,
:error_message,
:is_disabled,
:is_deleted
)`, host)
if err != nil {
return 0, err
}
last, lastErr := result.LastInsertId()
if lastErr != nil {
return 0, lastErr
}
logger.Debug("Created Host: %+v", host)
return int(last), nil
}
// update will Update a Host from this model
func update(host *Model) error {
if host.ID == 0 {
return eris.New("Cannot update host when model doesn't have an ID")
}
host.Touch(false)
db := database.GetInstance()
// nolint: gosec
_, err := db.NamedExec(`UPDATE `+fmt.Sprintf("`%s`", tableName)+` SET
created_on = :created_on,
modified_on = :modified_on,
user_id = :user_id,
type = :type,
nginx_template_id = :nginx_template_id,
listen_interface = :listen_interface,
domain_names = :domain_names,
upstream_id = :upstream_id,
proxy_scheme = :proxy_scheme,
proxy_host = :proxy_host,
proxy_port = :proxy_port,
certificate_id = :certificate_id,
access_list_id = :access_list_id,
ssl_forced = :ssl_forced,
caching_enabled = :caching_enabled,
block_exploits = :block_exploits,
allow_websocket_upgrade = :allow_websocket_upgrade,
http2_support = :http2_support,
hsts_enabled = :hsts_enabled,
hsts_subdomains = :hsts_subdomains,
paths = :paths,
advanced_config = :advanced_config,
status = :status,
error_message = :error_message,
is_disabled = :is_disabled,
is_deleted = :is_deleted
WHERE id = :id`, host)
logger.Debug("Updated Host: %+v", host)
return err
}
// List will return a list of hosts
func List(pageInfo model.PageInfo, filters []model.Filter, expand []string) (ListResponse, error) {
var result ListResponse
var exampleModel Model
func List(pageInfo model.PageInfo, filters []model.Filter, expand []string) (entity.ListResponse, error) {
var result entity.ListResponse
defaultSort := model.Sort{
Field: "domain_names",
Direction: "ASC",
}
db := database.GetInstance()
if db == nil {
return result, errors.ErrDatabaseUnavailable
}
dbo := entity.ListQueryBuilder(&pageInfo, defaultSort, filters)
// Get count of items in this search
query, params := entity.ListQueryBuilder(exampleModel, tableName, &pageInfo, defaultSort, filters, getFilterMapFunctions(), true)
countRow := db.QueryRowx(query, params...)
var totalRows int
queryErr := countRow.Scan(&totalRows)
if queryErr != nil && queryErr != sql.ErrNoRows {
logger.Debug("%s -- %+v", query, params)
return result, queryErr
var totalRows int64
if res := dbo.Model(&Model{}).Count(&totalRows); res.Error != nil {
return result, res.Error
}
// Get rows
items := make([]Model, 0)
query, params = entity.ListQueryBuilder(exampleModel, tableName, &pageInfo, defaultSort, filters, getFilterMapFunctions(), false)
err := db.Select(&items, query, params...)
if err != nil {
logger.Debug("%s -- %+v", query, params)
return result, err
if res := dbo.Find(&items); res.Error != nil {
return result, res.Error
}
if expand != nil {
@ -187,7 +46,7 @@ func List(pageInfo model.PageInfo, filters []model.Filter, expand []string) (Lis
}
}
result = ListResponse{
result = entity.ListResponse{
Items: items,
Total: totalRows,
Limit: pageInfo.Limit,
@ -201,32 +60,28 @@ func List(pageInfo model.PageInfo, filters []model.Filter, expand []string) (Lis
// GetUpstreamUseCount returns the number of hosts that are using
// an upstream, and have not been deleted.
func GetUpstreamUseCount(upstreamID int) int {
db := database.GetInstance()
query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE upstream_id = ? AND is_deleted = ?", tableName)
countRow := db.QueryRowx(query, upstreamID, 0)
var totalRows int
queryErr := countRow.Scan(&totalRows)
if queryErr != nil && queryErr != sql.ErrNoRows {
logger.Debug("%s", query)
func GetUpstreamUseCount(upstreamID uint) int64 {
db := database.GetDB()
var count int64
if result := db.Model(&Model{}).Where("upstream_id = ?", upstreamID).Count(&count); result.Error != nil {
logger.Debug("GetUpstreamUseCount Error: %v", result.Error)
return 0
}
return totalRows
return count
}
// GetCertificateUseCount returns the number of hosts that are using
// a certificate, and have not been deleted.
func GetCertificateUseCount(certificateID int) int {
db := database.GetInstance()
query := fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE certificate_id = ? AND is_deleted = ?", tableName)
countRow := db.QueryRowx(query, certificateID, 0)
var totalRows int
queryErr := countRow.Scan(&totalRows)
if queryErr != nil && queryErr != sql.ErrNoRows {
logger.Debug("%s", query)
func GetCertificateUseCount(certificateID uint) int64 {
db := database.GetDB()
var count int64
if result := db.Model(&Model{}).Where("certificate_id = ?", certificateID).Count(&count); result.Error != nil {
logger.Debug("GetUpstreamUseCount Error: %v", result.Error)
return 0
}
return totalRows
return count
}
// AddPendingJobs is intended to be used at startup to add

View File

@ -2,9 +2,8 @@ package host
import (
"fmt"
"time"
"npm/internal/database"
"npm/internal/entity"
"npm/internal/entity/certificate"
"npm/internal/entity/nginxtemplate"
"npm/internal/entity/upstream"
@ -17,8 +16,6 @@ import (
)
const (
tableName = "host"
// ProxyHostType is self explanatory
ProxyHostType = "proxy"
// RedirectionHostType is self explanatory
@ -27,67 +24,53 @@ const (
DeadHostType = "dead"
)
// Model is the user model
// Model is the model
type Model struct {
ID int `json:"id" db:"id" filter:"id,integer"`
CreatedOn types.DBDate `json:"created_on" db:"created_on" filter:"created_on,integer"`
ModifiedOn types.DBDate `json:"modified_on" db:"modified_on" filter:"modified_on,integer"`
UserID int `json:"user_id" db:"user_id" filter:"user_id,integer"`
Type string `json:"type" db:"type" filter:"type,string"`
NginxTemplateID int `json:"nginx_template_id" db:"nginx_template_id" filter:"nginx_template_id,integer"`
ListenInterface string `json:"listen_interface" db:"listen_interface" filter:"listen_interface,string"`
DomainNames types.JSONB `json:"domain_names" db:"domain_names" filter:"domain_names,string"`
UpstreamID int `json:"upstream_id" db:"upstream_id" filter:"upstream_id,integer"`
ProxyScheme string `json:"proxy_scheme" db:"proxy_scheme" filter:"proxy_scheme,string"`
ProxyHost string `json:"proxy_host" db:"proxy_host" filter:"proxy_host,string"`
ProxyPort int `json:"proxy_port" db:"proxy_port" filter:"proxy_port,integer"`
CertificateID int `json:"certificate_id" db:"certificate_id" filter:"certificate_id,integer"`
AccessListID int `json:"access_list_id" db:"access_list_id" filter:"access_list_id,integer"`
SSLForced bool `json:"ssl_forced" db:"ssl_forced" filter:"ssl_forced,boolean"`
CachingEnabled bool `json:"caching_enabled" db:"caching_enabled" filter:"caching_enabled,boolean"`
BlockExploits bool `json:"block_exploits" db:"block_exploits" filter:"block_exploits,boolean"`
AllowWebsocketUpgrade bool `json:"allow_websocket_upgrade" db:"allow_websocket_upgrade" filter:"allow_websocket_upgrade,boolean"`
HTTP2Support bool `json:"http2_support" db:"http2_support" filter:"http2_support,boolean"`
HSTSEnabled bool `json:"hsts_enabled" db:"hsts_enabled" filter:"hsts_enabled,boolean"`
HSTSSubdomains bool `json:"hsts_subdomains" db:"hsts_subdomains" filter:"hsts_subdomains,boolean"`
Paths string `json:"paths" db:"paths" filter:"paths,string"`
AdvancedConfig string `json:"advanced_config" db:"advanced_config" filter:"advanced_config,string"`
Status string `json:"status" db:"status" filter:"status,string"`
ErrorMessage string `json:"error_message" db:"error_message" filter:"error_message,string"`
IsDisabled bool `json:"is_disabled" db:"is_disabled" filter:"is_disabled,boolean"`
IsDeleted bool `json:"is_deleted,omitempty" db:"is_deleted"`
entity.ModelBase
UserID uint `json:"user_id" gorm:"column:user_id" filter:"user_id,integer"`
Type string `json:"type" gorm:"column:type" filter:"type,string"`
NginxTemplateID uint `json:"nginx_template_id" gorm:"column:nginx_template_id" filter:"nginx_template_id,integer"`
ListenInterface string `json:"listen_interface" gorm:"column:listen_interface" filter:"listen_interface,string"`
DomainNames types.JSONB `json:"domain_names" gorm:"column:domain_names" filter:"domain_names,string"`
UpstreamID uint `json:"upstream_id" gorm:"column:upstream_id" filter:"upstream_id,integer"`
ProxyScheme string `json:"proxy_scheme" gorm:"column:proxy_scheme" filter:"proxy_scheme,string"`
ProxyHost string `json:"proxy_host" gorm:"column:proxy_host" filter:"proxy_host,string"`
ProxyPort int `json:"proxy_port" gorm:"column:proxy_port" filter:"proxy_port,integer"`
CertificateID uint `json:"certificate_id" gorm:"column:certificate_id" filter:"certificate_id,integer"`
AccessListID uint `json:"access_list_id" gorm:"column:access_list_id" filter:"access_list_id,integer"`
SSLForced bool `json:"ssl_forced" gorm:"column:ssl_forced" filter:"ssl_forced,boolean"`
CachingEnabled bool `json:"caching_enabled" gorm:"column:caching_enabled" filter:"caching_enabled,boolean"`
BlockExploits bool `json:"block_exploits" gorm:"column:block_exploits" filter:"block_exploits,boolean"`
AllowWebsocketUpgrade bool `json:"allow_websocket_upgrade" gorm:"column:allow_websocket_upgrade" filter:"allow_websocket_upgrade,boolean"`
HTTP2Support bool `json:"http2_support" gorm:"column:http2_support" filter:"http2_support,boolean"`
HSTSEnabled bool `json:"hsts_enabled" gorm:"column:hsts_enabled" filter:"hsts_enabled,boolean"`
HSTSSubdomains bool `json:"hsts_subdomains" gorm:"column:hsts_subdomains" filter:"hsts_subdomains,boolean"`
Paths string `json:"paths" gorm:"column:paths" filter:"paths,string"`
AdvancedConfig string `json:"advanced_config" gorm:"column:advanced_config" filter:"advanced_config,string"`
Status string `json:"status" gorm:"column:status" filter:"status,string"`
ErrorMessage string `json:"error_message" gorm:"column:error_message" filter:"error_message,string"`
IsDisabled bool `json:"is_disabled" gorm:"column:is_disabled" filter:"is_disabled,boolean"`
// Expansions
Certificate *certificate.Model `json:"certificate,omitempty"`
NginxTemplate *nginxtemplate.Model `json:"nginx_template,omitempty"`
User *user.Model `json:"user,omitempty"`
Upstream *upstream.Model `json:"upstream,omitempty"`
Certificate *certificate.Model `json:"certificate,omitempty" gorm:"-"`
NginxTemplate *nginxtemplate.Model `json:"nginx_template,omitempty" gorm:"-"`
User *user.Model `json:"user,omitempty" gorm:"-"`
Upstream *upstream.Model `json:"upstream,omitempty" gorm:"-"`
}
func (m *Model) getByQuery(query string, params []interface{}) error {
return database.GetByQuery(m, query, params)
// TableName overrides the table name used by gorm
func (Model) TableName() string {
return "host"
}
// LoadByID will load from an ID
func (m *Model) LoadByID(id int) error {
query := fmt.Sprintf("SELECT * FROM `%s` WHERE id = ? AND is_deleted = ? LIMIT 1", tableName)
params := []interface{}{id, 0}
return m.getByQuery(query, params)
}
// Touch will update model's timestamp(s)
func (m *Model) Touch(created bool) {
var d types.DBDate
d.Time = time.Now()
if created {
m.CreatedOn = d
}
m.ModifiedOn = d
func (m *Model) LoadByID(id uint) error {
db := database.GetDB()
result := db.First(&m, id)
return result.Error
}
// Save will save this model to the DB
func (m *Model) Save(skipConfiguration bool) error {
var err error
if m.UserID == 0 {
return eris.Errorf("User ID must be specified")
}
@ -97,23 +80,20 @@ func (m *Model) Save(skipConfiguration bool) error {
m.Status = status.StatusReady
}
if m.ID == 0 {
m.ID, err = create(m)
} else {
err = update(m)
}
return err
db := database.GetDB()
result := db.Save(m)
return result.Error
}
// Delete will mark a host as deleted
// Delete will mark row as deleted
func (m *Model) Delete() bool {
m.Touch(false)
m.IsDeleted = true
if err := m.Save(false); err != nil {
if m.ID == 0 {
// Can't delete a new object
return false
}
return true
db := database.GetDB()
result := db.Delete(m)
return result.Error == nil
}
// Expand will fill in more properties
@ -160,8 +140,8 @@ func (m *Model) GetTemplate() Template {
t := Template{
ID: m.ID,
CreatedOn: m.CreatedOn.Time.String(),
ModifiedOn: m.ModifiedOn.Time.String(),
CreatedAt: fmt.Sprintf("%d", m.CreatedAt), // todo: format as nice string
UpdatedAt: fmt.Sprintf("%d", m.UpdatedAt), // todo: format as nice string
UserID: m.UserID,
Type: m.Type,
NginxTemplateID: m.NginxTemplateID,

View File

@ -1,15 +0,0 @@
package host
import (
"npm/internal/model"
)
// ListResponse is the JSON response for this list
type ListResponse struct {
Total int `json:"total"`
Offset int `json:"offset"`
Limit int `json:"limit"`
Sort []model.Sort `json:"sort"`
Filter []model.Filter `json:"filter,omitempty"`
Items []Model `json:"items,omitempty"`
}

View File

@ -4,20 +4,20 @@ import "npm/internal/entity/upstream"
// Template is the model given to the template parser, converted from the Model
type Template struct {
ID int
CreatedOn string
ModifiedOn string
UserID int
ID uint
CreatedAt string
UpdatedAt string
UserID uint
Type string
NginxTemplateID int
NginxTemplateID uint
ProxyScheme string
ProxyHost string
ProxyPort int
ListenInterface string
DomainNames []string
UpstreamID int
CertificateID int
AccessListID int
UpstreamID uint
CertificateID uint
AccessListID uint
SSLForced bool
CachingEnabled bool
BlockExploits bool