nre db bkp
This commit is contained in:
parent
9cffbbef74
commit
c092239ed5
25 changed files with 927 additions and 67 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -31,3 +31,5 @@ apps/infoalloggi/gi_comuni_cap.json
|
|||
apps/infoalloggi/gi_nazioni.json
|
||||
apps/infoalloggi/tmp/
|
||||
|
||||
|
||||
apps/db/backups/
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
type Storage struct {
|
||||
type MediaStorage struct {
|
||||
ID uuid.UUID `sql:"primary_key"`
|
||||
CreatedAt time.Time
|
||||
Filename string
|
||||
|
|
@ -11,9 +11,9 @@ import (
|
|||
"github.com/go-jet/jet/v2/postgres"
|
||||
)
|
||||
|
||||
var Storage = newStorageTable("public", "storage", "")
|
||||
var MediaStorage = newMediaStorageTable("public", "media_storage", "")
|
||||
|
||||
type storageTable struct {
|
||||
type mediaStorageTable struct {
|
||||
postgres.Table
|
||||
|
||||
// Columns
|
||||
|
|
@ -31,40 +31,40 @@ type storageTable struct {
|
|||
DefaultColumns postgres.ColumnList
|
||||
}
|
||||
|
||||
type StorageTable struct {
|
||||
storageTable
|
||||
type MediaStorageTable struct {
|
||||
mediaStorageTable
|
||||
|
||||
EXCLUDED storageTable
|
||||
EXCLUDED mediaStorageTable
|
||||
}
|
||||
|
||||
// AS creates new StorageTable with assigned alias
|
||||
func (a StorageTable) AS(alias string) *StorageTable {
|
||||
return newStorageTable(a.SchemaName(), a.TableName(), alias)
|
||||
// AS creates new MediaStorageTable with assigned alias
|
||||
func (a MediaStorageTable) AS(alias string) *MediaStorageTable {
|
||||
return newMediaStorageTable(a.SchemaName(), a.TableName(), alias)
|
||||
}
|
||||
|
||||
// Schema creates new StorageTable with assigned schema name
|
||||
func (a StorageTable) FromSchema(schemaName string) *StorageTable {
|
||||
return newStorageTable(schemaName, a.TableName(), a.Alias())
|
||||
// Schema creates new MediaStorageTable with assigned schema name
|
||||
func (a MediaStorageTable) FromSchema(schemaName string) *MediaStorageTable {
|
||||
return newMediaStorageTable(schemaName, a.TableName(), a.Alias())
|
||||
}
|
||||
|
||||
// WithPrefix creates new StorageTable with assigned table prefix
|
||||
func (a StorageTable) WithPrefix(prefix string) *StorageTable {
|
||||
return newStorageTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
|
||||
// WithPrefix creates new MediaStorageTable with assigned table prefix
|
||||
func (a MediaStorageTable) WithPrefix(prefix string) *MediaStorageTable {
|
||||
return newMediaStorageTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
|
||||
}
|
||||
|
||||
// WithSuffix creates new StorageTable with assigned table suffix
|
||||
func (a StorageTable) WithSuffix(suffix string) *StorageTable {
|
||||
return newStorageTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
|
||||
// WithSuffix creates new MediaStorageTable with assigned table suffix
|
||||
func (a MediaStorageTable) WithSuffix(suffix string) *MediaStorageTable {
|
||||
return newMediaStorageTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
|
||||
}
|
||||
|
||||
func newStorageTable(schemaName, tableName, alias string) *StorageTable {
|
||||
return &StorageTable{
|
||||
storageTable: newStorageTableImpl(schemaName, tableName, alias),
|
||||
EXCLUDED: newStorageTableImpl("", "excluded", ""),
|
||||
func newMediaStorageTable(schemaName, tableName, alias string) *MediaStorageTable {
|
||||
return &MediaStorageTable{
|
||||
mediaStorageTable: newMediaStorageTableImpl(schemaName, tableName, alias),
|
||||
EXCLUDED: newMediaStorageTableImpl("", "excluded", ""),
|
||||
}
|
||||
}
|
||||
|
||||
func newStorageTableImpl(schemaName, tableName, alias string) storageTable {
|
||||
func newMediaStorageTableImpl(schemaName, tableName, alias string) mediaStorageTable {
|
||||
var (
|
||||
IDColumn = postgres.StringColumn("id")
|
||||
CreatedAtColumn = postgres.TimestampColumn("created_at")
|
||||
|
|
@ -79,7 +79,7 @@ func newStorageTableImpl(schemaName, tableName, alias string) storageTable {
|
|||
defaultColumns = postgres.ColumnList{IDColumn, CreatedAtColumn}
|
||||
)
|
||||
|
||||
return storageTable{
|
||||
return mediaStorageTable{
|
||||
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
|
||||
|
||||
//Columns
|
||||
|
|
@ -12,5 +12,5 @@ package table
|
|||
func UseSchema(schema string) {
|
||||
Annunci = Annunci.FromSchema(schema)
|
||||
MediaRefs = MediaRefs.FromSchema(schema)
|
||||
Storage = Storage.FromSchema(schema)
|
||||
MediaStorage = MediaStorage.FromSchema(schema)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
#!/bin/bash
|
||||
jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum
|
||||
jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=media_storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum
|
||||
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool, i
|
|||
to_insert = append(to_insert, value)
|
||||
}
|
||||
if len(to_insert) > 0 {
|
||||
stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert).ON_CONFLICT(Annunci.ID).DO_UPDATE(SET(Annunci.MutableColumns.SET(Annunci.EXCLUDED.MutableColumns))).ON_CONFLICT(Annunci.Codice).DO_UPDATE(SET(Annunci.MutableColumns.SET(Annunci.EXCLUDED.MutableColumns)))
|
||||
stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert).ON_CONFLICT(Annunci.Codice).DO_UPDATE(SET(Annunci.MutableColumns.SET(Annunci.EXCLUDED.MutableColumns)))
|
||||
if isDryRun {
|
||||
log.Printf("Dry run: would have inserted %d records\n", len(to_insert))
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) {
|
|||
if err != nil {
|
||||
return uuid.UUID{}, fmt.Errorf("failed to read file: %w", err)
|
||||
}
|
||||
to_insert := models.Storage{
|
||||
to_insert := models.MediaStorage{
|
||||
MimeType: mimeType,
|
||||
Tag: &tag,
|
||||
FileData: bytes,
|
||||
|
|
@ -68,11 +68,11 @@ func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) {
|
|||
FileSizeBytes: int32(len(bytes)),
|
||||
}
|
||||
|
||||
stmt := Storage.INSERT(Storage.MimeType,
|
||||
Storage.Tag,
|
||||
Storage.FileData,
|
||||
Storage.Filename,
|
||||
Storage.FileSizeBytes).MODEL(to_insert).RETURNING(Storage.ID)
|
||||
stmt := MediaStorage.INSERT(MediaStorage.MimeType,
|
||||
MediaStorage.Tag,
|
||||
MediaStorage.FileData,
|
||||
MediaStorage.Filename,
|
||||
MediaStorage.FileSizeBytes).MODEL(to_insert).RETURNING(MediaStorage.ID)
|
||||
var newIdStrings []string
|
||||
|
||||
err = stmt.Query(postgres, &newIdStrings)
|
||||
|
|
|
|||
24
apps/db-backup/Dockerfile
Normal file
24
apps/db-backup/Dockerfile
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY main.go .
|
||||
|
||||
# Compile a highly optimized, completely static binary
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o backup-app main.go
|
||||
|
||||
# --- Stage 2: Final Light Execution Layer ---
|
||||
FROM postgres:18-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the compiled executable from the builder layer
|
||||
COPY --from=builder /src/backup-app .
|
||||
|
||||
# Set up environmental time out-of-the-box
|
||||
ENV TZ=Europe/Rome
|
||||
ENV PGTZ=Europe/Rome
|
||||
|
||||
ENTRYPOINT ["/app/backup-app"]
|
||||
CMD ["cron-init"]
|
||||
28
apps/db-backup/go.mod
Normal file
28
apps/db-backup/go.mod
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
module db-backup
|
||||
|
||||
go 1.25.10
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.18
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
|
||||
github.com/aws/smithy-go v1.25.1 // indirect
|
||||
)
|
||||
38
apps/db-backup/go.sum
Normal file
38
apps/db-backup/go.sum
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
595
apps/db-backup/main.go
Normal file
595
apps/db-backup/main.go
Normal file
|
|
@ -0,0 +1,595 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
// Config holds environmental configurations
|
||||
type Config struct {
|
||||
Host string
|
||||
Port string
|
||||
User string
|
||||
Password string
|
||||
DB string
|
||||
BackupDir string
|
||||
KeepCount int
|
||||
Schedule string
|
||||
AWSAccessKey string
|
||||
AWSSecretKey string
|
||||
S3Bucket string
|
||||
S3Endpoint string
|
||||
AWSRegion string
|
||||
}
|
||||
|
||||
func loadConfig() Config {
|
||||
keepCount, _ := strconv.Atoi(getEnv("BACKUP_KEEP_COUNT", "7"))
|
||||
return Config{
|
||||
Host: getEnv("POSTGRES_HOST", "db"),
|
||||
Port: getEnv("POSTGRES_PORT", "5432"),
|
||||
User: getEnv("POSTGRES_USER", "postgres"),
|
||||
Password: getEnv("POSTGRES_PASSWORD", ""),
|
||||
DB: getEnv("POSTGRES_DB", ""),
|
||||
BackupDir: getEnv("BACKUP_DIR", "/backups"),
|
||||
KeepCount: keepCount,
|
||||
Schedule: os.Getenv("BACKUP_SCHEDULE"),
|
||||
AWSAccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
|
||||
AWSSecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
S3Bucket: os.Getenv("S3_BUCKET"),
|
||||
S3Endpoint: os.Getenv("S3_ENDPOINT"),
|
||||
AWSRegion: getEnv("AWS_DEFAULT_REGION", "us-east-1"),
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
|
||||
printHelp()
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
cfg := loadConfig()
|
||||
mode := os.Args[1]
|
||||
|
||||
switch mode {
|
||||
case "cron-init":
|
||||
initScheduler(cfg)
|
||||
case "run":
|
||||
if err := runBackupPipeline(cfg); err != nil {
|
||||
logError("Pipeline failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "inspect":
|
||||
if len(os.Args) < 3 {
|
||||
logError("ERROR: No backup filename/S3 URL provided!")
|
||||
fmt.Println("Usage: backup-app inspect <filename.sql.gz | s3://bucket/key>")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := runInspectPipeline(cfg, os.Args[2]); err != nil {
|
||||
logError("Inspection failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "restore":
|
||||
if len(os.Args) < 3 {
|
||||
logError("ERROR: No backup filename/S3 URL provided!")
|
||||
fmt.Println("Usage: backup-app restore <filename.sql.gz | s3://bucket/key>")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := runRestorePipeline(cfg, os.Args[2]); err != nil {
|
||||
logError("Restore failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
default:
|
||||
fmt.Printf("Unknown mode: %s\n", mode)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// --- MODES OF OPERATION ---
|
||||
|
||||
func initScheduler(cfg Config) {
|
||||
if cfg.Schedule == "" {
|
||||
fmt.Println("No BACKUP_SCHEDULE set.")
|
||||
// Create a channel that listens for OS shutdown signals
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
<-sigChan // This blocks cleanly forever until Docker stops the container
|
||||
fmt.Println("Shutting down gracefully...")
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("Initializing backup scheduler...")
|
||||
c := cron.New(cron.WithSeconds()) // Supports standard cron or standard with seconds
|
||||
|
||||
_, err := c.AddFunc(cfg.Schedule, func() {
|
||||
logInfo("Scheduled job triggered.")
|
||||
if err := runBackupPipeline(cfg); err != nil {
|
||||
logError("Scheduled backup pipeline failed: %v", err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
logError("Failed to parse cron schedule '%s': %v", cfg.Schedule, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("Cron scheduled with pattern: %s\n", cfg.Schedule)
|
||||
fmt.Println("Starting native Go cron engine...")
|
||||
c.Run()
|
||||
}
|
||||
|
||||
func runBackupPipeline(cfg Config) error {
|
||||
timestamp := time.Now().Format("2006-01-02_15-04-05")
|
||||
fileName := fmt.Sprintf("%s-%s.sql.gz", cfg.DB, timestamp)
|
||||
fullPath := filepath.Join(cfg.BackupDir, fileName)
|
||||
|
||||
logInfo("Starting database backup...")
|
||||
if err := os.MkdirAll(cfg.BackupDir, os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create backup dir: %w", err)
|
||||
}
|
||||
|
||||
// 1. Generate Local Backup (Stream pg_dump through Go's gzip engine)
|
||||
outFile, err := os.Create(fullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create backup file: %w", err)
|
||||
}
|
||||
|
||||
gzipWriter := gzip.NewWriter(outFile)
|
||||
|
||||
args := []string{
|
||||
"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, cfg.DB,
|
||||
"--exclude-table-data=(media_storage|media_refs|tiles_cache)",
|
||||
}
|
||||
cmd := exec.Command("pg_dump", args...)
|
||||
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
|
||||
cmd.Stdout = gzipWriter
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
gzipWriter.Close()
|
||||
outFile.Close()
|
||||
os.Remove(fullPath)
|
||||
return fmt.Errorf("pg_dump execution failed: %w", err)
|
||||
}
|
||||
gzipWriter.Close()
|
||||
outFile.Close()
|
||||
logInfo("Success: Created local backup at %s", fullPath)
|
||||
|
||||
// 2. INTEGRITY TEST
|
||||
logInfo("Launching backup integrity test...")
|
||||
testDB := fmt.Sprintf("test_verify_%s", cfg.DB)
|
||||
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", testDB))
|
||||
if err := runPsqlCmd(cfg, "postgres", fmt.Sprintf("CREATE DATABASE %s;", testDB)); err != nil {
|
||||
return fmt.Errorf("integrity test setup failed: %w", err)
|
||||
}
|
||||
|
||||
// Stream gunzip restore to test DB
|
||||
if err := streamRestoreFile(cfg, fullPath, testDB); err != nil {
|
||||
logError("CRITICAL WARNING: Backup file was created but FAILED the restoration test! File is corrupted.")
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", testDB))
|
||||
return fmt.Errorf("integrity verification failed: %w", err)
|
||||
}
|
||||
|
||||
logInfo("PASSED: Backup file successfully passed restoration test syntax verification.")
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE %s;", testDB))
|
||||
|
||||
// 3. S3 UPLOAD (if AWS config is present)
|
||||
if cfg.AWSAccessKey != "" {
|
||||
logInfo("S3 configuration detected. Uploading to cloud storage...")
|
||||
if err := uploadToS3(cfg, fullPath, fileName); err != nil {
|
||||
return fmt.Errorf("S3 synchronization failed: %w", err)
|
||||
}
|
||||
logInfo("Cloud sync success: Uploaded to s3://%s/", cfg.S3Bucket)
|
||||
}
|
||||
|
||||
// 4. RETENTION MAINTENANCE
|
||||
logInfo("Evaluating retention policy (Keeping the latest %d backups)...", cfg.KeepCount)
|
||||
if err := maintainRetention(cfg); err != nil {
|
||||
logError("Retention maintenance error: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func runInspectPipeline(cfg Config, target string) error {
|
||||
localPath := target
|
||||
var err error
|
||||
|
||||
if strings.HasPrefix(target, "s3://") {
|
||||
logInfo("Target is on S3. Fetching metadata header...")
|
||||
bucket, key := parseS3URI(target)
|
||||
return inspectS3File(cfg, bucket, key)
|
||||
}
|
||||
|
||||
if !filepath.IsAbs(localPath) {
|
||||
localPath = filepath.Join(cfg.BackupDir, target)
|
||||
}
|
||||
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open backup file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid gzip compression format: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
fmt.Printf("\n--- BACKUP INSPECTION REPORT ---\n")
|
||||
fmt.Printf("File Source: %s\n", filepath.Base(localPath))
|
||||
fmt.Printf("Compressed Size: %.2f MB\n", float64(stat.Size())/(1024*1024))
|
||||
if !gzReader.ModTime.IsZero() {
|
||||
fmt.Printf("Gzip Timestamp: %s\n", gzReader.ModTime.Format(time.RFC1123))
|
||||
}
|
||||
|
||||
// Read first 2KB to look for pg_dump header details
|
||||
buffer := make([]byte, 2048)
|
||||
n, _ := io.ReadFull(gzReader, buffer)
|
||||
if n > 0 {
|
||||
content := string(buffer[:n])
|
||||
fmt.Println("\nHeader Snippet (First 5 comment lines):")
|
||||
lines := strings.Split(content, "\n")
|
||||
count := 0
|
||||
for _, line := range lines {
|
||||
if strings.HasPrefix(line, "--") {
|
||||
fmt.Printf(" %s\n", line)
|
||||
count++
|
||||
if count >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("--------------------------------\n\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func runRestorePipeline(cfg Config, target string) error {
|
||||
localPath := target
|
||||
|
||||
// Handle S3 targets seamlessly
|
||||
if strings.HasPrefix(target, "s3://") {
|
||||
logInfo("S3 URI detected (%s). Downloading archive to disk buffer...", target)
|
||||
bucket, key := parseS3URI(target)
|
||||
|
||||
tmpFile, err := os.CreateTemp(cfg.BackupDir, "s3-restore-*.sql.gz")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create secure disk temporary buffer: %w", err)
|
||||
}
|
||||
defer os.Remove(tmpFile.Name()) // Auto cleanup
|
||||
defer tmpFile.Close()
|
||||
|
||||
if err := downloadFromS3(cfg, bucket, key, tmpFile); err != nil {
|
||||
return fmt.Errorf("S3 file extraction streaming failed: %w", err)
|
||||
}
|
||||
localPath = tmpFile.Name()
|
||||
logInfo("Download finalized. Moving to verification sequence...")
|
||||
} else if !filepath.IsAbs(localPath) {
|
||||
localPath = filepath.Join(cfg.BackupDir, target)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(localPath); os.IsNotExist(err) {
|
||||
return fmt.Errorf("file not found at target resolution scope: %s", localPath)
|
||||
}
|
||||
|
||||
logInfo("STEP 1: Verifying integrity of backup file inside sandbox database...")
|
||||
sandboxDB := fmt.Sprintf("restore_sandbox_%s", cfg.DB)
|
||||
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", sandboxDB))
|
||||
if err := runPsqlCmd(cfg, "postgres", fmt.Sprintf("CREATE DATABASE %s;", sandboxDB)); err != nil {
|
||||
return fmt.Errorf("sandbox creation failed: %w", err)
|
||||
}
|
||||
|
||||
if err := streamRestoreFile(cfg, localPath, sandboxDB); err != nil {
|
||||
logError("CRITICAL FAILURE: The backup file failed verification tests! Main database will not be touched.")
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", sandboxDB))
|
||||
return fmt.Errorf("sandbox restore test failed: %w", err)
|
||||
}
|
||||
logInfo("PASSED: Verification successful. Sandbox database populated cleanly.")
|
||||
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE %s;", sandboxDB))
|
||||
|
||||
logInfo("STEP 2: Proceeding with Live Database Restoration...")
|
||||
logInfo("WARNING: Dropping active public schema on target database: %s...", cfg.DB)
|
||||
if err := runPsqlCmd(cfg, cfg.DB, "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"); err != nil {
|
||||
return fmt.Errorf("failed to flush target schema: %w", err)
|
||||
}
|
||||
|
||||
logInfo("STEP 3: Streaming verified records into production engine...")
|
||||
if err := streamRestoreFile(cfg, localPath, cfg.DB); err != nil {
|
||||
return fmt.Errorf("critical engine restoration stream failed: %w", err)
|
||||
}
|
||||
|
||||
logInfo("SUCCESS: Production database successfully restored to status: %s", filepath.Base(target))
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- HELPERS & UTILITIES ---
|
||||
|
||||
func runPsqlCmd(cfg Config, dbname, sqlCommand string) error {
|
||||
args := []string{"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, "-d", dbname, "-c", sqlCommand}
|
||||
cmd := exec.Command("psql", args...)
|
||||
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func streamRestoreFile(cfg Config, gzippedFilePath, targetDB string) error {
|
||||
file, err := os.Open(gzippedFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
args := []string{"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, "-d", targetDB}
|
||||
cmd := exec.Command("psql", args...)
|
||||
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
|
||||
|
||||
stdinPipe, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(stdinPipe, gzReader); err != nil {
|
||||
stdinPipe.Close()
|
||||
return err
|
||||
}
|
||||
stdinPipe.Close()
|
||||
|
||||
return cmd.Wait()
|
||||
}
|
||||
|
||||
func getS3Client(cfg Config) (*s3.Client, error) {
|
||||
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(cfg.AWSRegion),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AWSAccessKey, cfg.AWSSecretKey, "")),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if cfg.S3Endpoint != "" {
|
||||
o.BaseEndpoint = aws.String(cfg.S3Endpoint)
|
||||
o.UsePathStyle = true
|
||||
}
|
||||
})
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func uploadToS3(cfg Config, localPath, s3Key string) error {
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 1. Load the default credentials and region configuration
|
||||
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(cfg.AWSRegion),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AWSAccessKey, cfg.AWSSecretKey, "")),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. Initialize the S3 Client with the modern BaseEndpoint design pattern
|
||||
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
if cfg.S3Endpoint != "" {
|
||||
// This string replaces the old EndpointResolver logic perfectly
|
||||
o.BaseEndpoint = aws.String(cfg.S3Endpoint)
|
||||
|
||||
// If you use MinIO or DigitalOcean, you usually want virtual-host style path addressing disabled
|
||||
o.UsePathStyle = true
|
||||
}
|
||||
})
|
||||
|
||||
// 3. Dispatch the payload to your cloud storage
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(cfg.S3Bucket),
|
||||
Key: aws.String(s3Key),
|
||||
Body: file,
|
||||
})
|
||||
return err
|
||||
}
|
||||
func downloadFromS3(cfg Config, bucket, key string, targetFile *os.File) error {
|
||||
client, err := getS3Client(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
_, err = io.Copy(targetFile, result.Body)
|
||||
return err
|
||||
}
|
||||
func inspectS3File(cfg Config, bucket, key string) error {
|
||||
client, err := getS3Client(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch target from S3: %w", err)
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
gzReader, err := gzip.NewReader(result.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloaded target is not a valid gzip configuration file: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
fmt.Printf("\n--- S3 REMOTE BACKUP INSPECTION REPORT ---\n")
|
||||
fmt.Printf("Bucket target: s3://%s/%s\n", bucket, key)
|
||||
if result.ContentLength != nil {
|
||||
fmt.Printf("Compressed Size: %.2f MB\n", float64(*result.ContentLength)/(1024*1024))
|
||||
}
|
||||
if result.LastModified != nil {
|
||||
fmt.Printf("S3 Modification: %s\n", result.LastModified.Format(time.RFC1123))
|
||||
}
|
||||
|
||||
buffer := make([]byte, 2048)
|
||||
n, _ := io.ReadFull(gzReader, buffer)
|
||||
if n > 0 {
|
||||
fmt.Println("\nHeader Snippet (First 5 comment lines):")
|
||||
lines := strings.Split(string(buffer[:n]), "\n")
|
||||
count := 0
|
||||
for _, line := range lines {
|
||||
if strings.HasPrefix(line, "--") {
|
||||
fmt.Printf(" %s\n", line)
|
||||
count++
|
||||
if count >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("------------------------------------------\n\n")
|
||||
return nil
|
||||
}
|
||||
func maintainRetention(cfg Config) error {
|
||||
pattern := filepath.Join(cfg.BackupDir, fmt.Sprintf("%s-*.sql.gz", cfg.DB))
|
||||
files, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logInfo("Found %d total backup files locally.", len(files))
|
||||
if len(files) <= cfg.KeepCount {
|
||||
logInfo("No files require rotation.")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort files alphabetically (since naming includes timestamps, alpha sort maps cleanly to chronological sort)
|
||||
sort.Strings(files)
|
||||
|
||||
purgeCount := len(files) - cfg.KeepCount
|
||||
logInfo("Threshold exceeded. Removing oldest %d file(s)...", purgeCount)
|
||||
|
||||
for i := 0; i < purgeCount; i++ {
|
||||
logInfo("Deleted expired file: %s", filepath.Base(files[i]))
|
||||
if err := os.Remove(files[i]); err != nil {
|
||||
logError("Failed to delete %s: %v", files[i], err)
|
||||
}
|
||||
}
|
||||
logInfo("Retention cleanup process finished.")
|
||||
return nil
|
||||
}
|
||||
func parseS3URI(uri string) (bucket, key string) {
|
||||
cleanStr := strings.TrimPrefix(uri, "s3://")
|
||||
parts := strings.SplitN(cleanStr, "/", 2)
|
||||
if len(parts) == 2 {
|
||||
return parts[0], parts[1]
|
||||
}
|
||||
return parts[0], ""
|
||||
}
|
||||
|
||||
func getEnv(key, fallback string) string {
|
||||
if value, exists := os.LookupEnv(key); exists {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func logInfo(format string, v ...interface{}) {
|
||||
fmt.Printf("[%s] %s\n", time.Now().Format("2006-01-02 15:04:05"), fmt.Sprintf(format, v...))
|
||||
}
|
||||
|
||||
func logError(format string, v ...interface{}) {
|
||||
fmt.Fprintf(os.Stderr, "[%s] ERROR: %s\n", time.Now().Format("2006-01-02 15:04:05"), fmt.Sprintf(format, v...))
|
||||
}
|
||||
|
||||
func printHelp() {
|
||||
fmt.Println(`===================================================================
|
||||
Database Backup Manager (Go Edition)
|
||||
===================================================================
|
||||
Usage:
|
||||
backup-app <command> [arguments]
|
||||
|
||||
Commands:
|
||||
cron-init Starts the persistent background cron engine using the
|
||||
cron pattern defined in $BACKUP_SCHEDULE. If no schedule
|
||||
is defined, the process waits cleanly for termination.
|
||||
|
||||
run Triggers a single manual backup sequence immediately:
|
||||
Dumps DB -> Compresses -> Verifies -> Uploads to S3 -> Rotates.
|
||||
|
||||
inspect <target> Peeks into a local archive file or remote S3 object URI
|
||||
to extract GZIP headers, modification timestamps, and the
|
||||
first few lines of pg_dump comments without a full download.
|
||||
Examples:
|
||||
backup-app inspect my-backup.sql.gz
|
||||
backup-app inspect s3://my-bucket/my-backup.sql.gz
|
||||
|
||||
restore <target> Performs a safe, multi-stage restore of a local file or
|
||||
S3 object URI. Streams to a sandbox DB for verification
|
||||
before wiping and updating the production database schema.
|
||||
Examples:
|
||||
backup-app restore my-backup.sql.gz
|
||||
backup-app restore s3://my-bucket/my-backup.sql.gz
|
||||
|
||||
help, -h, --help Display this help screen.
|
||||
|
||||
Required Environment Variables:
|
||||
POSTGRES_HOST Database server hostname/IP
|
||||
POSTGRES_USER Database administrative user
|
||||
POSTGRES_PASSWORD Database password
|
||||
POSTGRES_DB Target database name to back up/restore
|
||||
|
||||
Optional Environment Variables:
|
||||
POSTGRES_PORT Database port (Default: 5432)
|
||||
BACKUP_DIR Local storage path for backups (Default: /backups)
|
||||
BACKUP_KEEP_COUNT Number of local archives to retain (Default: 7)
|
||||
BACKUP_SCHEDULE Cron interval string (e.g., "0 2 * * *")
|
||||
|
||||
S3 Cloud Variables (Required for s3:// commands):
|
||||
AWS_ACCESS_KEY_ID Your IAM Access Key ID
|
||||
AWS_SECRET_ACCESS_KEY Your IAM Secret Access Key
|
||||
S3_BUCKET Target S3 Bucket name (used during 'run')
|
||||
AWS_DEFAULT_REGION Target AWS region (Default: us-east-1)
|
||||
S3_ENDPOINT Custom URL for S3-compatible APIs (MinIO, DigitalOcean)
|
||||
===================================================================`)
|
||||
}
|
||||
|
|
@ -4,4 +4,5 @@ docker-compose.yml
|
|||
Dockerfile
|
||||
filter_sql.awk
|
||||
log_parser.sh
|
||||
pg_dumper.sh
|
||||
pg_dumper.sh
|
||||
backups
|
||||
|
|
@ -38,6 +38,27 @@ services:
|
|||
|
||||
]
|
||||
restart: "no"
|
||||
db-backup:
|
||||
build:
|
||||
context: ../db-backup/
|
||||
dockerfile: Dockerfile
|
||||
restart: "no"
|
||||
pull_policy: build
|
||||
depends_on:
|
||||
- db
|
||||
environment:
|
||||
POSTGRES_HOST: db
|
||||
POSTGRES_PORT: 5432
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
BACKUP_DIR: /backups
|
||||
BACKUP_KEEP_COUNT: 5
|
||||
BACKUP_SCHEDULE: null
|
||||
volumes:
|
||||
- ./backups:/backups
|
||||
networks:
|
||||
- dokploy-network
|
||||
|
||||
networks:
|
||||
dokploy-network:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
docker compose -p infoalloggi-dev-db -f dev-docker-compose.yml rm -f db-backup 2>/dev/null || true
|
||||
|
||||
docker compose -p infoalloggi-dev-db -f dev-docker-compose.yml rm -f migrate 2>/dev/null || true
|
||||
|
||||
|
|
|
|||
|
|
@ -44,24 +44,24 @@ END IF;
|
|||
END $$;
|
||||
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'media_refs'
|
||||
) THEN
|
||||
EXECUTE '
|
||||
DELETE FROM public.storage s
|
||||
WHERE s.tag IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.media_refs m
|
||||
WHERE s.id = m.storage_id
|
||||
);
|
||||
';
|
||||
END IF;
|
||||
END $$;
|
||||
-- DO $$
|
||||
-- BEGIN
|
||||
-- IF EXISTS (
|
||||
-- SELECT FROM information_schema.tables
|
||||
-- WHERE table_schema = 'public'
|
||||
-- AND table_name = 'media_refs'
|
||||
-- ) THEN
|
||||
-- EXECUTE '
|
||||
-- DELETE FROM public.storage s
|
||||
-- WHERE s.tag IS NOT NULL
|
||||
-- AND NOT EXISTS (
|
||||
-- SELECT 1
|
||||
-- FROM public.media_refs m
|
||||
-- WHERE s.id = m.storage_id
|
||||
-- );
|
||||
-- ';
|
||||
-- END IF;
|
||||
-- END $$;
|
||||
|
||||
-- Drop Tables if they exist
|
||||
DROP TABLE IF EXISTS public.images_refs;
|
||||
|
|
|
|||
53
apps/db/migrations/58_storage_split.up.sql
Normal file
53
apps/db/migrations/58_storage_split.up.sql
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
CREATE TABLE IF NOT EXISTS public.media_storage (
|
||||
id uuid NOT NULL DEFAULT uuidv7(),
|
||||
created_at timestamp without time zone NOT NULL DEFAULT now(),
|
||||
filename text NOT NULL,
|
||||
mime_type text NOT NULL,
|
||||
file_size_bytes INT NOT NULL,
|
||||
file_data BYTEA NOT NULL,
|
||||
expires_at timestamp without time zone,
|
||||
tag text
|
||||
);
|
||||
|
||||
|
||||
DO $$ BEGIN IF EXISTS (
|
||||
SELECT
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = 'media_storage'
|
||||
AND column_name = 'file_data'
|
||||
) THEN
|
||||
ALTER TABLE public.media_storage
|
||||
ALTER COLUMN file_data SET STORAGE EXTERNAL;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DO $$ BEGIN IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'media_storage_pkey'
|
||||
AND conrelid = 'public.media_storage'::regclass
|
||||
) THEN
|
||||
ALTER TABLE public.media_storage
|
||||
ADD CONSTRAINT media_storage_pkey PRIMARY KEY (id);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
DELETE FROM public.media_refs
|
||||
WHERE storage_id NOT IN (SELECT id FROM public.media_storage);
|
||||
|
||||
DELETE FROM public.storage
|
||||
WHERE tag is not null;
|
||||
|
||||
ALTER TABLE media_refs
|
||||
DROP CONSTRAINT IF EXISTS media_refs_storage;
|
||||
|
||||
|
||||
DO $$ BEGIN IF NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_constraint
|
||||
WHERE conname = 'media_refs_storage'
|
||||
) THEN
|
||||
ALTER TABLE public.media_refs
|
||||
ADD CONSTRAINT media_refs_storage FOREIGN KEY (storage_id) REFERENCES public.media_storage (id) ON UPDATE CASCADE ON DELETE CASCADE;
|
||||
END IF;
|
||||
END $$;
|
||||
|
|
@ -13,6 +13,8 @@ TODOS:
|
|||
- onboarding submit (toast e email che invitano a pagare)
|
||||
- migliorare onboarding e pag guida
|
||||
- rendere opzionale email e numero?
|
||||
- titolo tab in edit annuncio
|
||||
- on payment success webhook, provare ad aggiungere su fatture in cloud
|
||||
|
||||
AFTER MVP:
|
||||
- TODO migrazione app router https://nextjs.org/docs/pages/building-your-application/upgrading/app-router-migration#migrating-from-pages-to-app
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { env } from "~/env";
|
||||
import type { MediaStorageId } from "~/schemas/public/MediaStorage";
|
||||
import type { StorageId } from "~/schemas/public/Storage";
|
||||
|
||||
type StorageParams = {
|
||||
|
|
@ -12,7 +13,7 @@ export function getStorageUrl({
|
|||
params,
|
||||
host = env.NEXT_PUBLIC_BASE_URL,
|
||||
}: {
|
||||
id: StorageId;
|
||||
id: StorageId | MediaStorageId;
|
||||
params?: Partial<StorageParams>;
|
||||
host?: string;
|
||||
}) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { storageId, type StorageId } from './Storage';
|
||||
import { mediaStorageId, type MediaStorageId } from './MediaStorage';
|
||||
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
|
||||
import { z } from 'zod';
|
||||
|
||||
|
|
@ -8,7 +8,7 @@ export default interface MediaRefsTable {
|
|||
|
||||
ordine: ColumnType<number, number, number>;
|
||||
|
||||
storage_id: ColumnType<StorageId, StorageId, StorageId>;
|
||||
storage_id: ColumnType<MediaStorageId, MediaStorageId, MediaStorageId>;
|
||||
|
||||
og_url: ColumnType<string, string, string>;
|
||||
|
||||
|
|
@ -26,7 +26,7 @@ export type MediaRefsUpdate = Updateable<MediaRefsTable>;
|
|||
export const MediaRefsSchema = z.object({
|
||||
codice: z.string(),
|
||||
ordine: z.number(),
|
||||
storage_id: storageId,
|
||||
storage_id: mediaStorageId,
|
||||
og_url: z.string(),
|
||||
media_type: z.string(),
|
||||
is_thumbnail: z.boolean(),
|
||||
|
|
@ -35,7 +35,7 @@ export const MediaRefsSchema = z.object({
|
|||
export const NewMediaRefsSchema = z.object({
|
||||
codice: z.string(),
|
||||
ordine: z.number(),
|
||||
storage_id: storageId,
|
||||
storage_id: mediaStorageId,
|
||||
og_url: z.string(),
|
||||
media_type: z.string(),
|
||||
is_thumbnail: z.boolean().optional(),
|
||||
|
|
@ -44,7 +44,7 @@ export const NewMediaRefsSchema = z.object({
|
|||
export const MediaRefsUpdateSchema = z.object({
|
||||
codice: z.string().optional(),
|
||||
ordine: z.number().optional(),
|
||||
storage_id: storageId.optional(),
|
||||
storage_id: mediaStorageId.optional(),
|
||||
og_url: z.string().optional(),
|
||||
media_type: z.string().optional(),
|
||||
is_thumbnail: z.boolean().optional(),
|
||||
|
|
|
|||
65
apps/infoalloggi/src/schemas/public/MediaStorage.ts
Normal file
65
apps/infoalloggi/src/schemas/public/MediaStorage.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
|
||||
import { z } from 'zod';
|
||||
|
||||
/** Identifier type for public.media_storage */
|
||||
export type MediaStorageId = string & { __brand: 'public.media_storage' };
|
||||
|
||||
/** Represents the table public.media_storage */
|
||||
export default interface MediaStorageTable {
|
||||
id: ColumnType<MediaStorageId, MediaStorageId | undefined, MediaStorageId>;
|
||||
|
||||
created_at: ColumnType<Date, Date | string | undefined, Date | string>;
|
||||
|
||||
filename: ColumnType<string, string, string>;
|
||||
|
||||
mime_type: ColumnType<string, string, string>;
|
||||
|
||||
file_size_bytes: ColumnType<number, number, number>;
|
||||
|
||||
file_data: ColumnType<Buffer, Buffer, Buffer>;
|
||||
|
||||
expires_at: ColumnType<Date | null, Date | string | null, Date | string | null>;
|
||||
|
||||
tag: ColumnType<string | null, string | null, string | null>;
|
||||
}
|
||||
|
||||
export type MediaStorage = Selectable<MediaStorageTable>;
|
||||
|
||||
export type NewMediaStorage = Insertable<MediaStorageTable>;
|
||||
|
||||
export type MediaStorageUpdate = Updateable<MediaStorageTable>;
|
||||
|
||||
export const mediaStorageId = z.uuid().transform(value => value as MediaStorageId);
|
||||
|
||||
export const MediaStorageSchema = z.object({
|
||||
id: mediaStorageId,
|
||||
created_at: z.date(),
|
||||
filename: z.string(),
|
||||
mime_type: z.string(),
|
||||
file_size_bytes: z.number(),
|
||||
file_data: z.instanceof(Buffer),
|
||||
expires_at: z.date().nullable(),
|
||||
tag: z.string().nullable(),
|
||||
});
|
||||
|
||||
export const NewMediaStorageSchema = z.object({
|
||||
id: mediaStorageId.optional(),
|
||||
created_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional(),
|
||||
filename: z.string(),
|
||||
mime_type: z.string(),
|
||||
file_size_bytes: z.number(),
|
||||
file_data: z.instanceof(Buffer),
|
||||
expires_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional().nullable(),
|
||||
tag: z.string().optional().nullable(),
|
||||
});
|
||||
|
||||
export const MediaStorageUpdateSchema = z.object({
|
||||
id: mediaStorageId.optional(),
|
||||
created_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional(),
|
||||
filename: z.string().optional(),
|
||||
mime_type: z.string().optional(),
|
||||
file_size_bytes: z.number().optional(),
|
||||
file_data: z.instanceof(Buffer).optional(),
|
||||
expires_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional().nullable(),
|
||||
tag: z.string().optional().nullable(),
|
||||
});
|
||||
|
|
@ -20,6 +20,7 @@ import type { default as PrezziarioTable } from './Prezziario';
|
|||
import type { default as RatelimiterTable } from './Ratelimiter';
|
||||
import type { default as ChatsTable } from './Chats';
|
||||
import type { default as EtichetteTable } from './Etichette';
|
||||
import type { default as MediaStorageTable } from './MediaStorage';
|
||||
import type { default as MessagesTable } from './Messages';
|
||||
import type { default as UsersAnagraficaTable } from './UsersAnagrafica';
|
||||
import type { default as ProvincieTable } from './Provincie';
|
||||
|
|
@ -77,6 +78,8 @@ export default interface PublicSchema {
|
|||
|
||||
etichette: EtichetteTable;
|
||||
|
||||
media_storage: MediaStorageTable;
|
||||
|
||||
messages: MessagesTable;
|
||||
|
||||
users_anagrafica: UsersAnagraficaTable;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { z } from "zod/v4";
|
||||
import { AnnunciUpdateSchema, annunciId } from "~/schemas/public/Annunci";
|
||||
import { mediaStorageId } from "~/schemas/public/MediaStorage";
|
||||
import { servizioServizioId } from "~/schemas/public/Servizio";
|
||||
import { storageId } from "~/schemas/public/Storage";
|
||||
import {
|
||||
adminProcedure,
|
||||
apiProcedure,
|
||||
|
|
@ -174,7 +174,7 @@ export const annunciRouter = createTRPCRouter({
|
|||
return { status: "ok", timestamp: Date.now() };
|
||||
}),
|
||||
removeAnnuncioMedia: adminProcedure
|
||||
.input(storageId)
|
||||
.input(mediaStorageId)
|
||||
.mutation(async ({ input }) => {
|
||||
return await removeAnnuncioMedia(input);
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ import type {
|
|||
AnnunciId,
|
||||
AnnunciUpdate,
|
||||
} from "~/schemas/public/Annunci";
|
||||
import type { MediaStorageId } from "~/schemas/public/MediaStorage";
|
||||
import type { ServizioServizioId } from "~/schemas/public/Servizio";
|
||||
import type { StorageId } from "~/schemas/public/Storage";
|
||||
import { db } from "~/server/db";
|
||||
import type { AnnuncioTemplateData } from "~/server/services/typst.service";
|
||||
import { withImages, withVideos } from "~/utils/kysely-helper";
|
||||
|
|
@ -17,13 +17,13 @@ import { revalidate } from "../utils/revalidationHelper";
|
|||
|
||||
export type ImgMedia = {
|
||||
ordine: number;
|
||||
img: StorageId;
|
||||
thumb: StorageId | null;
|
||||
img: MediaStorageId;
|
||||
thumb: MediaStorageId | null;
|
||||
};
|
||||
export type VideoMedia = {
|
||||
ordine: number;
|
||||
video: StorageId;
|
||||
thumb: StorageId | null;
|
||||
video: MediaStorageId;
|
||||
thumb: MediaStorageId | null;
|
||||
};
|
||||
export type AnnunciWithMedia = Annunci & {
|
||||
tipo: string;
|
||||
|
|
@ -546,7 +546,7 @@ export const getProprietarioDataHandler = async ({
|
|||
}
|
||||
};
|
||||
|
||||
export const removeAnnuncioMedia = async (id: StorageId) => {
|
||||
export const removeAnnuncioMedia = async (id: MediaStorageId) => {
|
||||
try {
|
||||
await db
|
||||
.$pickTables<"media_refs">()
|
||||
|
|
|
|||
|
|
@ -1224,6 +1224,7 @@ export const getRichieste = async (): Promise<RichiesteData[]> => {
|
|||
| "annunci"
|
||||
| "ordini"
|
||||
| "prezziario"
|
||||
| "media_refs"
|
||||
>()
|
||||
.selectFrom("servizio")
|
||||
.innerJoin("users", "users.id", "servizio.user_id")
|
||||
|
|
|
|||
|
|
@ -34,11 +34,36 @@ services:
|
|||
[
|
||||
"-verbose",
|
||||
"-path", "/migrations",
|
||||
"-database", "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable",
|
||||
"-database", "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:${PGPORT}/${POSTGRES_DB}?sslmode=disable",
|
||||
"up"
|
||||
]
|
||||
restart: "no"
|
||||
|
||||
db-backup:
|
||||
build: ./apps/db-backup/
|
||||
restart: always
|
||||
pull_policy: build
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
POSTGRES_HOST: db
|
||||
POSTGRES_PORT: ${PGPORT}
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
BACKUP_DIR: /backups
|
||||
BACKUP_KEEP_COUNT: ${BACKUP_KEEP_COUNT}
|
||||
BACKUP_SCHEDULE: ${BACKUP_SCHEDULE}
|
||||
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
|
||||
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
volumes:
|
||||
- ./backups:/backups
|
||||
networks:
|
||||
- dokploy-network
|
||||
|
||||
backend:
|
||||
image: "${IMAGE_REGISTRY:-ghcr.io}/${OWNER:-marcopedone}/${PREFIX:-infoalloggi}-backend:${TAG:-latest}"
|
||||
networks:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue