Compare commits

...

10 commits

Author SHA1 Message Date
4ebdf32495 remove: + Annuncio
Some checks failed
CI / build (push) Has been cancelled
2026-05-26 16:35:27 +02:00
63a1bf2a8a fix media storage get 2026-05-26 14:28:13 +02:00
abb9dcfd36 skip update 2026-05-26 12:38:41 +02:00
43568aeb36 log and health check 2026-05-26 12:36:52 +02:00
97079fd3fa health 2026-05-26 12:12:23 +02:00
a37eaac2a3 backend update 2026-05-26 12:04:53 +02:00
c092239ed5 nre db bkp 2026-05-26 10:42:00 +02:00
9cffbbef74 a 2026-05-22 20:40:58 +02:00
cc246ee6fb a 2026-05-22 20:31:58 +02:00
31948a0845 asd 2026-05-22 17:39:17 +02:00
31 changed files with 1065 additions and 182 deletions

2
.gitignore vendored
View file

@ -31,3 +31,5 @@ apps/infoalloggi/gi_comuni_cap.json
apps/infoalloggi/gi_nazioni.json
apps/infoalloggi/tmp/
apps/db/backups/

View file

@ -12,7 +12,7 @@ import (
"time"
)
type Storage struct {
type MediaStorage struct {
ID uuid.UUID `sql:"primary_key"`
CreatedAt time.Time
Filename string

View file

@ -11,9 +11,9 @@ import (
"github.com/go-jet/jet/v2/postgres"
)
var Storage = newStorageTable("public", "storage", "")
var MediaStorage = newMediaStorageTable("public", "media_storage", "")
type storageTable struct {
type mediaStorageTable struct {
postgres.Table
// Columns
@ -31,40 +31,40 @@ type storageTable struct {
DefaultColumns postgres.ColumnList
}
type StorageTable struct {
storageTable
type MediaStorageTable struct {
mediaStorageTable
EXCLUDED storageTable
EXCLUDED mediaStorageTable
}
// AS creates new StorageTable with assigned alias
func (a StorageTable) AS(alias string) *StorageTable {
return newStorageTable(a.SchemaName(), a.TableName(), alias)
// AS creates new MediaStorageTable with assigned alias
func (a MediaStorageTable) AS(alias string) *MediaStorageTable {
return newMediaStorageTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new StorageTable with assigned schema name
func (a StorageTable) FromSchema(schemaName string) *StorageTable {
return newStorageTable(schemaName, a.TableName(), a.Alias())
// Schema creates new MediaStorageTable with assigned schema name
func (a MediaStorageTable) FromSchema(schemaName string) *MediaStorageTable {
return newMediaStorageTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new StorageTable with assigned table prefix
func (a StorageTable) WithPrefix(prefix string) *StorageTable {
return newStorageTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
// WithPrefix creates new MediaStorageTable with assigned table prefix
func (a MediaStorageTable) WithPrefix(prefix string) *MediaStorageTable {
return newMediaStorageTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new StorageTable with assigned table suffix
func (a StorageTable) WithSuffix(suffix string) *StorageTable {
return newStorageTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
// WithSuffix creates new MediaStorageTable with assigned table suffix
func (a MediaStorageTable) WithSuffix(suffix string) *MediaStorageTable {
return newMediaStorageTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newStorageTable(schemaName, tableName, alias string) *StorageTable {
return &StorageTable{
storageTable: newStorageTableImpl(schemaName, tableName, alias),
EXCLUDED: newStorageTableImpl("", "excluded", ""),
func newMediaStorageTable(schemaName, tableName, alias string) *MediaStorageTable {
return &MediaStorageTable{
mediaStorageTable: newMediaStorageTableImpl(schemaName, tableName, alias),
EXCLUDED: newMediaStorageTableImpl("", "excluded", ""),
}
}
func newStorageTableImpl(schemaName, tableName, alias string) storageTable {
func newMediaStorageTableImpl(schemaName, tableName, alias string) mediaStorageTable {
var (
IDColumn = postgres.StringColumn("id")
CreatedAtColumn = postgres.TimestampColumn("created_at")
@ -79,7 +79,7 @@ func newStorageTableImpl(schemaName, tableName, alias string) storageTable {
defaultColumns = postgres.ColumnList{IDColumn, CreatedAtColumn}
)
return storageTable{
return mediaStorageTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns

View file

@ -12,5 +12,5 @@ package table
func UseSchema(schema string) {
Annunci = Annunci.FromSchema(schema)
MediaRefs = MediaRefs.FromSchema(schema)
Storage = Storage.FromSchema(schema)
MediaStorage = MediaStorage.FromSchema(schema)
}

View file

@ -1,6 +1,6 @@
module backend
go 1.25.0
go 1.25.10
require (
github.com/go-jet/jet/v2 v2.14.1
@ -25,12 +25,14 @@ require (
golang.org/x/mod v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.45.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
require (
github.com/jackc/pgx-gofrs-uuid v0.0.0-20230224015001-1d428863c2e2
github.com/labstack/echo/v5 v5.1.1
github.com/labstack/gommon v0.5.0
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect

View file

@ -30,6 +30,8 @@ github.com/labstack/echo/v4 v4.15.1 h1:S9keusg26gZpjMmPqB5hOEvNKnmd1lNmcHrbbH2ln
github.com/labstack/echo/v4 v4.15.1/go.mod h1:xmw1clThob0BSVRX1CRQkGQ/vjwcpOMjQZSZa9fKA/c=
github.com/labstack/echo/v4 v4.15.2 h1:nnh2sCzGCVYnU+wCisMPiYapEg/QVo/gcI9ePKg5/T4=
github.com/labstack/echo/v4 v4.15.2/go.mod h1:Xzp1Ns1RA2c9fY7nSgUJkpkUZGNbEIVHZbtbOMPktBI=
github.com/labstack/echo/v5 v5.1.1 h1:4QkvKoS8ps5ch49t8b72QS9Z581ytgxhTzxuB/CBA2I=
github.com/labstack/echo/v5 v5.1.1/go.mod h1:SyvlSdObGjRXeQfCCXW/sybkZdOOQZBmpKF0bvALaeo=
github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c=
github.com/labstack/gommon v0.5.0/go.mod h1:Rzlg7HHy1maLfzBYGg9NZcVuz1sA68HHhLjhcEllYE0=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
@ -85,6 +87,8 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View file

@ -1,3 +1,3 @@
#!/bin/bash
jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum
jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=media_storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum

View file

@ -38,7 +38,7 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool, i
}
}
var to_insert []models.Annunci
to_insert := []models.Annunci{}
for _, annuncio := range annunci.Annuncio {
value := models.Annunci{
Locatore: annuncio.Locatore,
@ -102,7 +102,7 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool, i
} else {
_, err = stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to insert/update annunci batch: %w", err)
return fmt.Errorf("failed to insert/update annunci: %w", err)
}
}
}

View file

@ -60,7 +60,7 @@ func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) {
if err != nil {
return uuid.UUID{}, fmt.Errorf("failed to read file: %w", err)
}
to_insert := models.Storage{
to_insert := models.MediaStorage{
MimeType: mimeType,
Tag: &tag,
FileData: bytes,
@ -68,11 +68,11 @@ func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) {
FileSizeBytes: int32(len(bytes)),
}
stmt := Storage.INSERT(Storage.MimeType,
Storage.Tag,
Storage.FileData,
Storage.Filename,
Storage.FileSizeBytes).MODEL(to_insert).RETURNING(Storage.ID)
stmt := MediaStorage.INSERT(MediaStorage.MimeType,
MediaStorage.Tag,
MediaStorage.FileData,
MediaStorage.Filename,
MediaStorage.FileSizeBytes).MODEL(to_insert).RETURNING(MediaStorage.ID)
var newIdStrings []string
err = stmt.Query(postgres, &newIdStrings)

View file

@ -3,129 +3,108 @@ package main
import (
"backend/config"
"backend/queries"
"context"
"fmt"
"io"
"log"
"net/http"
"os"
"strconv"
"strings"
"text/template"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v5"
"github.com/labstack/echo/v5/middleware"
)
type Server struct {
config *config.Config
}
var Cfg *config.Config
var u *Updater
func NewServer(cfg *config.Config) (*Server, error) {
return &Server{
config: cfg,
}, nil
}
func main() {
Cfg = config.Load()
u = NewUpdater()
func (s *Server) SetupRoutes() *echo.Echo {
funcs := map[string]any{
"contains": strings.Contains,
"hasPrefix": strings.HasPrefix,
"hasSuffix": strings.HasSuffix,
"getImage": func(txt string) string {
return txt[7:]
},
"isWebp": func(txt string) bool {
return strings.Contains(txt, "webp")
},
}
t := &Template{
templates: template.Must(template.New("").Funcs(funcs).ParseGlob("public/views/*.html")),
}
e := echo.New()
e.Renderer = t
skipper := func(c *echo.Context) bool {
// Skip health check endpoint
return c.Request().URL.Path == "/health" || c.Request().URL.Path == "/"
}
// MISC
e.Static("/static", "public/static")
e.GET("/", func(c echo.Context) error {
return c.Render(http.StatusOK, "main", map[string]any{
"imageOption": s.config.Images.Enabled,
"concurrentImages": s.config.Images.ConcurrentLimit,
})
e.Use(middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{
LogStatus: true,
LogURI: true,
LogMethod: true,
Skipper: skipper,
HandleError: true, // forwards error to the global error handler, so it can decide appropriate status code
LogValuesFunc: func(c *echo.Context, v middleware.RequestLoggerValues) error {
var errMsg string
if v.Error != nil {
errMsg = fmt.Sprintf(" ERROR: %s", v.Error.Error())
}
fmt.Printf("[%s %s]: %d%s\n", v.Method, v.URI, v.Status, errMsg)
return nil
},
}))
e.GET("/", func(c *echo.Context) error {
return c.String(http.StatusOK, "OK")
})
e.GET("/test", func(c echo.Context) error {
e.GET("/test", func(c *echo.Context) error {
return c.JSONPretty(http.StatusOK,
map[string]any{
"files": "",
}, " ")
})
e.POST("/testpost", func(c echo.Context) error {
e.POST("/testpost", func(c *echo.Context) error {
// Get the form value
test := c.FormValue("test")
fmt.Println("test:", test)
return c.String(http.StatusOK, "OK")
})
e.GET("/health", func(c echo.Context) error {
e.GET("/health", func(c *echo.Context) error {
h := queries.HealthCheck()
return c.JSONPretty(http.StatusOK, fmt.Sprintf("DB Connection: %v", h), " ")
})
e.GET("/update", func(c echo.Context) error {
//da inizializzare on server startup non qui
u := NewUpdater(UpdaterConfig{
e.GET("/update", func(c *echo.Context) error {
updateParams := UpdateParams{
isDryRun: c.QueryParam("dryrun") == "true",
codFilter: c.QueryParam("cod"),
forceMediaRefresh: c.QueryParam("forcemedia") == "true",
})
err := u.Update()
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
return c.String(http.StatusOK, "update")
bgCtx := context.Background()
go func() {
err := u.Update(bgCtx, updateParams)
if err != nil {
e.Logger.Error(fmt.Sprintf("failed update, %v", err))
}
}()
return c.JSON(http.StatusAccepted, map[string]string{
"status": "Accepted",
"message": "Your task is being processed in the background.",
})
})
//IMAGE ROUTES
e.GET("/initcwebp", func(c echo.Context) error {
e.GET("/initcwebp", func(c *echo.Context) error {
Conversion("test.jpg")
return c.String(http.StatusOK, "Cwebp initialized")
})
e.POST("/image-option-toggle", func(c echo.Context) error {
e.POST("/image-option-toggle", func(c *echo.Context) error {
// Toggle the imageOption value
s.config.Images.Enabled = !s.config.Images.Enabled
Cfg.Images.Enabled = !Cfg.Images.Enabled
// Convert the boolean value to a string and set the environment variable
os.Setenv("IMAGEOPTION", strconv.FormatBool(s.config.Images.Enabled))
os.Setenv("IMAGEOPTION", strconv.FormatBool(Cfg.Images.Enabled))
return c.String(http.StatusOK, "OK")
})
return e
}
func (s *Server) Start() {
e := s.SetupRoutes()
Conversion("test.jpg")
fmt.Println("Image Option: ", s.config.Images.Enabled)
fmt.Println("Concurrent Images: ", s.config.Images.ConcurrentLimit)
fmt.Println("Server url: http://localhost:" + strconv.Itoa(s.config.Server.Port))
e.Logger.Info(e.Start(":" + strconv.Itoa(s.config.Server.Port)))
}
func main() {
cfg := config.Load()
server, err := NewServer(cfg)
if err != nil {
log.Fatalf("Server initialization error: %v", err)
fmt.Println("Image Option: ", Cfg.Images.Enabled)
fmt.Println("Concurrent Images: ", Cfg.Images.ConcurrentLimit)
port := fmt.Sprintf(":%s", strconv.Itoa(Cfg.Server.Port))
fmt.Println("Server url: http://localhost" + port)
if err := e.Start(port); err != nil {
e.Logger.Error("failed to start server", "error", err)
}
server.Start()
}
type Template struct {
templates *template.Template
}
func (t *Template) Render(w io.Writer, name string, data any, c echo.Context) error {
return t.templates.ExecuteTemplate(w, name, data)
}

View file

@ -7,6 +7,7 @@ import (
"backend/queries"
"backend/typesdefs"
"backend/utils"
"context"
"fmt"
"log"
"slices"
@ -14,7 +15,7 @@ import (
"time"
)
type UpdaterConfig struct {
type UpdateParams struct {
// Dry run flag to simulate the update process without making changes.
isDryRun bool
// Filter by codice filter for specific updates. If empty, all records are updated.
@ -27,21 +28,17 @@ type Updater struct {
isDryRun bool
codFilter string
forceMediaRefresh bool
caratterisiche typesdefs.ListaCaratteristiche
categorie []typesdefs.Categoria
imgPool []typesdefs.MediaToProcess
videoPool []typesdefs.MediaToProcess
annunci typesdefs.AnnunciParsed
caratterisiche typesdefs.ListaCaratteristiche
categorie []typesdefs.Categoria
imgPool []typesdefs.MediaToProcess
videoPool []typesdefs.MediaToProcess
annunci typesdefs.AnnunciParsed
}
func NewUpdater(props UpdaterConfig) *Updater {
u := Updater{
isDryRun: props.isDryRun,
codFilter: props.codFilter,
forceMediaRefresh: props.forceMediaRefresh,
}
fmt.Printf("updater cfg: %+v\n", props)
func NewUpdater() *Updater {
u := Updater{}
u.init_Updater()
return &u
}
@ -88,7 +85,14 @@ func genCategorie() ([]typesdefs.Categoria, error) {
return values, nil
}
func (u *Updater) Update() error {
func (u *Updater) Update(ctx context.Context, params UpdateParams) error {
u.isDryRun = params.isDryRun
u.forceMediaRefresh = params.forceMediaRefresh
u.codFilter = params.codFilter
u.annunci = typesdefs.AnnunciParsed{}
u.imgPool = []typesdefs.MediaToProcess{}
u.videoPool = []typesdefs.MediaToProcess{}
var err error
err = u.parseAnnunci()
if err != nil {
@ -136,7 +140,7 @@ func (u *Updater) parseAnnunci() error {
u.annunci.Annuncio = append(u.annunci.Annuncio, u.extractAnnuncioData(&xml.Annuncio[i]))
}
return err
return nil
}
func getAnnunciMiogest() (typesdefs.AnnunciXML, error) {

24
apps/db-backup/Dockerfile Normal file
View file

@ -0,0 +1,24 @@
FROM golang:1.25-alpine AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY main.go .
# Compile a highly optimized, completely static binary
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o backup-app main.go
# --- Stage 2: Final Light Execution Layer ---
FROM postgres:18-alpine
WORKDIR /app
# Copy the compiled executable from the builder layer
COPY --from=builder /src/backup-app .
# Set up environmental time out-of-the-box
ENV TZ=Europe/Rome
ENV PGTZ=Europe/Rome
ENTRYPOINT ["/app/backup-app"]
CMD ["cron-init"]

28
apps/db-backup/go.mod Normal file
View file

@ -0,0 +1,28 @@
module db-backup
go 1.25.10
require (
github.com/aws/aws-sdk-go-v2 v1.41.7
github.com/aws/aws-sdk-go-v2/config v1.32.18
github.com/aws/aws-sdk-go-v2/credentials v1.19.17
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
github.com/robfig/cron/v3 v3.0.1
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect
github.com/aws/smithy-go v1.25.1 // indirect
)

38
apps/db-backup/go.sum Normal file
View file

@ -0,0 +1,38 @@
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
github.com/aws/aws-sdk-go-v2/config v1.32.18 h1:Hcia46bxhGgF3BaSnG8nSNCWmqTK6bj9xN9/FJ3WK6Q=
github.com/aws/aws-sdk-go-v2/config v1.32.18/go.mod h1:zEjCAYmxqDadH1WX8CdBvmLKhUEUVFgKRQG38zjDmrY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.17 h1:gP2nkGsS+KMvF/jfFz2Vv2qiiOqWKyPACSzPsqHgoW8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.17/go.mod h1:Bsew3S/moG5iT77giPj1q8wb/s0RE5/QfH+ASjYtuQc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0 h1:nDARhv/oF55bcxF7rCI/4PDxOKnVXVWwDuDwCs2I2SQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.0/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk=
github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=

595
apps/db-backup/main.go Normal file
View file

@ -0,0 +1,595 @@
package main
import (
"compress/gzip"
"context"
"fmt"
"io"
"os"
"os/exec"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/robfig/cron/v3"
)
// Config holds environmental configurations
type Config struct {
Host string
Port string
User string
Password string
DB string
BackupDir string
KeepCount int
Schedule string
AWSAccessKey string
AWSSecretKey string
S3Bucket string
S3Endpoint string
AWSRegion string
}
func loadConfig() Config {
keepCount, _ := strconv.Atoi(getEnv("BACKUP_KEEP_COUNT", "7"))
return Config{
Host: getEnv("POSTGRES_HOST", "db"),
Port: getEnv("POSTGRES_PORT", "5432"),
User: getEnv("POSTGRES_USER", "postgres"),
Password: getEnv("POSTGRES_PASSWORD", ""),
DB: getEnv("POSTGRES_DB", ""),
BackupDir: getEnv("BACKUP_DIR", "/backups"),
KeepCount: keepCount,
Schedule: os.Getenv("BACKUP_SCHEDULE"),
AWSAccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
AWSSecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
S3Bucket: os.Getenv("S3_BUCKET"),
S3Endpoint: os.Getenv("S3_ENDPOINT"),
AWSRegion: getEnv("AWS_DEFAULT_REGION", "us-east-1"),
}
}
func main() {
if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
printHelp()
os.Exit(0)
}
cfg := loadConfig()
mode := os.Args[1]
switch mode {
case "cron-init":
initScheduler(cfg)
case "run":
if err := runBackupPipeline(cfg); err != nil {
logError("Pipeline failed: %v", err)
os.Exit(1)
}
case "inspect":
if len(os.Args) < 3 {
logError("ERROR: No backup filename/S3 URL provided!")
fmt.Println("Usage: backup-app inspect <filename.sql.gz | s3://bucket/key>")
os.Exit(1)
}
if err := runInspectPipeline(cfg, os.Args[2]); err != nil {
logError("Inspection failed: %v", err)
os.Exit(1)
}
case "restore":
if len(os.Args) < 3 {
logError("ERROR: No backup filename/S3 URL provided!")
fmt.Println("Usage: backup-app restore <filename.sql.gz | s3://bucket/key>")
os.Exit(1)
}
if err := runRestorePipeline(cfg, os.Args[2]); err != nil {
logError("Restore failed: %v", err)
os.Exit(1)
}
default:
fmt.Printf("Unknown mode: %s\n", mode)
os.Exit(1)
}
}
// --- MODES OF OPERATION ---
func initScheduler(cfg Config) {
if cfg.Schedule == "" {
fmt.Println("No BACKUP_SCHEDULE set.")
// Create a channel that listens for OS shutdown signals
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan // This blocks cleanly forever until Docker stops the container
fmt.Println("Shutting down gracefully...")
return
}
fmt.Println("Initializing backup scheduler...")
c := cron.New(cron.WithSeconds()) // Supports standard cron or standard with seconds
_, err := c.AddFunc(cfg.Schedule, func() {
logInfo("Scheduled job triggered.")
if err := runBackupPipeline(cfg); err != nil {
logError("Scheduled backup pipeline failed: %v", err)
}
})
if err != nil {
logError("Failed to parse cron schedule '%s': %v", cfg.Schedule, err)
os.Exit(1)
}
fmt.Printf("Cron scheduled with pattern: %s\n", cfg.Schedule)
fmt.Println("Starting native Go cron engine...")
c.Run()
}
func runBackupPipeline(cfg Config) error {
timestamp := time.Now().Format("2006-01-02_15-04-05")
fileName := fmt.Sprintf("%s-%s.sql.gz", cfg.DB, timestamp)
fullPath := filepath.Join(cfg.BackupDir, fileName)
logInfo("Starting database backup...")
if err := os.MkdirAll(cfg.BackupDir, os.ModePerm); err != nil {
return fmt.Errorf("failed to create backup dir: %w", err)
}
// 1. Generate Local Backup (Stream pg_dump through Go's gzip engine)
outFile, err := os.Create(fullPath)
if err != nil {
return fmt.Errorf("failed to create backup file: %w", err)
}
gzipWriter := gzip.NewWriter(outFile)
args := []string{
"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, cfg.DB,
"--exclude-table-data=(media_storage|media_refs|tiles_cache)",
}
cmd := exec.Command("pg_dump", args...)
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
cmd.Stdout = gzipWriter
if err := cmd.Run(); err != nil {
gzipWriter.Close()
outFile.Close()
os.Remove(fullPath)
return fmt.Errorf("pg_dump execution failed: %w", err)
}
gzipWriter.Close()
outFile.Close()
logInfo("Success: Created local backup at %s", fullPath)
// 2. INTEGRITY TEST
logInfo("Launching backup integrity test...")
testDB := fmt.Sprintf("test_verify_%s", cfg.DB)
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", testDB))
if err := runPsqlCmd(cfg, "postgres", fmt.Sprintf("CREATE DATABASE %s;", testDB)); err != nil {
return fmt.Errorf("integrity test setup failed: %w", err)
}
// Stream gunzip restore to test DB
if err := streamRestoreFile(cfg, fullPath, testDB); err != nil {
logError("CRITICAL WARNING: Backup file was created but FAILED the restoration test! File is corrupted.")
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", testDB))
return fmt.Errorf("integrity verification failed: %w", err)
}
logInfo("PASSED: Backup file successfully passed restoration test syntax verification.")
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE %s;", testDB))
// 3. S3 UPLOAD (if AWS config is present)
if cfg.AWSAccessKey != "" {
logInfo("S3 configuration detected. Uploading to cloud storage...")
if err := uploadToS3(cfg, fullPath, fileName); err != nil {
return fmt.Errorf("S3 synchronization failed: %w", err)
}
logInfo("Cloud sync success: Uploaded to s3://%s/", cfg.S3Bucket)
}
// 4. RETENTION MAINTENANCE
logInfo("Evaluating retention policy (Keeping the latest %d backups)...", cfg.KeepCount)
if err := maintainRetention(cfg); err != nil {
logError("Retention maintenance error: %v", err)
}
return nil
}
func runInspectPipeline(cfg Config, target string) error {
localPath := target
var err error
if strings.HasPrefix(target, "s3://") {
logInfo("Target is on S3. Fetching metadata header...")
bucket, key := parseS3URI(target)
return inspectS3File(cfg, bucket, key)
}
if !filepath.IsAbs(localPath) {
localPath = filepath.Join(cfg.BackupDir, target)
}
file, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("unable to open backup file: %w", err)
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}
gzReader, err := gzip.NewReader(file)
if err != nil {
return fmt.Errorf("invalid gzip compression format: %w", err)
}
defer gzReader.Close()
fmt.Printf("\n--- BACKUP INSPECTION REPORT ---\n")
fmt.Printf("File Source: %s\n", filepath.Base(localPath))
fmt.Printf("Compressed Size: %.2f MB\n", float64(stat.Size())/(1024*1024))
if !gzReader.ModTime.IsZero() {
fmt.Printf("Gzip Timestamp: %s\n", gzReader.ModTime.Format(time.RFC1123))
}
// Read first 2KB to look for pg_dump header details
buffer := make([]byte, 2048)
n, _ := io.ReadFull(gzReader, buffer)
if n > 0 {
content := string(buffer[:n])
fmt.Println("\nHeader Snippet (First 5 comment lines):")
lines := strings.Split(content, "\n")
count := 0
for _, line := range lines {
if strings.HasPrefix(line, "--") {
fmt.Printf(" %s\n", line)
count++
if count >= 5 {
break
}
}
}
}
fmt.Printf("--------------------------------\n\n")
return nil
}
func runRestorePipeline(cfg Config, target string) error {
localPath := target
// Handle S3 targets seamlessly
if strings.HasPrefix(target, "s3://") {
logInfo("S3 URI detected (%s). Downloading archive to disk buffer...", target)
bucket, key := parseS3URI(target)
tmpFile, err := os.CreateTemp(cfg.BackupDir, "s3-restore-*.sql.gz")
if err != nil {
return fmt.Errorf("failed to create secure disk temporary buffer: %w", err)
}
defer os.Remove(tmpFile.Name()) // Auto cleanup
defer tmpFile.Close()
if err := downloadFromS3(cfg, bucket, key, tmpFile); err != nil {
return fmt.Errorf("S3 file extraction streaming failed: %w", err)
}
localPath = tmpFile.Name()
logInfo("Download finalized. Moving to verification sequence...")
} else if !filepath.IsAbs(localPath) {
localPath = filepath.Join(cfg.BackupDir, target)
}
if _, err := os.Stat(localPath); os.IsNotExist(err) {
return fmt.Errorf("file not found at target resolution scope: %s", localPath)
}
logInfo("STEP 1: Verifying integrity of backup file inside sandbox database...")
sandboxDB := fmt.Sprintf("restore_sandbox_%s", cfg.DB)
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", sandboxDB))
if err := runPsqlCmd(cfg, "postgres", fmt.Sprintf("CREATE DATABASE %s;", sandboxDB)); err != nil {
return fmt.Errorf("sandbox creation failed: %w", err)
}
if err := streamRestoreFile(cfg, localPath, sandboxDB); err != nil {
logError("CRITICAL FAILURE: The backup file failed verification tests! Main database will not be touched.")
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE IF EXISTS %s;", sandboxDB))
return fmt.Errorf("sandbox restore test failed: %w", err)
}
logInfo("PASSED: Verification successful. Sandbox database populated cleanly.")
_ = runPsqlCmd(cfg, "postgres", fmt.Sprintf("DROP DATABASE %s;", sandboxDB))
logInfo("STEP 2: Proceeding with Live Database Restoration...")
logInfo("WARNING: Dropping active public schema on target database: %s...", cfg.DB)
if err := runPsqlCmd(cfg, cfg.DB, "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"); err != nil {
return fmt.Errorf("failed to flush target schema: %w", err)
}
logInfo("STEP 3: Streaming verified records into production engine...")
if err := streamRestoreFile(cfg, localPath, cfg.DB); err != nil {
return fmt.Errorf("critical engine restoration stream failed: %w", err)
}
logInfo("SUCCESS: Production database successfully restored to status: %s", filepath.Base(target))
return nil
}
// --- HELPERS & UTILITIES ---
func runPsqlCmd(cfg Config, dbname, sqlCommand string) error {
args := []string{"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, "-d", dbname, "-c", sqlCommand}
cmd := exec.Command("psql", args...)
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
return cmd.Run()
}
func streamRestoreFile(cfg Config, gzippedFilePath, targetDB string) error {
file, err := os.Open(gzippedFilePath)
if err != nil {
return err
}
defer file.Close()
gzReader, err := gzip.NewReader(file)
if err != nil {
return err
}
defer gzReader.Close()
args := []string{"-h", cfg.Host, "-p", cfg.Port, "-U", cfg.User, "-d", targetDB}
cmd := exec.Command("psql", args...)
cmd.Env = append(os.Environ(), "PGPASSWORD="+cfg.Password)
stdinPipe, err := cmd.StdinPipe()
if err != nil {
return err
}
if err := cmd.Start(); err != nil {
return err
}
if _, err := io.Copy(stdinPipe, gzReader); err != nil {
stdinPipe.Close()
return err
}
stdinPipe.Close()
return cmd.Wait()
}
func getS3Client(cfg Config) (*s3.Client, error) {
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion(cfg.AWSRegion),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AWSAccessKey, cfg.AWSSecretKey, "")),
)
if err != nil {
return nil, err
}
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if cfg.S3Endpoint != "" {
o.BaseEndpoint = aws.String(cfg.S3Endpoint)
o.UsePathStyle = true
}
})
return client, nil
}
func uploadToS3(cfg Config, localPath, s3Key string) error {
file, err := os.Open(localPath)
if err != nil {
return err
}
defer file.Close()
// 1. Load the default credentials and region configuration
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion(cfg.AWSRegion),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AWSAccessKey, cfg.AWSSecretKey, "")),
)
if err != nil {
return err
}
// 2. Initialize the S3 Client with the modern BaseEndpoint design pattern
client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if cfg.S3Endpoint != "" {
// This string replaces the old EndpointResolver logic perfectly
o.BaseEndpoint = aws.String(cfg.S3Endpoint)
// If you use MinIO or DigitalOcean, you usually want virtual-host style path addressing disabled
o.UsePathStyle = true
}
})
// 3. Dispatch the payload to your cloud storage
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(cfg.S3Bucket),
Key: aws.String(s3Key),
Body: file,
})
return err
}
func downloadFromS3(cfg Config, bucket, key string, targetFile *os.File) error {
client, err := getS3Client(cfg)
if err != nil {
return err
}
result, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return err
}
defer result.Body.Close()
_, err = io.Copy(targetFile, result.Body)
return err
}
func inspectS3File(cfg Config, bucket, key string) error {
client, err := getS3Client(cfg)
if err != nil {
return err
}
result, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
return fmt.Errorf("failed to fetch target from S3: %w", err)
}
defer result.Body.Close()
gzReader, err := gzip.NewReader(result.Body)
if err != nil {
return fmt.Errorf("downloaded target is not a valid gzip configuration file: %w", err)
}
defer gzReader.Close()
fmt.Printf("\n--- S3 REMOTE BACKUP INSPECTION REPORT ---\n")
fmt.Printf("Bucket target: s3://%s/%s\n", bucket, key)
if result.ContentLength != nil {
fmt.Printf("Compressed Size: %.2f MB\n", float64(*result.ContentLength)/(1024*1024))
}
if result.LastModified != nil {
fmt.Printf("S3 Modification: %s\n", result.LastModified.Format(time.RFC1123))
}
buffer := make([]byte, 2048)
n, _ := io.ReadFull(gzReader, buffer)
if n > 0 {
fmt.Println("\nHeader Snippet (First 5 comment lines):")
lines := strings.Split(string(buffer[:n]), "\n")
count := 0
for _, line := range lines {
if strings.HasPrefix(line, "--") {
fmt.Printf(" %s\n", line)
count++
if count >= 5 {
break
}
}
}
}
fmt.Printf("------------------------------------------\n\n")
return nil
}
func maintainRetention(cfg Config) error {
pattern := filepath.Join(cfg.BackupDir, fmt.Sprintf("%s-*.sql.gz", cfg.DB))
files, err := filepath.Glob(pattern)
if err != nil {
return err
}
logInfo("Found %d total backup files locally.", len(files))
if len(files) <= cfg.KeepCount {
logInfo("No files require rotation.")
return nil
}
// Sort files alphabetically (since naming includes timestamps, alpha sort maps cleanly to chronological sort)
sort.Strings(files)
purgeCount := len(files) - cfg.KeepCount
logInfo("Threshold exceeded. Removing oldest %d file(s)...", purgeCount)
for i := 0; i < purgeCount; i++ {
logInfo("Deleted expired file: %s", filepath.Base(files[i]))
if err := os.Remove(files[i]); err != nil {
logError("Failed to delete %s: %v", files[i], err)
}
}
logInfo("Retention cleanup process finished.")
return nil
}
func parseS3URI(uri string) (bucket, key string) {
cleanStr := strings.TrimPrefix(uri, "s3://")
parts := strings.SplitN(cleanStr, "/", 2)
if len(parts) == 2 {
return parts[0], parts[1]
}
return parts[0], ""
}
func getEnv(key, fallback string) string {
if value, exists := os.LookupEnv(key); exists {
return value
}
return fallback
}
func logInfo(format string, v ...interface{}) {
fmt.Printf("[%s] %s\n", time.Now().Format("2006-01-02 15:04:05"), fmt.Sprintf(format, v...))
}
func logError(format string, v ...interface{}) {
fmt.Fprintf(os.Stderr, "[%s] ERROR: %s\n", time.Now().Format("2006-01-02 15:04:05"), fmt.Sprintf(format, v...))
}
func printHelp() {
fmt.Println(`===================================================================
Database Backup Manager (Go Edition)
===================================================================
Usage:
backup-app <command> [arguments]
Commands:
cron-init Starts the persistent background cron engine using the
cron pattern defined in $BACKUP_SCHEDULE. If no schedule
is defined, the process waits cleanly for termination.
run Triggers a single manual backup sequence immediately:
Dumps DB -> Compresses -> Verifies -> Uploads to S3 -> Rotates.
inspect <target> Peeks into a local archive file or remote S3 object URI
to extract GZIP headers, modification timestamps, and the
first few lines of pg_dump comments without a full download.
Examples:
backup-app inspect my-backup.sql.gz
backup-app inspect s3://my-bucket/my-backup.sql.gz
restore <target> Performs a safe, multi-stage restore of a local file or
S3 object URI. Streams to a sandbox DB for verification
before wiping and updating the production database schema.
Examples:
backup-app restore my-backup.sql.gz
backup-app restore s3://my-bucket/my-backup.sql.gz
help, -h, --help Display this help screen.
Required Environment Variables:
POSTGRES_HOST Database server hostname/IP
POSTGRES_USER Database administrative user
POSTGRES_PASSWORD Database password
POSTGRES_DB Target database name to back up/restore
Optional Environment Variables:
POSTGRES_PORT Database port (Default: 5432)
BACKUP_DIR Local storage path for backups (Default: /backups)
BACKUP_KEEP_COUNT Number of local archives to retain (Default: 7)
BACKUP_SCHEDULE Cron interval string (e.g., "0 2 * * *")
S3 Cloud Variables (Required for s3:// commands):
AWS_ACCESS_KEY_ID Your IAM Access Key ID
AWS_SECRET_ACCESS_KEY Your IAM Secret Access Key
S3_BUCKET Target S3 Bucket name (used during 'run')
AWS_DEFAULT_REGION Target AWS region (Default: us-east-1)
S3_ENDPOINT Custom URL for S3-compatible APIs (MinIO, DigitalOcean)
===================================================================`)
}

View file

@ -5,3 +5,4 @@ Dockerfile
filter_sql.awk
log_parser.sh
pg_dumper.sh
backups

View file

@ -38,6 +38,27 @@ services:
]
restart: "no"
db-backup:
build:
context: ../db-backup/
dockerfile: Dockerfile
restart: "no"
pull_policy: build
depends_on:
- db
environment:
POSTGRES_HOST: db
POSTGRES_PORT: 5432
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
BACKUP_DIR: /backups
BACKUP_KEEP_COUNT: 5
BACKUP_SCHEDULE: null
volumes:
- ./backups:/backups
networks:
- dokploy-network
networks:
dokploy-network:

View file

@ -1,5 +1,6 @@
#!/bin/bash
docker compose -p infoalloggi-dev-db -f dev-docker-compose.yml rm -f db-backup 2>/dev/null || true
docker compose -p infoalloggi-dev-db -f dev-docker-compose.yml rm -f migrate 2>/dev/null || true

View file

@ -44,24 +44,24 @@ END IF;
END $$;
DO $$
BEGIN
IF EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'media_refs'
) THEN
EXECUTE '
DELETE FROM public.storage s
WHERE s.tag IS NOT NULL
AND NOT EXISTS (
SELECT 1
FROM public.media_refs m
WHERE s.id = m.storage_id
);
';
END IF;
END $$;
-- DO $$
-- BEGIN
-- IF EXISTS (
-- SELECT FROM information_schema.tables
-- WHERE table_schema = 'public'
-- AND table_name = 'media_refs'
-- ) THEN
-- EXECUTE '
-- DELETE FROM public.storage s
-- WHERE s.tag IS NOT NULL
-- AND NOT EXISTS (
-- SELECT 1
-- FROM public.media_refs m
-- WHERE s.id = m.storage_id
-- );
-- ';
-- END IF;
-- END $$;
-- Drop Tables if they exist
DROP TABLE IF EXISTS public.images_refs;

View file

@ -0,0 +1,53 @@
CREATE TABLE IF NOT EXISTS public.media_storage (
id uuid NOT NULL DEFAULT uuidv7(),
created_at timestamp without time zone NOT NULL DEFAULT now(),
filename text NOT NULL,
mime_type text NOT NULL,
file_size_bytes INT NOT NULL,
file_data BYTEA NOT NULL,
expires_at timestamp without time zone,
tag text
);
DO $$ BEGIN IF EXISTS (
SELECT
FROM information_schema.columns
WHERE table_name = 'media_storage'
AND column_name = 'file_data'
) THEN
ALTER TABLE public.media_storage
ALTER COLUMN file_data SET STORAGE EXTERNAL;
END IF;
END $$;
DO $$ BEGIN IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'media_storage_pkey'
AND conrelid = 'public.media_storage'::regclass
) THEN
ALTER TABLE public.media_storage
ADD CONSTRAINT media_storage_pkey PRIMARY KEY (id);
END IF;
END $$;
DELETE FROM public.media_refs
WHERE storage_id NOT IN (SELECT id FROM public.media_storage);
DELETE FROM public.storage
WHERE tag is not null;
ALTER TABLE media_refs
DROP CONSTRAINT IF EXISTS media_refs_storage;
DO $$ BEGIN IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'media_refs_storage'
) THEN
ALTER TABLE public.media_refs
ADD CONSTRAINT media_refs_storage FOREIGN KEY (storage_id) REFERENCES public.media_storage (id) ON UPDATE CASCADE ON DELETE CASCADE;
END IF;
END $$;

View file

@ -13,6 +13,8 @@ TODOS:
- onboarding submit (toast e email che invitano a pagare)
- migliorare onboarding e pag guida
- rendere opzionale email e numero?
- titolo tab in edit annuncio
- on payment success webhook, provare ad aggiungere su fatture in cloud
AFTER MVP:
- TODO migrazione app router https://nextjs.org/docs/pages/building-your-application/upgrading/app-router-migration#migrating-from-pages-to-app

View file

@ -1,4 +1,5 @@
import { env } from "~/env";
import type { MediaStorageId } from "~/schemas/public/MediaStorage";
import type { StorageId } from "~/schemas/public/Storage";
type StorageParams = {
@ -12,7 +13,7 @@ export function getStorageUrl({
params,
host = env.NEXT_PUBLIC_BASE_URL,
}: {
id: StorageId;
id: StorageId | MediaStorageId;
params?: Partial<StorageParams>;
host?: string;
}) {

View file

@ -1,10 +1,14 @@
import fs from "node:fs";
import path from "node:path";
import type { NextApiRequest, NextApiResponse } from "next";
import { storageId } from "~/schemas/public/Storage";
import {
type MediaStorage,
mediaStorageId,
} from "~/schemas/public/MediaStorage";
import { type Storage, storageId } from "~/schemas/public/Storage";
import { TOKEN_CONFIG } from "~/server/auth/configs";
import { verifyToken } from "~/server/auth/jwt";
import { getFile } from "~/server/services/storage.service";
import { getFile, getMediaFile } from "~/server/services/storage.service";
export default async function handler(
req: NextApiRequest,
res: NextApiResponse,
@ -55,19 +59,33 @@ export default async function handler(
return res.status(401);
}
}
const parsedId = storageId.safeParse(id);
if (!parsedId.success) {
if (isImageRequest || isNextImageRequest) {
return getFallback("fallback-image.png", req, res);
let file: Storage | MediaStorage | undefined;
if (isImageRequest || isNextImageRequest || isVideoRequest) {
const parsedId = mediaStorageId.safeParse(id);
if (!parsedId.success) {
if (isImageRequest || isNextImageRequest) {
return getFallback("fallback-image.png", req, res);
}
if (isVideoRequest) {
return getFallback("fallback-video.png", req, res);
}
return res.status(404).json({ error: "Invalid input id" });
}
if (isVideoRequest) {
return getFallback("fallback-video.png", req, res);
file = await getMediaFile(parsedId.data);
} else {
const parsedId = storageId.safeParse(id);
if (!parsedId.success) {
if (isImageRequest || isNextImageRequest) {
return getFallback("fallback-image.png", req, res);
}
if (isVideoRequest) {
return getFallback("fallback-video.png", req, res);
}
return res.status(404).json({ error: "Invalid input id" });
}
return res.status(404).json({ error: "Invalid input id" });
file = await getFile(parsedId.data);
}
const file = await getFile(parsedId.data);
if (!file) {
if (isImageRequest || isNextImageRequest) {
return getFallback("fallback-image.png", req, res);

View file

@ -1,4 +1,4 @@
import { storageId, type StorageId } from './Storage';
import { mediaStorageId, type MediaStorageId } from './MediaStorage';
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
import { z } from 'zod';
@ -8,7 +8,7 @@ export default interface MediaRefsTable {
ordine: ColumnType<number, number, number>;
storage_id: ColumnType<StorageId, StorageId, StorageId>;
storage_id: ColumnType<MediaStorageId, MediaStorageId, MediaStorageId>;
og_url: ColumnType<string, string, string>;
@ -26,7 +26,7 @@ export type MediaRefsUpdate = Updateable<MediaRefsTable>;
export const MediaRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
storage_id: storageId,
storage_id: mediaStorageId,
og_url: z.string(),
media_type: z.string(),
is_thumbnail: z.boolean(),
@ -35,7 +35,7 @@ export const MediaRefsSchema = z.object({
export const NewMediaRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
storage_id: storageId,
storage_id: mediaStorageId,
og_url: z.string(),
media_type: z.string(),
is_thumbnail: z.boolean().optional(),
@ -44,7 +44,7 @@ export const NewMediaRefsSchema = z.object({
export const MediaRefsUpdateSchema = z.object({
codice: z.string().optional(),
ordine: z.number().optional(),
storage_id: storageId.optional(),
storage_id: mediaStorageId.optional(),
og_url: z.string().optional(),
media_type: z.string().optional(),
is_thumbnail: z.boolean().optional(),

View file

@ -0,0 +1,65 @@
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
import { z } from 'zod';
/** Identifier type for public.media_storage */
export type MediaStorageId = string & { __brand: 'public.media_storage' };
/** Represents the table public.media_storage */
export default interface MediaStorageTable {
id: ColumnType<MediaStorageId, MediaStorageId | undefined, MediaStorageId>;
created_at: ColumnType<Date, Date | string | undefined, Date | string>;
filename: ColumnType<string, string, string>;
mime_type: ColumnType<string, string, string>;
file_size_bytes: ColumnType<number, number, number>;
file_data: ColumnType<Buffer, Buffer, Buffer>;
expires_at: ColumnType<Date | null, Date | string | null, Date | string | null>;
tag: ColumnType<string | null, string | null, string | null>;
}
export type MediaStorage = Selectable<MediaStorageTable>;
export type NewMediaStorage = Insertable<MediaStorageTable>;
export type MediaStorageUpdate = Updateable<MediaStorageTable>;
export const mediaStorageId = z.uuid().transform(value => value as MediaStorageId);
export const MediaStorageSchema = z.object({
id: mediaStorageId,
created_at: z.date(),
filename: z.string(),
mime_type: z.string(),
file_size_bytes: z.number(),
file_data: z.instanceof(Buffer),
expires_at: z.date().nullable(),
tag: z.string().nullable(),
});
export const NewMediaStorageSchema = z.object({
id: mediaStorageId.optional(),
created_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional(),
filename: z.string(),
mime_type: z.string(),
file_size_bytes: z.number(),
file_data: z.instanceof(Buffer),
expires_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional().nullable(),
tag: z.string().optional().nullable(),
});
export const MediaStorageUpdateSchema = z.object({
id: mediaStorageId.optional(),
created_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional(),
filename: z.string().optional(),
mime_type: z.string().optional(),
file_size_bytes: z.number().optional(),
file_data: z.instanceof(Buffer).optional(),
expires_at: z.union([z.date(), z.string()]).pipe(z.coerce.date()).optional().nullable(),
tag: z.string().optional().nullable(),
});

View file

@ -20,6 +20,7 @@ import type { default as PrezziarioTable } from './Prezziario';
import type { default as RatelimiterTable } from './Ratelimiter';
import type { default as ChatsTable } from './Chats';
import type { default as EtichetteTable } from './Etichette';
import type { default as MediaStorageTable } from './MediaStorage';
import type { default as MessagesTable } from './Messages';
import type { default as UsersAnagraficaTable } from './UsersAnagrafica';
import type { default as ProvincieTable } from './Provincie';
@ -77,6 +78,8 @@ export default interface PublicSchema {
etichette: EtichetteTable;
media_storage: MediaStorageTable;
messages: MessagesTable;
users_anagrafica: UsersAnagraficaTable;

View file

@ -1,7 +1,7 @@
import { z } from "zod/v4";
import { AnnunciUpdateSchema, annunciId } from "~/schemas/public/Annunci";
import { mediaStorageId } from "~/schemas/public/MediaStorage";
import { servizioServizioId } from "~/schemas/public/Servizio";
import { storageId } from "~/schemas/public/Storage";
import {
adminProcedure,
apiProcedure,
@ -174,7 +174,7 @@ export const annunciRouter = createTRPCRouter({
return { status: "ok", timestamp: Date.now() };
}),
removeAnnuncioMedia: adminProcedure
.input(storageId)
.input(mediaStorageId)
.mutation(async ({ input }) => {
return await removeAnnuncioMedia(input);
}),

View file

@ -7,8 +7,8 @@ import type {
AnnunciId,
AnnunciUpdate,
} from "~/schemas/public/Annunci";
import type { MediaStorageId } from "~/schemas/public/MediaStorage";
import type { ServizioServizioId } from "~/schemas/public/Servizio";
import type { StorageId } from "~/schemas/public/Storage";
import { db } from "~/server/db";
import type { AnnuncioTemplateData } from "~/server/services/typst.service";
import { withImages, withVideos } from "~/utils/kysely-helper";
@ -17,13 +17,13 @@ import { revalidate } from "../utils/revalidationHelper";
export type ImgMedia = {
ordine: number;
img: StorageId;
thumb: StorageId | null;
img: MediaStorageId;
thumb: MediaStorageId | null;
};
export type VideoMedia = {
ordine: number;
video: StorageId;
thumb: StorageId | null;
video: MediaStorageId;
thumb: MediaStorageId | null;
};
export type AnnunciWithMedia = Annunci & {
tipo: string;
@ -546,7 +546,7 @@ export const getProprietarioDataHandler = async ({
}
};
export const removeAnnuncioMedia = async (id: StorageId) => {
export const removeAnnuncioMedia = async (id: MediaStorageId) => {
try {
await db
.$pickTables<"media_refs">()

View file

@ -720,9 +720,9 @@ export const getAnnunciAvailableToAdd = async (
let qry = tx
.$pickTables<"annunci">()
.selectFrom("annunci")
.select(["id", "codice", "titolo_it"])
.where("annunci.web", "=", true)
.where("stato", "!=", "Sospeso");
.select(["id", "codice", "titolo_it"]);
//.where("annunci.web", "=", true)
//.where("stato", "!=", "Sospeso");
if (ids.length > 0) {
qry = qry.where("id", "not in", ids);
@ -1224,6 +1224,7 @@ export const getRichieste = async (): Promise<RichiesteData[]> => {
| "annunci"
| "ordini"
| "prezziario"
| "media_refs"
>()
.selectFrom("servizio")
.innerJoin("users", "users.id", "servizio.user_id")

View file

@ -1,4 +1,5 @@
import { TRPCError } from "@trpc/server";
import type { MediaStorageId } from "~/schemas/public/MediaStorage";
import type {
Storage,
StorageId,
@ -108,6 +109,22 @@ export const getFile = async (fileId: StorageId) => {
}
};
export const getMediaFile = async (fileId: MediaStorageId) => {
try {
return await db
.$pickTables<"media_storage">()
.selectFrom("media_storage")
.selectAll()
.where("id", "=", fileId)
.executeTakeFirst();
} catch (e) {
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
message: `Failed to get media_file: ${(e as Error).message}`,
});
}
};
export const editFile = async (fileId: StorageId, data: StorageUpdate) => {
try {
await db

View file

@ -13,7 +13,7 @@ services:
expose:
- 5432
volumes:
- dbinfo-main-zkx9vb_dbdata_v18:/var/lib/postgresql
- dbdata_v18:/var/lib/postgresql
healthcheck:
test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"]
interval: 1s
@ -34,11 +34,36 @@ services:
[
"-verbose",
"-path", "/migrations",
"-database", "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable",
"-database", "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:${PGPORT}/${POSTGRES_DB}?sslmode=disable",
"up"
]
restart: "no"
db-backup:
build: ./apps/db-backup/
restart: always
pull_policy: build
depends_on:
db:
condition: service_healthy
environment:
POSTGRES_HOST: db
POSTGRES_PORT: ${PGPORT}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
BACKUP_DIR: /backups
BACKUP_KEEP_COUNT: ${BACKUP_KEEP_COUNT}
BACKUP_SCHEDULE: ${BACKUP_SCHEDULE}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT}
volumes:
- ./backups:/backups
networks:
- dokploy-network
backend:
image: "${IMAGE_REGISTRY:-ghcr.io}/${OWNER:-marcopedone}/${PREFIX:-infoalloggi}-backend:${TAG:-latest}"
networks:
@ -56,7 +81,7 @@ services:
IMAGEOPTION: "true"
CONCURRENT_IMAGES: ${CONCURRENT_IMAGES}
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:1323/"]
test: ["CMD", "curl", "-f", "http://localhost:1323"]
interval: 10s
timeout: 5s
retries: 5
@ -151,5 +176,4 @@ networks:
external: true
volumes:
info-main-zkx9vb_dbdata_v18:
external: true
dbdata_v18: