From 40bd07ded3bf2237875955e9eeaee88c1a724c0d Mon Sep 17 00:00:00 2001 From: Marco Pedone Date: Tue, 18 Nov 2025 18:22:47 +0100 Subject: [PATCH] refactor: enable logging in database handler and improve image/video processing feedback --- apps/backend/.air.toml | 2 +- apps/backend/db/handler.go | 17 +++++++++------ apps/backend/go.mod | 8 +++++-- apps/backend/go.sum | 10 +++++++++ apps/backend/images.go | 13 +++++++++++- apps/backend/miogest_handler.go | 9 ++++++-- apps/backend/queries/queries.go | 11 +++++----- apps/backend/server.go | 37 +++++++++++++++++---------------- apps/backend/video.go | 12 +++++++++++ 9 files changed, 84 insertions(+), 35 deletions(-) diff --git a/apps/backend/.air.toml b/apps/backend/.air.toml index 400b463..62d07d4 100644 --- a/apps/backend/.air.toml +++ b/apps/backend/.air.toml @@ -35,7 +35,7 @@ tmp_dir = "tmp" watcher = "cyan" [log] - main_only = false + main_only = true time = false [misc] diff --git a/apps/backend/db/handler.go b/apps/backend/db/handler.go index 43ce2fa..35429ab 100644 --- a/apps/backend/db/handler.go +++ b/apps/backend/db/handler.go @@ -16,11 +16,12 @@ type DbHandler struct { } type Configs struct { - Host string - Dbname string - Port string - User string - Password string + Host string + Dbname string + Port string + User string + Password string + LoggerEnabled bool } func NewDbHandler(configs Configs) *DbHandler { @@ -38,7 +39,11 @@ func NewDbHandler(configs Configs) *DbHandler { if err != nil { panic(err) } - config.ConnConfig.Tracer = NewMultiQueryTracer(NewLoggingQueryTracer(slog.Default())) + + if configs.LoggerEnabled { + fmt.Println("Database query logging is enabled") + config.ConnConfig.Tracer = NewMultiQueryTracer(NewLoggingQueryTracer(slog.Default())) + } config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { pgxuuid.Register(conn.TypeMap()) diff --git a/apps/backend/go.mod b/apps/backend/go.mod index c451225..8a1fa81 100644 --- a/apps/backend/go.mod +++ b/apps/backend/go.mod @@ -1,6 +1,6 @@ module backend -go 1.24 +go 1.24.0 require ( github.com/georgysavva/scany/v2 v2.1.4 @@ -28,16 +28,20 @@ require ( github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/minio/crc64nvme v1.1.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.6.0 // indirect + github.com/schollz/progressbar/v3 v3.18.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect golang.org/x/image v0.29.0 // indirect golang.org/x/sync v0.16.0 // indirect + golang.org/x/term v0.37.0 // indirect ) require ( @@ -52,6 +56,6 @@ require ( github.com/valyala/fasttemplate v1.2.2 // indirect golang.org/x/crypto v0.40.0 // indirect golang.org/x/net v0.42.0 // indirect - golang.org/x/sys v0.34.0 // indirect + golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.27.0 // indirect ) diff --git a/apps/backend/go.sum b/apps/backend/go.sum index 83550c2..8495e28 100644 --- a/apps/backend/go.sum +++ b/apps/backend/go.sum @@ -70,6 +70,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 h1:/6MoQlTdk1eAi0J9O89ypO8umkp+H7mpnSF2ggSL62Q= @@ -88,10 +90,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= +github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -123,6 +129,10 @@ golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= diff --git a/apps/backend/images.go b/apps/backend/images.go index 716fb9d..df5cf5e 100644 --- a/apps/backend/images.go +++ b/apps/backend/images.go @@ -21,6 +21,7 @@ import ( "github.com/labstack/echo/v4" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" + "github.com/schollz/progressbar/v3" ) func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) { @@ -45,7 +46,8 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload wg sync.WaitGroup ) sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) - + totalImages := len(*toProcess) + bar := progressbar.Default(int64(totalImages), "Processing images") // clear images folder err := clearImageFolder() if err != nil { @@ -78,9 +80,15 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload OGUrl: img.Url, }) mu.Unlock() + bar.Add(1) }(img) } wg.Wait() + bar.Finish() + fmt.Println() + + totalToUpload := len(ProcessedImages) + uploadBar := progressbar.Default(int64(totalToUpload), "Uploading images") uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) var uploadWg sync.WaitGroup for _, img := range ProcessedImages { @@ -108,10 +116,13 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload ThumbId: thumbId, OGUrl: img.OGUrl, }) + uploadBar.Add(1) mu.Unlock() }(img) } uploadWg.Wait() + uploadBar.Finish() + fmt.Println() return UploadedImages, nil } diff --git a/apps/backend/miogest_handler.go b/apps/backend/miogest_handler.go index 4c56fe6..f62987b 100644 --- a/apps/backend/miogest_handler.go +++ b/apps/backend/miogest_handler.go @@ -12,6 +12,8 @@ import ( "slices" "strings" "time" + + "github.com/schollz/progressbar/v3" ) type MiogestHandler struct { @@ -187,6 +189,8 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med } + totalAnnunci := len(annunci.Annuncio) + bar := progressbar.Default(int64(totalAnnunci), "Parsing annunci") var AnnunciArray typesdefs.AnnunciParsed AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio)) m.InitDefaults() @@ -196,9 +200,10 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med updateVideos := slices.Contains(video_codes_to_process, *annunci.Annuncio[i].Codice) result := extractData_update(&annunci.Annuncio[i], imgToUpdate, updateVideos, m, imgPool, videoPool) AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result) - + bar.Add(1) } - + bar.Finish() + fmt.Println() return AnnunciArray } diff --git a/apps/backend/queries/queries.go b/apps/backend/queries/queries.go index 6b284cb..95c5bcc 100644 --- a/apps/backend/queries/queries.go +++ b/apps/backend/queries/queries.go @@ -15,11 +15,12 @@ func init() { _ = godotenv.Load(".env") Db = db.NewDbHandler(db.Configs{ - Host: os.Getenv("PGHOST"), - Dbname: os.Getenv("POSTGRES_DB"), - Port: os.Getenv("PGPORT"), - User: os.Getenv("POSTGRES_USER"), - Password: os.Getenv("POSTGRES_PASSWORD"), + Host: os.Getenv("PGHOST"), + Dbname: os.Getenv("POSTGRES_DB"), + Port: os.Getenv("PGPORT"), + User: os.Getenv("POSTGRES_USER"), + Password: os.Getenv("POSTGRES_PASSWORD"), + LoggerEnabled: os.Getenv("DB_LOGGER") == "true", }) } diff --git a/apps/backend/server.go b/apps/backend/server.go index 867a3b4..48c7d29 100644 --- a/apps/backend/server.go +++ b/apps/backend/server.go @@ -119,25 +119,24 @@ func (s *Server) SetupRoutes() *echo.Echo { videoPool := []typesdefs.MediaToProcess{} var data = s.miogest.GetAnnunciParsed(&imgPool, &videoPool, forceMediaStale) + err := queries.AnnunciInsert(data, true) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } imgRefs, err := ProcessImagePool(&imgPool) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } - videoRefs, err := ProcessVideoPool(&videoPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - err = queries.AnnunciInsert(data, true) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.SetImagesToDB(&imgRefs) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } + + videoRefs, err := ProcessVideoPool(&videoPool) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } err = queries.SetVideosToDB(&videoRefs) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) @@ -150,22 +149,24 @@ func (s *Server) SetupRoutes() *echo.Echo { imgPool := []typesdefs.MediaToProcess{} videoPool := []typesdefs.MediaToProcess{} var data = s.miogest.GetAnnuncioParsed(cod, &imgPool, &videoPool) + err := queries.AnnunciInsert(data, false) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + imgRefs, err := ProcessImagePool(&imgPool) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } - videoRefs, err := ProcessVideoPool(&videoPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.AnnunciInsert(data, false) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } err = queries.SetImagesToDB(&imgRefs) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } + + videoRefs, err := ProcessVideoPool(&videoPool) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } err = queries.SetVideosToDB(&videoRefs) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) diff --git a/apps/backend/video.go b/apps/backend/video.go index df3db54..d39d359 100644 --- a/apps/backend/video.go +++ b/apps/backend/video.go @@ -20,6 +20,7 @@ import ( "github.com/labstack/echo/v4" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" + "github.com/schollz/progressbar/v3" ) func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) { @@ -52,6 +53,8 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload mu sync.Mutex wg sync.WaitGroup ) + totalToProcess := len(*toProcess) + bar := progressbar.Default(int64(totalToProcess), "Processing videos") sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit) // clear videos folder @@ -87,10 +90,16 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload ThumnailPath: thumbnailPath, OGUrl: video.Url, }) + bar.Add(1) mu.Unlock() }(video) } wg.Wait() + bar.Finish() + fmt.Println() + + totalToUpload := len(ProcessedVideos) + uploadBar := progressbar.Default(int64(totalToUpload), "Uploading videos") uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit) var uploadWg sync.WaitGroup for _, video := range ProcessedVideos { @@ -118,10 +127,13 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload ThumbId: thumbId, OGUrl: video.OGUrl, }) + uploadBar.Add(1) mu.Unlock() }(video) } uploadWg.Wait() + uploadBar.Finish() + fmt.Println() return UploadedVideos, nil }