refactor: enable logging in database handler and improve image/video processing feedback

apps/backend/.air.toml

@@ -35,7 +35,7 @@ tmp_dir = "tmp"
   watcher = "cyan"
 
 [log]
-  main_only = false
+  main_only = true
   time = false
 
 [misc]

apps/backend/db/handler.go

@@ -21,6 +21,7 @@ type Configs struct {
 	Port          string
 	User          string
 	Password      string
+	LoggerEnabled bool
 }
 
 func NewDbHandler(configs Configs) *DbHandler {
@@ -38,7 +39,11 @@ func NewDbHandler(configs Configs) *DbHandler {
 	if err != nil {
 		panic(err)
 	}
+
+	if configs.LoggerEnabled {
+		fmt.Println("Database query logging is enabled")
 		config.ConnConfig.Tracer = NewMultiQueryTracer(NewLoggingQueryTracer(slog.Default()))
+	}
 
 	config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
 		pgxuuid.Register(conn.TypeMap())

apps/backend/go.mod

@@ -1,6 +1,6 @@
 module backend
 
-go 1.24
+go 1.24.0
 
 require (
 	github.com/georgysavva/scany/v2 v2.1.4
@@ -28,16 +28,20 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible // indirect
 	github.com/minio/crc64nvme v1.1.0 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/xid v1.6.0 // indirect
+	github.com/schollz/progressbar/v3 v3.18.0 // indirect
 	github.com/tinylib/msgp v1.3.0 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	golang.org/x/image v0.29.0 // indirect
 	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/term v0.37.0 // indirect
 )
 
 require (
@@ -52,6 +56,6 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	golang.org/x/crypto v0.40.0 // indirect
 	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/text v0.27.0 // indirect
 )

apps/backend/go.sum

@@ -70,6 +70,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
 github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 h1:/6MoQlTdk1eAi0J9O89ypO8umkp+H7mpnSF2ggSL62Q=
@@ -88,10 +90,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
+github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -123,6 +129,10 @@ golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
 golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
 golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=

apps/backend/images.go

@@ -21,6 +21,7 @@ import (
 	"github.com/labstack/echo/v4"
 	"github.com/nfnt/resize"
 	"github.com/nickalie/go-webpbin"
+	"github.com/schollz/progressbar/v3"
 )
 
 func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
@@ -45,7 +46,8 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 		wg              sync.WaitGroup
 	)
 	sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
-
+	totalImages := len(*toProcess)
+	bar := progressbar.Default(int64(totalImages), "Processing images")
 	// clear images folder
 	err := clearImageFolder()
 	if err != nil {
@@ -78,9 +80,15 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 				OGUrl:        img.Url,
 			})
 			mu.Unlock()
+			bar.Add(1)
 		}(img)
 	}
 	wg.Wait()
+	bar.Finish()
+	fmt.Println()
+
+	totalToUpload := len(ProcessedImages)
+	uploadBar := progressbar.Default(int64(totalToUpload), "Uploading images")
 	uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
 	var uploadWg sync.WaitGroup
 	for _, img := range ProcessedImages {
@@ -108,10 +116,13 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 				ThumbId: thumbId,
 				OGUrl:   img.OGUrl,
 			})
+			uploadBar.Add(1)
 			mu.Unlock()
 		}(img)
 	}
 	uploadWg.Wait()
+	uploadBar.Finish()
+	fmt.Println()
 
 	return UploadedImages, nil
 }

apps/backend/miogest_handler.go

@@ -12,6 +12,8 @@ import (
 	"slices"
 	"strings"
 	"time"
+
+	"github.com/schollz/progressbar/v3"
 )
 
 type MiogestHandler struct {
@@ -187,6 +189,8 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
 
 	}
 
+	totalAnnunci := len(annunci.Annuncio)
+	bar := progressbar.Default(int64(totalAnnunci), "Parsing annunci")
 	var AnnunciArray typesdefs.AnnunciParsed
 	AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio))
 	m.InitDefaults()
@@ -196,9 +200,10 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
 		updateVideos := slices.Contains(video_codes_to_process, *annunci.Annuncio[i].Codice)
 		result := extractData_update(&annunci.Annuncio[i], imgToUpdate, updateVideos, m, imgPool, videoPool)
 		AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result)
-
+		bar.Add(1)
 	}
-
+	bar.Finish()
+	fmt.Println()
 	return AnnunciArray
 }
 

apps/backend/queries/queries.go

@@ -20,6 +20,7 @@ func init() {
 		Port:          os.Getenv("PGPORT"),
 		User:          os.Getenv("POSTGRES_USER"),
 		Password:      os.Getenv("POSTGRES_PASSWORD"),
+		LoggerEnabled: os.Getenv("DB_LOGGER") == "true",
 	})
 
 }

apps/backend/server.go

@@ -119,25 +119,24 @@ func (s *Server) SetupRoutes() *echo.Echo {
 		videoPool := []typesdefs.MediaToProcess{}
 
 		var data = s.miogest.GetAnnunciParsed(&imgPool, &videoPool, forceMediaStale)
+		err := queries.AnnunciInsert(data, true)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
 
 		imgRefs, err := ProcessImagePool(&imgPool)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())
 		}
-		videoRefs, err := ProcessVideoPool(&videoPool)
-		if err != nil {
-			return c.String(http.StatusInternalServerError, err.Error())
-		}
-
-		err = queries.AnnunciInsert(data, true)
-		if err != nil {
-			return c.String(http.StatusInternalServerError, err.Error())
-		}
-
 		err = queries.SetImagesToDB(&imgRefs)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())
 		}
+
+		videoRefs, err := ProcessVideoPool(&videoPool)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
 		err = queries.SetVideosToDB(&videoRefs)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())
@@ -150,22 +149,24 @@ func (s *Server) SetupRoutes() *echo.Echo {
 		imgPool := []typesdefs.MediaToProcess{}
 		videoPool := []typesdefs.MediaToProcess{}
 		var data = s.miogest.GetAnnuncioParsed(cod, &imgPool, &videoPool)
+		err := queries.AnnunciInsert(data, false)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
+
 		imgRefs, err := ProcessImagePool(&imgPool)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())
 		}
-		videoRefs, err := ProcessVideoPool(&videoPool)
-		if err != nil {
-			return c.String(http.StatusInternalServerError, err.Error())
-		}
-		err = queries.AnnunciInsert(data, false)
-		if err != nil {
-			return c.String(http.StatusInternalServerError, err.Error())
-		}
 		err = queries.SetImagesToDB(&imgRefs)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())
 		}
+
+		videoRefs, err := ProcessVideoPool(&videoPool)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
 		err = queries.SetVideosToDB(&videoRefs)
 		if err != nil {
 			return c.String(http.StatusInternalServerError, err.Error())

apps/backend/video.go

@@ -20,6 +20,7 @@ import (
 	"github.com/labstack/echo/v4"
 	"github.com/nfnt/resize"
 	"github.com/nickalie/go-webpbin"
+	"github.com/schollz/progressbar/v3"
 )
 
 func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
@@ -52,6 +53,8 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 		mu              sync.Mutex
 		wg              sync.WaitGroup
 	)
+	totalToProcess := len(*toProcess)
+	bar := progressbar.Default(int64(totalToProcess), "Processing videos")
 	sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
 
 	// clear videos folder
@@ -87,10 +90,16 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 				ThumnailPath: thumbnailPath,
 				OGUrl:        video.Url,
 			})
+			bar.Add(1)
 			mu.Unlock()
 		}(video)
 	}
 	wg.Wait()
+	bar.Finish()
+	fmt.Println()
+
+	totalToUpload := len(ProcessedVideos)
+	uploadBar := progressbar.Default(int64(totalToUpload), "Uploading videos")
 	uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
 	var uploadWg sync.WaitGroup
 	for _, video := range ProcessedVideos {
@@ -118,10 +127,13 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
 				ThumbId: thumbId,
 				OGUrl:   video.OGUrl,
 			})
+			uploadBar.Add(1)
 			mu.Unlock()
 		}(video)
 	}
 	uploadWg.Wait()
+	uploadBar.Finish()
+	fmt.Println()
 
 	return UploadedVideos, nil
 }