Marco/infoalloggi-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 6 Wiki Activity Actions

refactor: enable logging in database handler and improve image/video processing feedback

Browse source
This commit is contained in:
Marco Pedone 2025-11-18 18:22:47 +01:00
parent 194b9345aa
commit 40bd07ded3
9 changed files with 84 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
apps/backend/.air.toml
View file

@ -35,7 +35,7 @@ tmp_dir = "tmp"
watcher = "cyan"
[log]
main_only = false
main_only = true
time = false
[misc]

5
apps/backend/db/handler.go
View file

@ -21,6 +21,7 @@ type Configs struct {
Port string
User string
Password string
LoggerEnabled bool
}
func NewDbHandler(configs Configs) *DbHandler {
@ -38,7 +39,11 @@ func NewDbHandler(configs Configs) *DbHandler {
if err != nil {
panic(err)
}
if configs.LoggerEnabled {
fmt.Println("Database query logging is enabled")
config.ConnConfig.Tracer = NewMultiQueryTracer(NewLoggingQueryTracer(slog.Default()))
}
config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
pgxuuid.Register(conn.TypeMap())

8
apps/backend/go.mod
View file

@ -1,6 +1,6 @@
module backend
go 1.24
go 1.24.0
require (
github.com/georgysavva/scany/v2 v2.1.4
@ -28,16 +28,20 @@ require (
github.com/mholt/archiver v3.1.1+incompatible // indirect
github.com/minio/crc64nvme v1.1.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/schollz/progressbar/v3 v3.18.0 // indirect
github.com/tinylib/msgp v1.3.0 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
golang.org/x/image v0.29.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/term v0.37.0 // indirect
)
require (
@ -52,6 +56,6 @@ require (
github.com/valyala/fasttemplate v1.2.2 // indirect
golang.org/x/crypto v0.40.0 // indirect
golang.org/x/net v0.42.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.27.0 // indirect
)

10
apps/backend/go.sum
View file

@ -70,6 +70,8 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/nickalie/go-binwrapper v0.0.0-20190114141239-525121d43c84 h1:/6MoQlTdk1eAi0J9O89ypO8umkp+H7mpnSF2ggSL62Q=
@ -88,10 +90,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@ -123,6 +129,10 @@ golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=

13
apps/backend/images.go
View file

@ -21,6 +21,7 @@ import (
"github.com/labstack/echo/v4"
"github.com/nfnt/resize"
"github.com/nickalie/go-webpbin"
"github.com/schollz/progressbar/v3"
)
func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
@ -45,7 +46,8 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
wg sync.WaitGroup
)
sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
totalImages := len(*toProcess)
bar := progressbar.Default(int64(totalImages), "Processing images")
// clear images folder
err := clearImageFolder()
if err != nil {
@ -78,9 +80,15 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
OGUrl: img.Url,
})
mu.Unlock()
bar.Add(1)
}(img)
}
wg.Wait()
bar.Finish()
fmt.Println()
totalToUpload := len(ProcessedImages)
uploadBar := progressbar.Default(int64(totalToUpload), "Uploading images")
uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
var uploadWg sync.WaitGroup
for _, img := range ProcessedImages {
@ -108,10 +116,13 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
ThumbId: thumbId,
OGUrl: img.OGUrl,
})
uploadBar.Add(1)
mu.Unlock()
}(img)
}
uploadWg.Wait()
uploadBar.Finish()
fmt.Println()
return UploadedImages, nil
}

9
apps/backend/miogest_handler.go
View file

@ -12,6 +12,8 @@ import (
"slices"
"strings"
"time"
"github.com/schollz/progressbar/v3"
)
type MiogestHandler struct {
@ -187,6 +189,8 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
}
totalAnnunci := len(annunci.Annuncio)
bar := progressbar.Default(int64(totalAnnunci), "Parsing annunci")
var AnnunciArray typesdefs.AnnunciParsed
AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio))
m.InitDefaults()
@ -196,9 +200,10 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
updateVideos := slices.Contains(video_codes_to_process, *annunci.Annuncio[i].Codice)
result := extractData_update(&annunci.Annuncio[i], imgToUpdate, updateVideos, m, imgPool, videoPool)
AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result)
bar.Add(1)
}
bar.Finish()
fmt.Println()
return AnnunciArray
}

1
apps/backend/queries/queries.go
View file

@ -20,6 +20,7 @@ func init() {
Port: os.Getenv("PGPORT"),
User: os.Getenv("POSTGRES_USER"),
Password: os.Getenv("POSTGRES_PASSWORD"),
LoggerEnabled: os.Getenv("DB_LOGGER") == "true",
})
}

37
apps/backend/server.go
View file

@ -119,25 +119,24 @@ func (s *Server) SetupRoutes() *echo.Echo {
videoPool := []typesdefs.MediaToProcess{}
var data = s.miogest.GetAnnunciParsed(&imgPool, &videoPool, forceMediaStale)
err := queries.AnnunciInsert(data, true)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
imgRefs, err := ProcessImagePool(&imgPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.AnnunciInsert(data, true)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetImagesToDB(&imgRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetVideosToDB(&videoRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
@ -150,22 +149,24 @@ func (s *Server) SetupRoutes() *echo.Echo {
imgPool := []typesdefs.MediaToProcess{}
videoPool := []typesdefs.MediaToProcess{}
var data = s.miogest.GetAnnuncioParsed(cod, &imgPool, &videoPool)
err := queries.AnnunciInsert(data, false)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
imgRefs, err := ProcessImagePool(&imgPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.AnnunciInsert(data, false)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetImagesToDB(&imgRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetVideosToDB(&videoRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())

12
apps/backend/video.go
View file

@ -20,6 +20,7 @@ import (
"github.com/labstack/echo/v4"
"github.com/nfnt/resize"
"github.com/nickalie/go-webpbin"
"github.com/schollz/progressbar/v3"
)
func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
@ -52,6 +53,8 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
mu sync.Mutex
wg sync.WaitGroup
)
totalToProcess := len(*toProcess)
bar := progressbar.Default(int64(totalToProcess), "Processing videos")
sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
// clear videos folder
@ -87,10 +90,16 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
ThumnailPath: thumbnailPath,
OGUrl: video.Url,
})
bar.Add(1)
mu.Unlock()
}(video)
}
wg.Wait()
bar.Finish()
fmt.Println()
totalToUpload := len(ProcessedVideos)
uploadBar := progressbar.Default(int64(totalToUpload), "Uploading videos")
uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
var uploadWg sync.WaitGroup
for _, video := range ProcessedVideos {
@ -118,10 +127,13 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
ThumbId: thumbId,
OGUrl: video.OGUrl,
})
uploadBar.Add(1)
mu.Unlock()
}(video)
}
uploadWg.Wait()
uploadBar.Finish()
fmt.Println()
return UploadedVideos, nil
}