From 4524f1b4bca0bd14b6ba8338f8bb1a941b803005 Mon Sep 17 00:00:00 2001 From: Marco Pedone Date: Wed, 15 Oct 2025 14:27:06 +0200 Subject: [PATCH] feat: refactor image handling and parsing logic, improve utility functions, and enhance route management --- apps/backend/from_bkp.go | 3 +- apps/backend/image_handler.go | 74 ++++++++++++++++++++++++---- apps/backend/miogest_handler.go | 87 ++++++++++++++++++++++++++++----- apps/backend/queries/annunci.go | 4 +- apps/backend/queries/images.go | 8 +++ apps/backend/server.go | 34 +++++++++++-- apps/backend/utils/utils.go | 41 ++++++++++++---- 7 files changed, 213 insertions(+), 38 deletions(-) diff --git a/apps/backend/from_bkp.go b/apps/backend/from_bkp.go index 601aecc..9d68cb8 100644 --- a/apps/backend/from_bkp.go +++ b/apps/backend/from_bkp.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "slices" "strings" "sync" "time" @@ -242,7 +243,7 @@ func TestTipologieBkp() { var tipologie []string for _, r := range bkp.Annunci.Record { if r.Tipo != nil { - if !utils.Contains(tipologie, *r.Tipo) { + if !slices.Contains(tipologie, *r.Tipo) { tipologie = append(tipologie, *r.Tipo) } } diff --git a/apps/backend/image_handler.go b/apps/backend/image_handler.go index c5960ce..140ab47 100644 --- a/apps/backend/image_handler.go +++ b/apps/backend/image_handler.go @@ -4,10 +4,12 @@ import ( "backend/config" "backend/queries" "backend/typesdefs" + "backend/utils" "fmt" "log" "os" "path/filepath" + "slices" ) type ImageRefManager struct { @@ -17,28 +19,29 @@ func NewImageRefManager() *ImageRefManager { return &ImageRefManager{} } -func (i *ImageRefManager) InsertImagesRef(annunci *typesdefs.AnnunciXML) error { +// add or update the miogest url in the miogest_images_ref table to keep track of the images associated with each codice +func (i *ImageRefManager) InsertImagesMiogestRef(annunci *typesdefs.AnnunciXML) error { return queries.ImagesBatchInsert(annunci) } -func (i *ImageRefManager) FindIsUpdate(annunci *typesdefs.AnnunciXML) ([]string, error) { - r, err := queries.ImagesGetAll() +func (i *ImageRefManager) GetImageCodesForProcessing(annunci *typesdefs.AnnunciXML) ([]string, error) { + db_images, err := queries.ImagesGetAll() if err != nil { return nil, err } - refMap := make(map[string][]string) // Convert slice to map for faster lookup - for _, ref := range r { + dbImagesMap := make(map[string][]string) // Convert slice to map for faster lookup + for _, ref := range db_images { if ref.Foto != nil { - refMap[ref.Codice] = *ref.Foto + dbImagesMap[ref.Codice] = *ref.Foto } } var result []string for _, annuncio := range annunci.Annuncio { if annuncio.Codice != nil && annuncio.Foto != nil { - refFoto, exists := refMap[*annuncio.Codice] - if !exists { + refFoto, existsInDb := dbImagesMap[*annuncio.Codice] + if !existsInDb { fmt.Println("new codice", *annuncio.Codice) result = append(result, *annuncio.Codice) continue @@ -65,17 +68,70 @@ func (i *ImageRefManager) FindIsUpdate(annunci *typesdefs.AnnunciXML) ([]string, func (i *ImageRefManager) ClearImages(to_clear []string) error { for _, codice := range to_clear { - folder := filepath.Join(config.Cfg.Images.Path, codice) + err := utils.SafeRemoveAll(folder) + if err != nil { + return err + } + } + return nil +} + +func (i *ImageRefManager) RemoveImage(to_clear string) error { + + folder := filepath.Join(config.Cfg.Images.Path, to_clear) + fmt.Println("Removing folder:", folder) + exists, err := utils.PathExists(folder) + fmt.Println("Folder exists:", exists) + if err != nil { + return err + } + if folder != "" && exists { + fmt.Println("Removing folder:", folder) err := os.RemoveAll(folder) + fmt.Println("Folder removed:", folder) if err != nil { return err } } + err = queries.ImagesClearByCodice(to_clear) + fmt.Println("DB entries removed for codice:", to_clear) + if err != nil { + return err + } + return nil } +func difference(a, b []string) []string { + var diff []string + for _, v := range a { + if !slices.Contains(b, v) { + diff = append(diff, v) + } + } + return diff +} + +func (i *ImageRefManager) MissingImageFolders() ([]string, error) { + + // Get all image folders + folders := utils.GetPathDirsArray(config.Cfg.Images.Path) + // Get all image codes from DB + db_images, err := queries.ImagesGetAll() + if err != nil { + return []string{}, err + } + dbArray := make([]string, len(db_images)) + for i, ref := range db_images { + dbArray[i] = ref.Codice + } + + return difference(dbArray, folders), nil + +} + func (i *ImageRefManager) Reset() error { err := queries.ImagesClear() if err != nil { diff --git a/apps/backend/miogest_handler.go b/apps/backend/miogest_handler.go index 307715b..3917703 100644 --- a/apps/backend/miogest_handler.go +++ b/apps/backend/miogest_handler.go @@ -96,23 +96,67 @@ func (m *MiogestHandler) GenerateCategorie() { // ANNUNCI PARSED func (m *MiogestHandler) GetAnnunciParsed(i *ImageRefManager) typesdefs.AnnunciParsed { - m.ParseAnnunci(i) + // salvo il parsing nello struct nel caso voglio implementare un caching e non fare il parsing ad ogni chiamata + m.AnnunciParsed = m.ParseAnnunci("", i) return m.AnnunciParsed } -func (m *MiogestHandler) ParseAnnunci(i *ImageRefManager) { +// versione per singolo annuncio +func (m *MiogestHandler) GetAnnuncioParsed(cod string, i *ImageRefManager) typesdefs.AnnuncioParsed { + data := m.ParseAnnunci2(cod, i) + if len(data.Annuncio) == 0 { + return typesdefs.AnnuncioParsed{} + } + return data.Annuncio[0] + +} +func (m *MiogestHandler) ParseAnnunci2(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed { annunci := m.GetAnnunci() - cod_image_to_update := []string{} + + if codFilter != "" { + if len(annunci.Annuncio) == 0 { + return typesdefs.AnnunciParsed{} + } + if slices.ContainsFunc(annunci.Annuncio, func(a typesdefs.AnnuncioXML) bool { + + return *a.Codice == codFilter + }) { + + annunci.Annuncio = []typesdefs.AnnuncioXML{annunci.Annuncio[slices.IndexFunc(annunci.Annuncio, func(a typesdefs.AnnuncioXML) bool { + return *a.Codice == codFilter + })]} + //go on + } else { + return typesdefs.AnnunciParsed{} + } + } + return typesdefs.AnnunciParsed{} +} + +func (m *MiogestHandler) ParseAnnunci(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed { + annunci := m.GetAnnunci() + image_codes_to_process := []string{} if config.Cfg.Images.Enabled { var err error - cod_image_to_update, err = i.FindIsUpdate(&annunci) + image_codes_to_process, err = i.GetImageCodesForProcessing(&annunci) if err != nil { log.Fatalf("Failed to find images to update: %v", err.Error()) } - if err := i.ClearImages(cod_image_to_update); err != nil { + missing_codes, err := i.MissingImageFolders() + if err != nil { + log.Fatalf("Failed to find missing image folders: %v", err.Error()) + } + + if len(missing_codes) > 0 { + image_codes_to_process = append(image_codes_to_process, missing_codes...) + image_codes_to_process = slices.Compact(image_codes_to_process) + } + // Clear images folder of codes that need to be updated + if err := i.ClearImages(image_codes_to_process); err != nil { log.Fatalf("Failed to clear images: %v", err.Error()) } - i.InsertImagesRef(&annunci) + + i.InsertImagesMiogestRef(&annunci) } var AnnunciArray typesdefs.AnnunciParsed @@ -129,20 +173,20 @@ func (m *MiogestHandler) ParseAnnunci(i *ImageRefManager) { go func(i int) { defer wg.Done() defer func() { <-sem }() - result := extractData_update(&annunci.Annuncio[i], utils.Contains(cod_image_to_update, *annunci.Annuncio[i].Codice), m) + result := extractData_update(&annunci.Annuncio[i], slices.Contains(image_codes_to_process, *annunci.Annuncio[i].Codice), m) AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result) }(i) } wg.Wait() if config.Cfg.Images.Enabled { for _, codice := range lista_codici { - err := folderCleanup(codice) + err := removeNonWebpImages(codice) if err != nil { log.Fatalf("Failed to cleanup folder: %v", err.Error()) } } } - m.AnnunciParsed = AnnunciArray + return AnnunciArray } var re = regexp.MustCompile(`[^+\d]`) @@ -343,7 +387,19 @@ func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Annu func processImages(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { path := filepath.Join(config.Cfg.Images.Path, tmp.Codice) - err := os.MkdirAll(path, 0755) + folderExists, err := utils.PathExists(path) + if err != nil { + log.Fatalf("Failed to check if folder exists: %v", err) + } + if folderExists { + // Remove existing folder to avoid stale images + err := utils.SafeRemoveAll(path) + if err != nil { + log.Fatalf("Failed to remove existing folder: %v", err) + } + } + // Create directory if it doesn't exist + err = os.MkdirAll(path, 0755) if err != nil { fmt.Printf("Failed to create directory: %v\n", err) panic(err) @@ -416,11 +472,18 @@ func processWebpImages(path string) { wg.Wait() } -func folderCleanup(codice string) error { +func removeNonWebpImages(codice string) error { folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, codice)) non_webp := []string{} for _, folder := range folders { folderpath := filepath.Join(config.Cfg.Images.Path, codice, folder) + exists, err := utils.PathExists(folderpath) + if err != nil { + return err + } + if !exists { + continue + } files, err := os.ReadDir(folderpath) if err != nil { return err @@ -432,7 +495,7 @@ func folderCleanup(codice string) error { } } for _, file := range non_webp { - err := os.Remove(file) + err := utils.SafeRemove(file) if err != nil { return err } diff --git a/apps/backend/queries/annunci.go b/apps/backend/queries/annunci.go index 632c9d0..b03a661 100644 --- a/apps/backend/queries/annunci.go +++ b/apps/backend/queries/annunci.go @@ -2,8 +2,8 @@ package queries import ( "backend/typesdefs" - "backend/utils" "fmt" + "slices" "github.com/georgysavva/scany/v2/pgxscan" "github.com/jackc/pgx/v5" @@ -35,7 +35,7 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed) error { batch := &pgx.Batch{} for _, annuncio := range annunci.Annuncio { - if utils.Contains(codici, annuncio.Codice) { + if slices.Contains(codici, annuncio.Codice) { batch.Queue(` UPDATE public.annunci SET locatore = $1, diff --git a/apps/backend/queries/images.go b/apps/backend/queries/images.go index 3c152d8..e92e7a1 100644 --- a/apps/backend/queries/images.go +++ b/apps/backend/queries/images.go @@ -42,3 +42,11 @@ func ImagesClear() error { } return nil } + +func ImagesClearByCodice(codice string) error { + _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.miogest_images_ref WHERE codice = $1", codice) + if err != nil { + return fmt.Errorf("failed to clear images by codice: %w", err) + } + return nil +} diff --git a/apps/backend/server.go b/apps/backend/server.go index 44d48a3..0731e6a 100644 --- a/apps/backend/server.go +++ b/apps/backend/server.go @@ -4,6 +4,7 @@ import ( "backend/config" "backend/queries" storagehandlers "backend/storage_handlers" + "backend/utils" "bytes" "fmt" "io" @@ -79,6 +80,8 @@ func (s *Server) SetupRoutes() *echo.Echo { }) }) e.GET("/test", func(c echo.Context) error { + + s.imageManager.MissingImageFolders() return c.String(http.StatusOK, "OK") }) e.POST("/testpost", func(c echo.Context) error { @@ -127,6 +130,13 @@ func (s *Server) SetupRoutes() *echo.Echo { } return c.String(http.StatusOK, "Updated") }) + e.GET("/update-cod/:cod", func(c echo.Context) error { + + cod := c.Param("cod") + + var _ = s.miogest.GetAnnuncioParsed(cod, s.imageManager) + return c.String(http.StatusOK, "Updated") + }) // BACKUP ROUTES e.GET("/upload-backup", func(c echo.Context) error { @@ -255,11 +265,15 @@ func (s *Server) SetupRoutes() *echo.Echo { }) e.GET("/images/list", func(c echo.Context) error { - annunci, err := queries.AnnunciGetActive() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - return c.JSON(http.StatusOK, annunci) + folders := utils.GetPathDirsArray(config.Cfg.Images.Path) + return c.JSON(http.StatusOK, folders) + + // annunci, err := queries.AnnunciGetActive() + // if err != nil { + // return c.String(http.StatusInternalServerError, err.Error()) + // } + // return c.JSON(http.StatusOK, annunci) + }) e.GET("/images/get/:cod/:imgNum", func(c echo.Context) error { @@ -322,6 +336,16 @@ func (s *Server) SetupRoutes() *echo.Echo { return c.String(http.StatusOK, "OK") }) + e.GET("/images/remove/:cod", func(c echo.Context) error { + cod := c.Param("cod") + + err := s.imageManager.RemoveImage(cod) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + + return c.String(http.StatusOK, "OK") + }) e.POST("/image-option-toggle", func(c echo.Context) error { // Toggle the imageOption value diff --git a/apps/backend/utils/utils.go b/apps/backend/utils/utils.go index d98d086..c601936 100644 --- a/apps/backend/utils/utils.go +++ b/apps/backend/utils/utils.go @@ -90,6 +90,38 @@ func GetPathsArray(path string) ([]string, error) { return paths, nil } +func SafeRemove(path string) error { + err := os.Remove(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return nil +} +func SafeRemoveAll(path string) error { + err := os.RemoveAll(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return nil +} + +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + func GetPathDirsArray(path string) []string { var folderNames []string files, err := os.ReadDir(path) @@ -122,12 +154,3 @@ func CommaToDot(str *string) *string { } return nil } - -func Contains(s []string, str string) bool { - for _, v := range s { - if v == str { - return true - } - } - return false -}