feat: refactor image handling and parsing logic, improve utility functions, and enhance route management

apps/backend/from_bkp.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -242,7 +243,7 @@ func TestTipologieBkp() {
 	var tipologie []string
 	for _, r := range bkp.Annunci.Record {
 		if r.Tipo != nil {
-			if !utils.Contains(tipologie, *r.Tipo) {
+			if !slices.Contains(tipologie, *r.Tipo) {
 				tipologie = append(tipologie, *r.Tipo)
 			}
 		}

apps/backend/image_handler.go

@@ -4,10 +4,12 @@ import (
 	"backend/config"
 	"backend/queries"
 	"backend/typesdefs"
+	"backend/utils"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"slices"
 )
 
 type ImageRefManager struct {
@@ -17,28 +19,29 @@ func NewImageRefManager() *ImageRefManager {
 	return &ImageRefManager{}
 }
 
-func (i *ImageRefManager) InsertImagesRef(annunci *typesdefs.AnnunciXML) error {
+// add or update the miogest url in the miogest_images_ref table to keep track of the images associated with each codice
+func (i *ImageRefManager) InsertImagesMiogestRef(annunci *typesdefs.AnnunciXML) error {
 
 	return queries.ImagesBatchInsert(annunci)
 }
 
-func (i *ImageRefManager) FindIsUpdate(annunci *typesdefs.AnnunciXML) ([]string, error) {
-	r, err := queries.ImagesGetAll()
+func (i *ImageRefManager) GetImageCodesForProcessing(annunci *typesdefs.AnnunciXML) ([]string, error) {
+	db_images, err := queries.ImagesGetAll()
 	if err != nil {
 		return nil, err
 	}
-	refMap := make(map[string][]string) // Convert slice to map for faster lookup
-	for _, ref := range r {
+	dbImagesMap := make(map[string][]string) // Convert slice to map for faster lookup
+	for _, ref := range db_images {
 		if ref.Foto != nil {
-			refMap[ref.Codice] = *ref.Foto
+			dbImagesMap[ref.Codice] = *ref.Foto
 		}
 	}
 
 	var result []string
 	for _, annuncio := range annunci.Annuncio {
 		if annuncio.Codice != nil && annuncio.Foto != nil {
-			refFoto, exists := refMap[*annuncio.Codice]
-			if !exists {
+			refFoto, existsInDb := dbImagesMap[*annuncio.Codice]
+			if !existsInDb {
 				fmt.Println("new codice", *annuncio.Codice)
 				result = append(result, *annuncio.Codice)
 				continue
@@ -65,17 +68,70 @@ func (i *ImageRefManager) FindIsUpdate(annunci *typesdefs.AnnunciXML) ([]string,
 func (i *ImageRefManager) ClearImages(to_clear []string) error {
 
 	for _, codice := range to_clear {
-
 		folder := filepath.Join(config.Cfg.Images.Path, codice)
+		err := utils.SafeRemoveAll(folder)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (i *ImageRefManager) RemoveImage(to_clear string) error {
+
+	folder := filepath.Join(config.Cfg.Images.Path, to_clear)
+	fmt.Println("Removing folder:", folder)
+	exists, err := utils.PathExists(folder)
+	fmt.Println("Folder exists:", exists)
+	if err != nil {
+		return err
+	}
+	if folder != "" && exists {
+		fmt.Println("Removing folder:", folder)
 		err := os.RemoveAll(folder)
+		fmt.Println("Folder removed:", folder)
 		if err != nil {
 			return err
 		}
 
 	}
+	err = queries.ImagesClearByCodice(to_clear)
+	fmt.Println("DB entries removed for codice:", to_clear)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
+func difference(a, b []string) []string {
+	var diff []string
+	for _, v := range a {
+		if !slices.Contains(b, v) {
+			diff = append(diff, v)
+		}
+	}
+	return diff
+}
+
+func (i *ImageRefManager) MissingImageFolders() ([]string, error) {
+
+	// Get all image folders
+	folders := utils.GetPathDirsArray(config.Cfg.Images.Path)
+	// Get all image codes from DB
+	db_images, err := queries.ImagesGetAll()
+	if err != nil {
+		return []string{}, err
+	}
+	dbArray := make([]string, len(db_images))
+	for i, ref := range db_images {
+		dbArray[i] = ref.Codice
+	}
+
+	return difference(dbArray, folders), nil
+
+}
+
 func (i *ImageRefManager) Reset() error {
 	err := queries.ImagesClear()
 	if err != nil {

apps/backend/miogest_handler.go

@@ -96,23 +96,67 @@ func (m *MiogestHandler) GenerateCategorie() {
 
 // ANNUNCI PARSED
 func (m *MiogestHandler) GetAnnunciParsed(i *ImageRefManager) typesdefs.AnnunciParsed {
-	m.ParseAnnunci(i)
+	// salvo il parsing nello struct nel caso voglio implementare un caching e non fare il parsing ad ogni chiamata
+	m.AnnunciParsed = m.ParseAnnunci("", i)
 	return m.AnnunciParsed
 }
 
-func (m *MiogestHandler) ParseAnnunci(i *ImageRefManager) {
+// versione per singolo annuncio
+func (m *MiogestHandler) GetAnnuncioParsed(cod string, i *ImageRefManager) typesdefs.AnnuncioParsed {
+	data := m.ParseAnnunci2(cod, i)
+	if len(data.Annuncio) == 0 {
+		return typesdefs.AnnuncioParsed{}
+	}
+	return data.Annuncio[0]
+
+}
+func (m *MiogestHandler) ParseAnnunci2(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed {
 	annunci := m.GetAnnunci()
-	cod_image_to_update := []string{}
+
+	if codFilter != "" {
+		if len(annunci.Annuncio) == 0 {
+			return typesdefs.AnnunciParsed{}
+		}
+		if slices.ContainsFunc(annunci.Annuncio, func(a typesdefs.AnnuncioXML) bool {
+
+			return *a.Codice == codFilter
+		}) {
+
+			annunci.Annuncio = []typesdefs.AnnuncioXML{annunci.Annuncio[slices.IndexFunc(annunci.Annuncio, func(a typesdefs.AnnuncioXML) bool {
+				return *a.Codice == codFilter
+			})]}
+			//go on
+		} else {
+			return typesdefs.AnnunciParsed{}
+		}
+	}
+	return typesdefs.AnnunciParsed{}
+}
+
+func (m *MiogestHandler) ParseAnnunci(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed {
+	annunci := m.GetAnnunci()
+	image_codes_to_process := []string{}
 	if config.Cfg.Images.Enabled {
 		var err error
-		cod_image_to_update, err = i.FindIsUpdate(&annunci)
+		image_codes_to_process, err = i.GetImageCodesForProcessing(&annunci)
 		if err != nil {
 			log.Fatalf("Failed to find images to update: %v", err.Error())
 		}
-		if err := i.ClearImages(cod_image_to_update); err != nil {
+		missing_codes, err := i.MissingImageFolders()
+		if err != nil {
+			log.Fatalf("Failed to find missing image folders: %v", err.Error())
+		}
+
+		if len(missing_codes) > 0 {
+			image_codes_to_process = append(image_codes_to_process, missing_codes...)
+			image_codes_to_process = slices.Compact(image_codes_to_process)
+		}
+		// Clear images folder of codes that need to be updated
+		if err := i.ClearImages(image_codes_to_process); err != nil {
 			log.Fatalf("Failed to clear images: %v", err.Error())
 		}
-		i.InsertImagesRef(&annunci)
+
+		i.InsertImagesMiogestRef(&annunci)
 	}
 
 	var AnnunciArray typesdefs.AnnunciParsed
@@ -129,20 +173,20 @@ func (m *MiogestHandler) ParseAnnunci(i *ImageRefManager) {
 		go func(i int) {
 			defer wg.Done()
 			defer func() { <-sem }()
-			result := extractData_update(&annunci.Annuncio[i], utils.Contains(cod_image_to_update, *annunci.Annuncio[i].Codice), m)
+			result := extractData_update(&annunci.Annuncio[i], slices.Contains(image_codes_to_process, *annunci.Annuncio[i].Codice), m)
 			AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result)
 		}(i)
 	}
 	wg.Wait()
 	if config.Cfg.Images.Enabled {
 		for _, codice := range lista_codici {
-			err := folderCleanup(codice)
+			err := removeNonWebpImages(codice)
 			if err != nil {
 				log.Fatalf("Failed to cleanup folder: %v", err.Error())
 			}
 		}
 	}
-	m.AnnunciParsed = AnnunciArray
+	return AnnunciArray
 }
 
 var re = regexp.MustCompile(`[^+\d]`)
@@ -343,7 +387,19 @@ func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Annu
 
 func processImages(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
 	path := filepath.Join(config.Cfg.Images.Path, tmp.Codice)
-	err := os.MkdirAll(path, 0755)
+	folderExists, err := utils.PathExists(path)
+	if err != nil {
+		log.Fatalf("Failed to check if folder exists: %v", err)
+	}
+	if folderExists {
+		// Remove existing folder to avoid stale images
+		err := utils.SafeRemoveAll(path)
+		if err != nil {
+			log.Fatalf("Failed to remove existing folder: %v", err)
+		}
+	}
+	// Create directory if it doesn't exist
+	err = os.MkdirAll(path, 0755)
 	if err != nil {
 		fmt.Printf("Failed to create directory: %v\n", err)
 		panic(err)
@@ -416,11 +472,18 @@ func processWebpImages(path string) {
 	wg.Wait()
 }
 
-func folderCleanup(codice string) error {
+func removeNonWebpImages(codice string) error {
 	folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, codice))
 	non_webp := []string{}
 	for _, folder := range folders {
 		folderpath := filepath.Join(config.Cfg.Images.Path, codice, folder)
+		exists, err := utils.PathExists(folderpath)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			continue
+		}
 		files, err := os.ReadDir(folderpath)
 		if err != nil {
 			return err
@@ -432,7 +495,7 @@ func folderCleanup(codice string) error {
 		}
 	}
 	for _, file := range non_webp {
-		err := os.Remove(file)
+		err := utils.SafeRemove(file)
 		if err != nil {
 			return err
 		}

apps/backend/queries/annunci.go

@@ -2,8 +2,8 @@ package queries
 
 import (
 	"backend/typesdefs"
-	"backend/utils"
 	"fmt"
+	"slices"
 
 	"github.com/georgysavva/scany/v2/pgxscan"
 	"github.com/jackc/pgx/v5"
@@ -35,7 +35,7 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed) error {
 	batch := &pgx.Batch{}
 	for _, annuncio := range annunci.Annuncio {
 
-		if utils.Contains(codici, annuncio.Codice) {
+		if slices.Contains(codici, annuncio.Codice) {
 			batch.Queue(`
 			 UPDATE public.annunci
 			SET locatore = $1,

apps/backend/queries/images.go

@@ -42,3 +42,11 @@ func ImagesClear() error {
 	}
 	return nil
 }
+
+func ImagesClearByCodice(codice string) error {
+	_, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.miogest_images_ref WHERE codice = $1", codice)
+	if err != nil {
+		return fmt.Errorf("failed to clear images by codice: %w", err)
+	}
+	return nil
+}

apps/backend/server.go

@@ -4,6 +4,7 @@ import (
 	"backend/config"
 	"backend/queries"
 	storagehandlers "backend/storage_handlers"
+	"backend/utils"
 	"bytes"
 	"fmt"
 	"io"
@@ -79,6 +80,8 @@ func (s *Server) SetupRoutes() *echo.Echo {
 		})
 	})
 	e.GET("/test", func(c echo.Context) error {
+
+		s.imageManager.MissingImageFolders()
 		return c.String(http.StatusOK, "OK")
 	})
 	e.POST("/testpost", func(c echo.Context) error {
@@ -127,6 +130,13 @@ func (s *Server) SetupRoutes() *echo.Echo {
 		}
 		return c.String(http.StatusOK, "Updated")
 	})
+	e.GET("/update-cod/:cod", func(c echo.Context) error {
+
+		cod := c.Param("cod")
+
+		var _ = s.miogest.GetAnnuncioParsed(cod, s.imageManager)
+		return c.String(http.StatusOK, "Updated")
+	})
 
 	// BACKUP ROUTES
 	e.GET("/upload-backup", func(c echo.Context) error {
@@ -255,11 +265,15 @@ func (s *Server) SetupRoutes() *echo.Echo {
 	})
 
 	e.GET("/images/list", func(c echo.Context) error {
-		annunci, err := queries.AnnunciGetActive()
-		if err != nil {
-			return c.String(http.StatusInternalServerError, err.Error())
-		}
-		return c.JSON(http.StatusOK, annunci)
+		folders := utils.GetPathDirsArray(config.Cfg.Images.Path)
+		return c.JSON(http.StatusOK, folders)
+
+		// annunci, err := queries.AnnunciGetActive()
+		// if err != nil {
+		// 	return c.String(http.StatusInternalServerError, err.Error())
+		// }
+		// return c.JSON(http.StatusOK, annunci)
+
 	})
 
 	e.GET("/images/get/:cod/:imgNum", func(c echo.Context) error {
@@ -322,6 +336,16 @@ func (s *Server) SetupRoutes() *echo.Echo {
 
 		return c.String(http.StatusOK, "OK")
 	})
+	e.GET("/images/remove/:cod", func(c echo.Context) error {
+		cod := c.Param("cod")
+
+		err := s.imageManager.RemoveImage(cod)
+		if err != nil {
+			return c.String(http.StatusInternalServerError, err.Error())
+		}
+
+		return c.String(http.StatusOK, "OK")
+	})
 
 	e.POST("/image-option-toggle", func(c echo.Context) error {
 		// Toggle the imageOption value

apps/backend/utils/utils.go

@@ -90,6 +90,38 @@ func GetPathsArray(path string) ([]string, error) {
 	return paths, nil
 }
 
+func SafeRemove(path string) error {
+	err := os.Remove(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+func SafeRemoveAll(path string) error {
+	err := os.RemoveAll(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
 func GetPathDirsArray(path string) []string {
 	var folderNames []string
 	files, err := os.ReadDir(path)
@@ -122,12 +154,3 @@ func CommaToDot(str *string) *string {
 	}
 	return nil
 }
-
-func Contains(s []string, str string) bool {
-	for _, v := range s {
-		if v == str {
-			return true
-		}
-	}
-	return false
-}