diff --git a/apps/backend/.air.toml b/apps/backend/.air.toml index d1cd51c..8d61370 100644 --- a/apps/backend/.air.toml +++ b/apps/backend/.air.toml @@ -7,7 +7,7 @@ tmp_dir = "tmp" bin = "tmp\\main.exe" cmd = "go build -o ./tmp/main.exe ." delay = 1000 - exclude_dir = ["assets", "tmp", "vendor", "testdata", "images", ".bin", "storage"] + exclude_dir = ["assets", "tmp", "vendor", "testdata", "images", ".bin"] exclude_file = [] exclude_regex = ["_test.go"] exclude_unchanged = false diff --git a/apps/backend/.dockerignore b/apps/backend/.dockerignore index 097143d..19d2d19 100644 --- a/apps/backend/.dockerignore +++ b/apps/backend/.dockerignore @@ -1,5 +1,4 @@ images -storage tmp vscode .vscode diff --git a/apps/backend/.gitignore b/apps/backend/.gitignore index 77c05a4..f62a31b 100644 --- a/apps/backend/.gitignore +++ b/apps/backend/.gitignore @@ -1,7 +1,6 @@ tmp images bkp.xml -storage .bin web.config .env diff --git a/apps/backend/Dockerfile b/apps/backend/Dockerfile index 7729cca..f687977 100644 --- a/apps/backend/Dockerfile +++ b/apps/backend/Dockerfile @@ -26,7 +26,7 @@ ARG PGHOST ARG PGPORT ARG IMAGEOPTION ARG CONCURRENT_IMAGES -ARG STORAGE_STRATEGY +ARG STORAGE_URL ARG MINIO_ENDPOINT ARG MINIO_PORT ARG MINIO_ROOT_USER @@ -42,7 +42,7 @@ ENV POSTGRES_USER=$POSTGRES_USER ENV POSTGRES_PASSWORD=$POSTGRES_PASSWORD ENV IMAGEOPTION=$IMAGEOPTION ENV CONCURRENT_IMAGES=$CONCURRENT_IMAGES -ENV STORAGE_STRATEGY=$STORAGE_STRATEGY +ENV STORAGE_URL=$STORAGE_URL ENV MINIO_ENDPOINT=$MINIO_ENDPOINT ENV MINIO_PORT=$MINIO_PORT ENV MINIO_ROOT_USER=$MINIO_ROOT_USER @@ -58,10 +58,8 @@ COPY --from=builder /app/scripts /app/scripts RUN chmod +x /app/scripts/*.sh #COPY --from=builder /app/.env /app/.env -# create images and storage folder +# create images RUN mkdir images -RUN mkdir storage -RUN mkdir storage/tmp RUN apk add curl diff --git a/apps/backend/config/config.go b/apps/backend/config/config.go index b55f28b..5566f07 100644 --- a/apps/backend/config/config.go +++ b/apps/backend/config/config.go @@ -1,7 +1,6 @@ package config import ( - "backend/typesdefs" "fmt" "os" "path/filepath" @@ -28,17 +27,7 @@ type Config struct { Path string } Storage struct { - Strategy typesdefs.StorageStrategy - Path string - TempLimit int - Minio struct { - URL string - RootUser string - Password string - } - GOFS struct { - Endpoint string - } + Endpoint string } Server struct { @@ -78,28 +67,7 @@ func Load() *Config { Cfg.Images.Path = filepath.Join(pwd, "images") // Storage config - Cfg.Storage.Path = filepath.Join(pwd, "storage") - Cfg.Storage.TempLimit = getEnvAsInt("TEMP_STORAGE_LIMIT", 5) - strategy := getEnv("STORAGE_STRATEGY", "fs") - - switch strategy { - case "s3", "minio": - Cfg.Storage.Minio.URL = getEnv("MINIO_URL", "") - Cfg.Storage.Minio.RootUser = getEnv("MINIO_ROOT_USER", "") - Cfg.Storage.Minio.Password = getEnv("MINIO_ROOT_PASSWORD", "") - - if Cfg.Storage.Minio.URL == "" || Cfg.Storage.Minio.RootUser == "" || Cfg.Storage.Minio.Password == "" { - fmt.Println("Error: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD and MINIO_URL must be set") - Cfg.Storage.Strategy = typesdefs.FS - } else { - Cfg.Storage.Strategy = typesdefs.S3 - } - case "gofs": - Cfg.Storage.GOFS.Endpoint = getEnv("GOFS_ENDPOINT", "localhost:8080") - Cfg.Storage.Strategy = typesdefs.GOFS - default: - Cfg.Storage.Strategy = typesdefs.FS - } + Cfg.Storage.Endpoint = filepath.Join(pwd, "STORAGE_URL") // Server config Cfg.Server.Port = getEnvAsInt("PORT", 1323) diff --git a/apps/backend/docker-compose.yml b/apps/backend/docker-compose.yml deleted file mode 100644 index 88af893..0000000 --- a/apps/backend/docker-compose.yml +++ /dev/null @@ -1,98 +0,0 @@ -services: - minio: - image: minio/minio - expose: - - "9000" - - "9001" - volumes: - - minio_storage:/data - networks: - - dokploy-network - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - command: server --console-address ":9001" /data - healthcheck: - test: [ "CMD", "curl", "-f", "http://minio:9000/minio/health/live" ] - interval: 10s - timeout: 10s - retries: 5 - - tiles: - image: eqalpha/keydb:latest - volumes: - - tiles-data:/data - networks: - - dokploy-network - expose: - - "6379" - command: keydb-server /etc/keydb/keydb.conf --maxmemory 100mb --maxmemory-policy volatile-lru --maxmemory-samples 5 - healthcheck: - test: ["CMD", "keydb-cli", "-h", "localhost", "ping"] - interval: 5s - timeout: 3s - retries: 5 - restart: unless-stopped - deploy: - resources: - limits: - cpus: "0.5" - - keydb: - image: eqalpha/keydb:latest - volumes: - - keydb-data:/data - networks: - - dokploy-network - expose: - - "6379" - healthcheck: - test: ["CMD", "keydb-cli", "-h", "localhost", "ping"] - interval: 5s - timeout: 3s - retries: 5 - restart: unless-stopped - deploy: - resources: - limits: - cpus: "0.5" - - backend: - build: - context: . - dockerfile: Dockerfile - networks: - - dokploy-network - expose: - - 1323 - restart: unless-stopped - volumes: - - images_volume:/app/images - - storage_volume:/app/storage - environment: - STORAGE_STRATEGY: "minio" - MINIO_URL: minio:9000 - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB} - PGHOST: db - PGPORT: "5432" - IMAGEOPTION: "true" - CONCURRENT_IMAGES: ${CONCURRENT_IMAGES} - depends_on: - minio: - condition: service_healthy - -volumes: - minio_storage: - images_volume: - storage_volume: - tiles-data: - keydb-data: - - -networks: - dokploy-network: - external: true diff --git a/apps/backend/from_bkp.go b/apps/backend/from_bkp.go index 9d68cb8..0384da1 100644 --- a/apps/backend/from_bkp.go +++ b/apps/backend/from_bkp.go @@ -230,7 +230,6 @@ func extractData_bkp(annuncio typesdefs.AnnuncioBKP, extractedData chan typesdef } tmp.Caratteristiche = &tmpCaratteristiche - tmp.Url_foto = nil tmp.Url_video = nil extractedData <- tmp diff --git a/apps/backend/image_handler.go b/apps/backend/image_handler.go deleted file mode 100644 index 140ab47..0000000 --- a/apps/backend/image_handler.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "backend/config" - "backend/queries" - "backend/typesdefs" - "backend/utils" - "fmt" - "log" - "os" - "path/filepath" - "slices" -) - -type ImageRefManager struct { -} - -func NewImageRefManager() *ImageRefManager { - return &ImageRefManager{} -} - -// add or update the miogest url in the miogest_images_ref table to keep track of the images associated with each codice -func (i *ImageRefManager) InsertImagesMiogestRef(annunci *typesdefs.AnnunciXML) error { - - return queries.ImagesBatchInsert(annunci) -} - -func (i *ImageRefManager) GetImageCodesForProcessing(annunci *typesdefs.AnnunciXML) ([]string, error) { - db_images, err := queries.ImagesGetAll() - if err != nil { - return nil, err - } - dbImagesMap := make(map[string][]string) // Convert slice to map for faster lookup - for _, ref := range db_images { - if ref.Foto != nil { - dbImagesMap[ref.Codice] = *ref.Foto - } - } - - var result []string - for _, annuncio := range annunci.Annuncio { - if annuncio.Codice != nil && annuncio.Foto != nil { - refFoto, existsInDb := dbImagesMap[*annuncio.Codice] - if !existsInDb { - fmt.Println("new codice", *annuncio.Codice) - result = append(result, *annuncio.Codice) - continue - } - - if len(*annuncio.Foto) != len(refFoto) { - fmt.Println("Different length", *annuncio.Foto, refFoto) - result = append(result, *annuncio.Codice) - continue - } - - for i, foto := range *annuncio.Foto { - if foto != refFoto[i] { - fmt.Println("Different content", *annuncio.Foto, refFoto) - result = append(result, *annuncio.Codice) - break - } - } - } - } - return result, nil -} - -func (i *ImageRefManager) ClearImages(to_clear []string) error { - - for _, codice := range to_clear { - folder := filepath.Join(config.Cfg.Images.Path, codice) - err := utils.SafeRemoveAll(folder) - if err != nil { - return err - } - } - return nil -} - -func (i *ImageRefManager) RemoveImage(to_clear string) error { - - folder := filepath.Join(config.Cfg.Images.Path, to_clear) - fmt.Println("Removing folder:", folder) - exists, err := utils.PathExists(folder) - fmt.Println("Folder exists:", exists) - if err != nil { - return err - } - if folder != "" && exists { - fmt.Println("Removing folder:", folder) - err := os.RemoveAll(folder) - fmt.Println("Folder removed:", folder) - if err != nil { - return err - } - - } - err = queries.ImagesClearByCodice(to_clear) - fmt.Println("DB entries removed for codice:", to_clear) - if err != nil { - return err - } - - return nil -} - -func difference(a, b []string) []string { - var diff []string - for _, v := range a { - if !slices.Contains(b, v) { - diff = append(diff, v) - } - } - return diff -} - -func (i *ImageRefManager) MissingImageFolders() ([]string, error) { - - // Get all image folders - folders := utils.GetPathDirsArray(config.Cfg.Images.Path) - // Get all image codes from DB - db_images, err := queries.ImagesGetAll() - if err != nil { - return []string{}, err - } - dbArray := make([]string, len(db_images)) - for i, ref := range db_images { - dbArray[i] = ref.Codice - } - - return difference(dbArray, folders), nil - -} - -func (i *ImageRefManager) Reset() error { - err := queries.ImagesClear() - if err != nil { - return err - } - - err = queries.AnnunciResetImages() - if err != nil { - return err - } - - err = os.RemoveAll(config.Cfg.Images.Path) - if err != nil { - log.Fatal(err) - } - err = os.Mkdir(config.Cfg.Images.Path, 0755) - if err != nil { - log.Fatal(err) - } - - return nil -} diff --git a/apps/backend/images.go b/apps/backend/images.go index a4f8ea1..346b602 100644 --- a/apps/backend/images.go +++ b/apps/backend/images.go @@ -1,21 +1,162 @@ package main import ( + "backend/config" + "backend/queries" + "backend/typesdefs" + "backend/utils" "fmt" "image" "image/jpeg" "image/png" + "io" "log" "net/http" "os" "path/filepath" + "slices" + "sync" "github.com/labstack/echo/v4" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" ) -func Conversion(input string) error { +type ImageProcessing struct { + ImagesToProcess []typesdefs.ImagesToProcess + ProcessedImages []typesdefs.ProcessedImage + UploadedImages []typesdefs.UploadedImage +} + +func NewImageProcessing() *ImageProcessing { + return &ImageProcessing{ + ImagesToProcess: []typesdefs.ImagesToProcess{}, + ProcessedImages: []typesdefs.ProcessedImage{}, + UploadedImages: []typesdefs.UploadedImage{}, + } +} + +func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.ImagesToProcess) { + + if annuncio.Foto == nil || len(*annuncio.Foto) == 0 { + return + } + for idx, url := range *annuncio.Foto { + *p = append(*p, typesdefs.ImagesToProcess{ + Codice: tmp.Codice, + Url: url, + Order: idx, + }) + } +} + +func ProcessImagePool(toProcess *[]typesdefs.ImagesToProcess) ([]typesdefs.UploadedImage, error) { + var ( + ProcessedImages []typesdefs.ProcessedImage + UploadedImages []typesdefs.UploadedImage + mu sync.Mutex + wg sync.WaitGroup + ) + sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) + + // clear images folder + err := clearImageFolder() + if err != nil { + log.Fatalf("Failed to initialize image processing: %v", err) + } + + // process images + for _, img := range *toProcess { + sem <- struct{}{} // acquire semaphore + wg.Add(1) + go func(img typesdefs.ImagesToProcess) { + defer wg.Done() + defer func() { <-sem }() // release semaphore + imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url))) + downloadImage(img.Url, imgPath) + webpPath, thumbnailwebpPath, err := Conversion(imgPath) + if err != nil { + log.Fatalf("Failed to convert image: %v", err) + } + mu.Lock() + ProcessedImages = append(ProcessedImages, typesdefs.ProcessedImage{ + Codice: img.Codice, + Order: img.Order, + Path: webpPath, + ThumnailPath: thumbnailwebpPath, + OGUrl: img.Url, + }) + mu.Unlock() + }(img) + } + wg.Wait() + uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) + var uploadWg sync.WaitGroup + for _, img := range ProcessedImages { + uploadSem <- struct{}{} // acquire semaphore + uploadWg.Add(1) + go func(img typesdefs.ProcessedImage) { + defer uploadWg.Done() + defer func() { <-uploadSem }() // release semaphore + + imgId, err := UploadFile(img.Path) + if err != nil { + log.Printf("failed to upload image: %v", err) + return + } + thumbId, err := UploadFile(img.ThumnailPath) + if err != nil { + log.Printf("failed to upload thumbnail: %v", err) + return + } + mu.Lock() + UploadedImages = append(UploadedImages, typesdefs.UploadedImage{ + Codice: img.Codice, + Order: img.Order, + ImgId: imgId, + ThumbId: thumbId, + OGUrl: img.OGUrl, + }) + mu.Unlock() + }(img) + } + uploadWg.Wait() + + return UploadedImages, nil +} + +func clearImageFolder() error { + err := utils.SafeRemoveAll(config.Cfg.Images.Path) + if err != nil { + return err + } + err = os.Mkdir(config.Cfg.Images.Path, 0755) + if err != nil { + return err + } + return nil +} +func downloadImage(url, filename string) error { + response, err := http.Get(url) + if err != nil { + return fmt.Errorf("failed to download image: %v", err) + } + defer response.Body.Close() + file, err := os.Create(filename) + if err != nil { + + return fmt.Errorf("failed to create image file: %v", err) + } + defer file.Close() + _, err = io.Copy(file, response.Body) + if err != nil { + return fmt.Errorf("failed to save image file: %v", err) + } + return nil +} + +func Conversion(input string) (string, string, error) { + var empty string var width uint = 1920 output := filepath.Base(input) outputdir := filepath.Dir(input) @@ -24,7 +165,7 @@ func Conversion(input string) error { log.Println("Converting", input, "to", output+".webp") f, err := os.Open(input) if err != nil { - return err + return empty, empty, err } defer f.Close() var img image.Image @@ -34,37 +175,37 @@ func Conversion(input string) error { log.Println("Image already in WebP format") img, err = webpbin.Decode(f) if err != nil { - return err + return empty, empty, err } case ".png", ".PNG": // Convert from PNG img, err = png.Decode(f) if err != nil { - return err + return empty, empty, err } case ".jpeg", ".jpg", ".JPG", ".JPEG": // Convert from JPEG img, err = jpeg.Decode(f) if err != nil { - return err + return empty, empty, err } default: - return fmt.Errorf("error: Unsupported file type: %s", ext) + return empty, empty, fmt.Errorf("error: Unsupported file type: %s", ext) } err = image_creation(img, width, output, outputdir) if err != nil { - return err + return empty, empty, err } err = thumbnail_creation(img, output, outputdir) if err != nil { - return err + return empty, empty, err } fmt.Println("Conversion completed successfully") - return nil + return fmt.Sprintf("%s/%s.webp", outputdir, output), fmt.Sprintf("%s/thumbnail-%s.webp", outputdir, output), nil } func image_creation(img image.Image, width uint, output string, outputdir string) error { @@ -117,3 +258,111 @@ func FallbackImage(c echo.Context) error { } return c.File(fallbackPath) } + +type imgRef struct { + Og_url string + Img string + Thumb string +} + +func CodiciToProcessImages(annunci *typesdefs.AnnunciXML) ([]string, error) { + currentBucketState, err := ListBucket("images") + if err != nil { + return nil, fmt.Errorf("failed to list bucket: %w", err) + } + //make a list of bucket ids + bucketIds := []string{} + for _, item := range currentBucketState { + bucketIds = append(bucketIds, item.ID) + } + + db_images, err := queries.GetImagesFromDB() + if err != nil { + return nil, err + } + + dbImagesMap := make(map[string][]imgRef) // Convert slice to map for faster lookup + for _, ref := range db_images { + dbImagesMap[ref.Codice] = append(dbImagesMap[ref.Codice], imgRef{ + Og_url: ref.Og_url, + Img: ref.Img, + Thumb: ref.Thumb, + }) + } + + var codiciToProcess []string + for _, annuncio := range annunci.Annuncio { + + if annuncio.Codice != nil && annuncio.Foto != nil { + refFoto, existsInDb := dbImagesMap[*annuncio.Codice] + if !existsInDb { + codiciToProcess = append(codiciToProcess, *annuncio.Codice) + continue + } + needProcess := false + for _, fotoUrl := range *annuncio.Foto { + found := false + for _, ref := range refFoto { + if fotoUrl == ref.Og_url { + if slices.Contains(bucketIds, ref.Img) && slices.Contains(bucketIds, ref.Thumb) { + found = true + break + } else { + needProcess = true + break + } + } + } + if !found { + needProcess = true + break + } + if needProcess { + break + } + } + if needProcess { + codiciToProcess = append(codiciToProcess, *annuncio.Codice) + } + } + } + + return codiciToProcess, nil + +} + +func ForceClearImages(codici []string) error { + var idsToDelete []string + for _, codice := range codici { + refs, err := queries.GetCodiceImagesFromDB(codice) + if err != nil { + return fmt.Errorf("failed to get images by codice %s: %w", codice, err) + } + for _, ref := range refs { + idsToDelete = append(idsToDelete, ref.Img) + idsToDelete = append(idsToDelete, ref.Thumb) + } + + } + err := DeleteIdsFromStorage(idsToDelete) + if err != nil { + return fmt.Errorf("failed to delete images from storage: %w", err) + } + for _, codice := range codici { + err := queries.ClearImagesRefsByCodice(codice) + if err != nil { + return fmt.Errorf("failed to clear images by codice %s: %w", codice, err) + } + } + return nil +} + +func DeleteIdsFromStorage(ids []string) error { + for _, id := range ids { + err := DeleteFile(id) + if err != nil { + return fmt.Errorf("failed to delete file with id %s: %w", id, err) + } + } + return nil +} diff --git a/apps/backend/miogest_handler.go b/apps/backend/miogest_handler.go index 4b3f133..4dd62c2 100644 --- a/apps/backend/miogest_handler.go +++ b/apps/backend/miogest_handler.go @@ -6,16 +6,11 @@ import ( "backend/typesdefs" "backend/utils" "fmt" - "io" "log" - "net/http" - "os" - "path/filepath" "reflect" "regexp" "slices" "strings" - "sync" "time" ) @@ -95,15 +90,15 @@ func (m *MiogestHandler) GenerateCategorie() { } // ANNUNCI PARSED -func (m *MiogestHandler) GetAnnunciParsed(i *ImageRefManager) typesdefs.AnnunciParsed { +func (m *MiogestHandler) GetAnnunciParsed(p *[]typesdefs.ImagesToProcess) typesdefs.AnnunciParsed { // salvo il parsing nello struct nel caso voglio implementare un caching e non fare il parsing ad ogni chiamata - m.AnnunciParsed = m.ParseAnnunci("", i) + m.AnnunciParsed = m.ParseAnnunci("", p) return m.AnnunciParsed } // versione per singolo annuncio -func (m *MiogestHandler) GetAnnuncioParsed(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed { - data := m.ParseAnnunci(codFilter, i) +func (m *MiogestHandler) GetAnnuncioParsed(codFilter string, p *[]typesdefs.ImagesToProcess) typesdefs.AnnunciParsed { + data := m.ParseAnnunci(codFilter, p) if len(data.Annuncio) == 0 { return typesdefs.AnnunciParsed{} } @@ -111,7 +106,7 @@ func (m *MiogestHandler) GetAnnuncioParsed(codFilter string, i *ImageRefManager) } -func (m *MiogestHandler) ParseAnnunci(codFilter string, i *ImageRefManager) typesdefs.AnnunciParsed { +func (m *MiogestHandler) ParseAnnunci(codFilter string, p *[]typesdefs.ImagesToProcess) typesdefs.AnnunciParsed { annunci := m.GetAnnunci() if len(annunci.Annuncio) == 0 { @@ -131,64 +126,34 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, i *ImageRefManager) type image_codes_to_process := []string{} if config.Cfg.Images.Enabled { var err error - image_codes_to_process, err = i.GetImageCodesForProcessing(&annunci) + image_codes_to_process, err = CodiciToProcessImages(&annunci) if err != nil { log.Fatalf("Failed to find images to update: %v", err.Error()) } - if codFilter == "" { - // if processing all, also check for missing folders - missing_codes, err := i.MissingImageFolders() - if err != nil { - log.Fatalf("Failed to find missing image folders: %v", err.Error()) - } - - if len(missing_codes) > 0 { - - image_codes_to_process = append(image_codes_to_process, missing_codes...) - image_codes_to_process = slices.Compact(image_codes_to_process) - - } - } else { + if codFilter != "" { // if processing single codice, ensure it's in the list if !slices.Contains(image_codes_to_process, codFilter) { image_codes_to_process = append(image_codes_to_process, codFilter) } - } - // Clears images folder of codes that need to be updated - if err := i.ClearImages(image_codes_to_process); err != nil { - log.Fatalf("Failed to clear images: %v", err.Error()) + err := ForceClearImages([]string{codFilter}) + if err != nil { + log.Fatalf("Failed to clear images for codice %s: %v", codFilter, err.Error()) + } } - i.InsertImagesMiogestRef(&annunci) } var AnnunciArray typesdefs.AnnunciParsed AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio)) m.InitDefaults() - var wg sync.WaitGroup - wg.Add(len(annunci.Annuncio)) - sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) //Limit concurrent goroutines - lista_codici := []string{} - for i := range annunci.Annuncio { - lista_codici = append(lista_codici, utils.RemoveWhitespace(*annunci.Annuncio[i].Codice)) - sem <- struct{}{} - go func(i int) { - defer wg.Done() - defer func() { <-sem }() - result := extractData_update(&annunci.Annuncio[i], slices.Contains(image_codes_to_process, *annunci.Annuncio[i].Codice), m) - AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result) - }(i) - } - wg.Wait() - if config.Cfg.Images.Enabled { - for _, codice := range lista_codici { - err := removeNonWebpImages(codice) - if err != nil { - log.Fatalf("Failed to cleanup folder: %v", err.Error()) - } - } + for i := range annunci.Annuncio { + imgToUpdate := slices.Contains(image_codes_to_process, *annunci.Annuncio[i].Codice) + result := extractData_update(&annunci.Annuncio[i], imgToUpdate, m, p) + AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result) + } + return AnnunciArray } @@ -202,7 +167,7 @@ func strUpd(s string) *string { return &s } -func extractData_update(annuncio *typesdefs.AnnuncioXML, updateImages bool, m *MiogestHandler) typesdefs.AnnuncioParsed { +func extractData_update(annuncio *typesdefs.AnnuncioXML, updateImages bool, m *MiogestHandler, p *[]typesdefs.ImagesToProcess) typesdefs.AnnuncioParsed { var tmp typesdefs.AnnuncioParsed = typesdefs.AnnuncioParsed{} tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice)) processLocatore(&tmp, annuncio) @@ -275,39 +240,9 @@ func extractData_update(annuncio *typesdefs.AnnuncioXML, updateImages bool, m *M } tmpcaratt := parseCaratteristiche(annuncio, m) tmp.Caratteristiche = &tmpcaratt - tmp.Og_url = nil - if len(*annuncio.Foto) != 0 { - fotos := *annuncio.Foto - first := fotos[0] - tmp.Og_url = &first - } + if updateImages { - processImages(&tmp, annuncio) - } - folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, tmp.Codice)) - for _, folder := range folders { - folderpath := filepath.Join(filepath.Join(config.Cfg.Images.Path, tmp.Codice), folder) - exists, err := utils.PathExists(folderpath) - if err != nil { - log.Fatalf("Failed to check if folder exists: %v", err) - } - if !exists { - // something went wrong, skip - continue - } - files, err := os.ReadDir(folderpath) - if err != nil { - log.Fatalf("Failed to read dir: %v", err) - } - for _, file := range files { - if strings.Contains(file.Name(), "webp") && !strings.HasPrefix(file.Name(), "thumbnail") { - tmpImgUrl := tmp.Codice + "/" + folder - if tmp.Url_foto == nil { - tmp.Url_foto = &[]string{} - } - *tmp.Url_foto = append(*tmp.Url_foto, tmpImgUrl) - } - } + AddToImageProcessing(&tmp, annuncio, p) } return tmp @@ -396,124 +331,6 @@ func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Annu tmp.Numero_postiauto = annuncio.PostiAuto } -func processImages(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { - path := filepath.Join(config.Cfg.Images.Path, tmp.Codice) - folderExists, err := utils.PathExists(path) - if err != nil { - log.Fatalf("Failed to check if folder exists: %v", err) - } - if folderExists { - // Remove existing folder to avoid stale images - err := utils.SafeRemoveAll(path) - if err != nil { - log.Fatalf("Failed to remove existing folder: %v", err) - } - } - // Create directory if it doesn't exist - err = os.MkdirAll(path, 0755) - if err != nil { - fmt.Printf("Failed to create directory: %v\n", err) - panic(err) - } - var wg sync.WaitGroup - sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) // Limit concurrent goroutines - nfoto := 0 - var fotos []string - if annuncio.Foto != nil { - fotos = *annuncio.Foto - } - for _, url := range fotos { - fmt.Println("Processing image", url) - imgpath := filepath.Join(path, fmt.Sprintf("%d", nfoto)) - if _, err := os.Stat(imgpath); os.IsNotExist(err) { - os.Mkdir(imgpath, 0755) - } - filename := filepath.Join(imgpath, fmt.Sprintf("%s_%d%s", tmp.Codice, nfoto, filepath.Ext(url))) - wg.Add(1) - sem <- struct{}{} - go func(url, filename string) { - defer wg.Done() - defer func() { <-sem }() - downloadImage(url, filename) - }(url, filename) - nfoto++ - } - wg.Wait() - processWebpImages(path) -} - -func downloadImage(url, filename string) { - response, err := http.Get(url) - if err != nil { - log.Fatalf("Failed to download image: %v", err) - return - } - defer response.Body.Close() - file, err := os.Create(filename) - if err != nil { - log.Fatalf("Failed to create file: %v", err) - return - } - defer file.Close() - _, err = io.Copy(file, response.Body) - if err != nil { - log.Fatalf("Failed to save image: %v", err) - } -} - -func processWebpImages(path string) { - array, err := utils.GetPathsArray(path) - if err != nil { - log.Fatal(err) - } - var wg sync.WaitGroup - sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) // Limit concurrent goroutines - for _, img := range array { - wg.Add(1) - sem <- struct{}{} - go func(img string) { - defer wg.Done() - defer func() { <-sem }() - err := Conversion(img) - if err != nil { - log.Fatal(err) - } - }(img) - } - wg.Wait() -} - -func removeNonWebpImages(codice string) error { - folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, codice)) - non_webp := []string{} - for _, folder := range folders { - folderpath := filepath.Join(config.Cfg.Images.Path, codice, folder) - exists, err := utils.PathExists(folderpath) - if err != nil { - return err - } - if !exists { - continue - } - files, err := os.ReadDir(folderpath) - if err != nil { - return err - } - for _, file := range files { - if !strings.Contains(file.Name(), "webp") { - non_webp = append(non_webp, filepath.Join(folderpath, file.Name())) - } - } - } - for _, file := range non_webp { - err := utils.SafeRemove(file) - if err != nil { - return err - } - } - return nil -} - func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, m *MiogestHandler) map[string]interface{} { if reflect.TypeOf(wrkrecord).Kind() != reflect.Ptr || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct { return nil diff --git a/apps/backend/queries/annunci.go b/apps/backend/queries/annunci.go index 84ff619..878bb22 100644 --- a/apps/backend/queries/annunci.go +++ b/apps/backend/queries/annunci.go @@ -84,14 +84,12 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e stato = $42, web = $43, homepage = $44, - url_immagini = $45, - url_video = $46, - og_url = $48, - disponibile_da = $49, - permanenza = $50, - persone = $51, + url_video = $45, + disponibile_da = $47, + permanenza = $48, + persone = $49, updated_at = NOW() - WHERE codice = $47 + WHERE codice = $46 `, annuncio.Locatore, annuncio.Numero, annuncio.Idlocatore, annuncio.Email, annuncio.Creato_il, annuncio.Modificato_il, annuncio.Indirizzo, annuncio.Civico, annuncio.Comune, annuncio.Cap, @@ -102,8 +100,8 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e annuncio.Unita_condominio, annuncio.Numero_vani, annuncio.Numero_camere, annuncio.Numero_bagni, annuncio.Numero_balconi, annuncio.Numero_terrazzi, annuncio.Numero_box, annuncio.Numero_postiauto, annuncio.Caratteristiche, annuncio.Accessori, annuncio.Titolo_it, annuncio.Desc_it, annuncio.Titolo_en, - annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, annuncio.Url_foto, - annuncio.Url_video, annuncio.Codice, annuncio.Og_url, annuncio.Disponibile_da, annuncio.Permanenza, annuncio.Persone) + annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, + annuncio.Url_video, annuncio.Codice, annuncio.Disponibile_da, annuncio.Permanenza, annuncio.Persone) } else { batch.Queue(` @@ -152,16 +150,14 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e stato, web, homepage, - url_immagini, url_video, codice, - og_url, disponibile_da, permanenza, persone, updated_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, NOW()) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, NOW()) `, annuncio.Locatore, annuncio.Numero, annuncio.Idlocatore, annuncio.Email, annuncio.Creato_il, annuncio.Modificato_il, annuncio.Indirizzo, annuncio.Civico, annuncio.Comune, annuncio.Cap, @@ -172,8 +168,8 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e annuncio.Unita_condominio, annuncio.Numero_vani, annuncio.Numero_camere, annuncio.Numero_bagni, annuncio.Numero_balconi, annuncio.Numero_terrazzi, annuncio.Numero_box, annuncio.Numero_postiauto, annuncio.Caratteristiche, annuncio.Accessori, annuncio.Titolo_it, annuncio.Desc_it, annuncio.Titolo_en, - annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, annuncio.Url_foto, - annuncio.Url_video, annuncio.Codice, annuncio.Og_url, annuncio.Disponibile_da, annuncio.Permanenza, annuncio.Persone) + annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, + annuncio.Url_video, annuncio.Codice, annuncio.Disponibile_da, annuncio.Permanenza, annuncio.Persone) } } err = Db.Dbpool.SendBatch(Db.Ctx, batch).Close() @@ -245,11 +241,10 @@ func SetFromBkp(annunci typesdefs.AnnunciParsed) error { stato, web, homepage, - url_immagini, url_video, codice ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46) `, annuncio.Locatore, annuncio.Numero, annuncio.Idlocatore, annuncio.Email, annuncio.Creato_il, annuncio.Modificato_il, annuncio.Indirizzo, annuncio.Civico, annuncio.Comune, annuncio.Cap, @@ -260,7 +255,7 @@ func SetFromBkp(annunci typesdefs.AnnunciParsed) error { annuncio.Unita_condominio, annuncio.Numero_vani, annuncio.Numero_camere, annuncio.Numero_bagni, annuncio.Numero_balconi, annuncio.Numero_terrazzi, annuncio.Numero_box, annuncio.Numero_postiauto, annuncio.Caratteristiche, annuncio.Accessori, annuncio.Titolo_it, annuncio.Desc_it, annuncio.Titolo_en, - annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, annuncio.Url_foto, + annuncio.Desc_en, annuncio.Stato, annuncio.Web, annuncio.Homepage, annuncio.Url_video, annuncio.Codice) } diff --git a/apps/backend/queries/images.go b/apps/backend/queries/images.go index e92e7a1..08446f8 100644 --- a/apps/backend/queries/images.go +++ b/apps/backend/queries/images.go @@ -8,24 +8,15 @@ import ( "github.com/jackc/pgx/v5" ) -func ImagesGetAll() ([]typesdefs.MiogestImagesRef_Table, error) { - var images []typesdefs.MiogestImagesRef_Table - err := pgxscan.Select(Db.Ctx, Db.Dbpool, &images, "SELECT * FROM public.miogest_images_ref") - if err != nil && err != pgx.ErrNoRows { - return nil, fmt.Errorf("failed to get all images: %w", err) - } - return images, nil -} - -func ImagesBatchInsert(annunci *typesdefs.AnnunciXML) error { +func SetImagesToDB(pool *[]typesdefs.UploadedImage) error { batch := &pgx.Batch{} - for _, annuncio := range annunci.Annuncio { + for _, data := range *pool { batch.Queue(` - INSERT INTO public.miogest_images_ref (codice, foto) - VALUES ($1, $2) - ON CONFLICT (codice) DO UPDATE SET foto = $2 - `, &annuncio.Codice, &annuncio.Foto) + INSERT INTO public.images_refs (codice, ordine, img, thumb, og_url) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (codice, ordine) DO UPDATE SET img = $3, thumb = $4 + `, data.Codice, data.Order, data.ImgId, data.ThumbId, data.OGUrl) } if err := Db.Dbpool.SendBatch(Db.Ctx, batch).Close(); err != nil { @@ -35,18 +26,28 @@ func ImagesBatchInsert(annunci *typesdefs.AnnunciXML) error { return nil } -func ImagesClear() error { - _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.miogest_images_ref") - if err != nil { - return fmt.Errorf("failed to clear images: %w", err) +func GetImagesFromDB() ([]typesdefs.ImagesRefs, error) { + var images []typesdefs.ImagesRefs + err := pgxscan.Select(Db.Ctx, Db.Dbpool, &images, "SELECT * FROM public.images_refs ORDER BY ordine ASC") + if err != nil && err != pgx.ErrNoRows { + return nil, fmt.Errorf("failed to get images: %w", err) } - return nil + return images, nil } -func ImagesClearByCodice(codice string) error { - _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.miogest_images_ref WHERE codice = $1", codice) +func GetCodiceImagesFromDB(codice string) ([]typesdefs.ImagesRefs, error) { + var images []typesdefs.ImagesRefs + err := pgxscan.Select(Db.Ctx, Db.Dbpool, &images, "SELECT * FROM public.images_refs WHERE codice = $1 ORDER BY ordine ASC", codice) + if err != nil && err != pgx.ErrNoRows { + return nil, fmt.Errorf("failed to get images: %w", err) + } + return images, nil +} + +func ClearImagesRefsByCodice(codice string) error { + _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.images_refs WHERE codice = $1", codice) if err != nil { - return fmt.Errorf("failed to clear images by codice: %w", err) + return fmt.Errorf("failed to clear images refs by codice: %w", err) } return nil } diff --git a/apps/backend/queries/storage.go b/apps/backend/queries/storage.go deleted file mode 100644 index ed743e5..0000000 --- a/apps/backend/queries/storage.go +++ /dev/null @@ -1,95 +0,0 @@ -package queries - -import ( - "backend/typesdefs" - "fmt" - "time" - - "github.com/georgysavva/scany/v2/pgxscan" -) - -// STORAGE - -func StorageNewId() (string, error) { - var newId string - err := Db.Dbpool.QueryRow(Db.Ctx, "INSERT INTO public.storageindex(filename)VALUES (null) RETURNING id;").Scan(&newId) - if err != nil { - return "", err - } - if newId == "" { - return "", fmt.Errorf("failed to create new id") - } - return newId, nil - -} - -func StorageGet_min() ([]struct { - Id string - Ext string -}, error) { - var items_db []struct { - Id string - Ext string - } - - err := pgxscan.Select(Db.Ctx, Db.Dbpool, &items_db, "SELECT id, ext FROM public.storageindex") - if err != nil { - return nil, fmt.Errorf("failed to get items: %w", err) - } - return items_db, nil -} - -func StorageUpdate(id, filename, ext string, from_admin bool, expires_at *time.Time) error { - _, err := Db.Dbpool.Exec(Db.Ctx, "UPDATE public.storageindex set filename= $2, ext = $3, from_admin = $4, expires_at = $5 where id = $1", id, filename, ext, from_admin, expires_at) - if err != nil { - return fmt.Errorf("failed to update storage index: %w", err) - } - return nil -} - -func StorageDelete(id string) error { - _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.storageindex WHERE id = $1", id) - if err != nil { - return fmt.Errorf("failed to delete storage index: %w", err) - } - return nil - -} - -func StorageDeleteUnused() error { - _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.storageindex WHERE filename IS NULL") - if err != nil { - return fmt.Errorf("failed to delete storage index: %w", err) - } - return nil -} - -func StorageCheckToken(token string) error { - var tk []string - err := pgxscan.Select(Db.Ctx, Db.Dbpool, &tk, "SELECT token FROM public.temp_tokens WHERE token = $1 and elapse > $2", token, time.Now()) - if err != nil { - return fmt.Errorf("failed to check token: %w", err) - } - if tk == nil { - return fmt.Errorf("invalid or expired token") - } - return nil -} - -func StorageCleanExpiredTokens(token string) error { - _, err := Db.Dbpool.Exec(Db.Ctx, "DELETE FROM public.temp_tokens WHERE elapse < $1", time.Now()) - if err != nil { - return err - } - return nil -} - -func StorageGetExpiredFiles() ([]typesdefs.Storageindex, error) { - - var items_db []typesdefs.Storageindex - err := pgxscan.Select(Db.Ctx, Db.Dbpool, &items_db, "SELECT * FROM public.storageindex WHERE expires_at is not null and expires_at < NOW()") - if err != nil { - return nil, fmt.Errorf("failed to get expired items: %w", err) - } - return items_db, nil -} diff --git a/apps/backend/scripts/clean_expired.sh b/apps/backend/scripts/clean_expired.sh deleted file mode 100644 index 30e3893..0000000 --- a/apps/backend/scripts/clean_expired.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -e # Exit on any error - -echo "Starting expired storage cleanup..." -if curl -f --connect-timeout 30 --max-time 300 "http://localhost:1323/storage/remove-expired"; then - echo "Expired storage cleanup completed successfully" -else - echo "Failed to clean expired storage" >&2 - exit 1 -fi \ No newline at end of file diff --git a/apps/backend/server.go b/apps/backend/server.go index da91a80..fea8fda 100644 --- a/apps/backend/server.go +++ b/apps/backend/server.go @@ -3,31 +3,25 @@ package main import ( "backend/config" "backend/queries" - storagehandlers "backend/storage_handlers" - "backend/utils" - "bytes" + "backend/typesdefs" + "fmt" "io" "log" "math/rand" "net/http" "os" - "path/filepath" "strconv" "strings" "text/template" "time" "github.com/labstack/echo/v4" - "github.com/nfnt/resize" - "github.com/nickalie/go-webpbin" ) type Server struct { - miogest *MiogestHandler - imageManager *ImageRefManager - storage storagehandlers.StorageHandler - config *config.Config + miogest *MiogestHandler + config *config.Config } func NewServer(cfg *config.Config) (*Server, error) { @@ -36,21 +30,9 @@ func NewServer(cfg *config.Config) (*Server, error) { return nil, fmt.Errorf("error connecting to miogest") } - imageManager := NewImageRefManager() - if imageManager == nil { - return nil, fmt.Errorf("error creating image manager") - } - - storage, err := storagehandlers.NewStorageHandler(cfg.Storage.Strategy) - if err != nil { - return nil, fmt.Errorf("error creating storage handler: %w", err) - } - return &Server{ - miogest: miogest, - imageManager: imageManager, - storage: storage, - config: cfg, + miogest: miogest, + config: cfg, }, nil } func (s *Server) SetupRoutes() *echo.Echo { @@ -80,9 +62,15 @@ func (s *Server) SetupRoutes() *echo.Echo { }) }) e.GET("/test", func(c echo.Context) error { + files, err := ListBucket("images") + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } - s.imageManager.MissingImageFolders() - return c.String(http.StatusOK, "OK") + return c.JSONPretty(http.StatusOK, + map[string]any{ + "files": files, + }, " ") }) e.POST("/testpost", func(c echo.Context) error { // Get the form value @@ -117,14 +105,24 @@ func (s *Server) SetupRoutes() *echo.Echo { }) e.GET("/parse", func(c echo.Context) error { start := time.Now() - var data = s.miogest.GetAnnunciParsed(s.imageManager) + var data = s.miogest.GetAnnunciParsed(nil) end := time.Now() fmt.Println("time taken:", end.Sub(start).String()) return c.JSONPretty(http.StatusOK, data, " ") }) e.GET("/update", func(c echo.Context) error { - var data = s.miogest.GetAnnunciParsed(s.imageManager) - err := queries.AnnunciInsert(data, true) + imgPool := []typesdefs.ImagesToProcess{} + var data = s.miogest.GetAnnunciParsed(&imgPool) + imgRefs, err := ProcessImagePool(&imgPool) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + + err = queries.AnnunciInsert(data, true) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + err = queries.SetImagesToDB(&imgRefs) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } @@ -133,12 +131,21 @@ func (s *Server) SetupRoutes() *echo.Echo { e.GET("/update-cod/:cod", func(c echo.Context) error { cod := c.Param("cod") - - var data = s.miogest.GetAnnuncioParsed(cod, s.imageManager) - err := queries.AnnunciInsert(data, false) + imgPool := []typesdefs.ImagesToProcess{} + var data = s.miogest.GetAnnuncioParsed(cod, &imgPool) + imgRefs, err := ProcessImagePool(&imgPool) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } + err = queries.AnnunciInsert(data, false) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + err = queries.SetImagesToDB(&imgRefs) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + return c.String(http.StatusOK, "Updated") }) @@ -183,174 +190,12 @@ func (s *Server) SetupRoutes() *echo.Echo { return c.String(http.StatusOK, "Setted") }) - //STORAGE ROUTES - e.GET("/storage/list", func(c echo.Context) error { - filenames, err := s.storage.List() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - return c.JSON(http.StatusOK, filenames) - }) - - e.GET("/storage/get/:filename", func(c echo.Context) error { - err := TempFolderCleaner(s.config.Storage.TempLimit) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to clean temp folder: "+err.Error()) - } - err = StorageAuth(c) - if err != nil { - return c.JSON(http.StatusUnauthorized, "Unauthorized: "+err.Error()) - } - - filename := c.Param("filename") - path, err := s.storage.Get(filename) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to get file: "+err.Error()) - } - - return c.File(path) - }) - e.POST("/storage/upload", func(c echo.Context) error { - fmt.Println("Uploading file") - - err := StorageAuth(c) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) - } - - from_admin := c.QueryParam("from_admin") == "true" - file, err := c.FormFile("file") - if err != nil { - return err - } - - fileId, err := Upload(file, from_admin, s.storage) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to upload file: "+err.Error()) - } - - return c.String(http.StatusOK, fileId) - }) - e.DELETE("/storage/delete/:filename", func(c echo.Context) error { - err := StorageAuth(c) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) - } - filename := c.Param("filename") - - err = s.storage.Delete(filename) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to remove file / file not found: "+err.Error()) - } - return c.String(http.StatusOK, "File removed") - - }) - e.GET("/storage/remove-expired", func(c echo.Context) error { - - err := RemoveExpired(s.storage) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to remove files / files not found: "+err.Error()) - } - return c.String(http.StatusOK, "File removed") - - }) - e.GET("/storage/test", func(c echo.Context) error { - err := CheckConcruency(s.storage) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to check concruency: "+err.Error()) - } - return c.String(http.StatusOK, "OK") - }) - //IMAGE ROUTES e.GET("/initcwebp", func(c echo.Context) error { Conversion("test.jpg") return c.String(http.StatusOK, "Cwebp initialized") }) - e.GET("/images/list", func(c echo.Context) error { - folders := utils.GetPathDirsArray(config.Cfg.Images.Path) - return c.JSON(http.StatusOK, folders) - - // annunci, err := queries.AnnunciGetActive() - // if err != nil { - // return c.String(http.StatusInternalServerError, err.Error()) - // } - // return c.JSON(http.StatusOK, annunci) - - }) - - e.GET("/images/get/:cod/:imgNum", func(c echo.Context) error { - cod := c.Param("cod") - imgNum := c.Param("imgNum") - thumb := c.QueryParam("thumbnail") == "true" - widthStr := c.QueryParam("w") - //quality := c.QueryParam("q") - fmt.Printf("request for %s/%s, thumbnail=%t, w=%s\n", cod, imgNum, thumb, widthStr) - prefix := "" - if thumb { - prefix = "thumbnail-" - } - - Path := filepath.Join(config.Cfg.Images.Path, cod, imgNum, fmt.Sprintf("%s%s_%s.webp", prefix, cod, imgNum)) - fmt.Println(Path) - if _, err := os.Stat(Path); os.IsNotExist(err) { - - fmt.Println("Image does not exist:", Path) - // Serve a fallback image if the requested image is not found - - return FallbackImage(c) - } - if widthStr != "" { - width, err := strconv.Atoi(widthStr) - if err != nil { - fmt.Println("Invalid width parameter:", widthStr) - - return FallbackImage(c) - } - f, err := os.Open(Path) - if err != nil { - return err - } - img, err := webpbin.Decode(f) - if err != nil { - fmt.Println("Failed to decode image:", err) - return FallbackImage(c) - } - resizedImg := resize.Resize(uint(width), 0, img, resize.Lanczos3) - buffer := new(bytes.Buffer) - err = webpbin.Encode(buffer, resizedImg) - if err != nil { - fmt.Println("Failed to encode image:", err) - return FallbackImage(c) - } - return c.Blob(http.StatusOK, "image/webp", buffer.Bytes()) - } - - return c.File(Path) - - }) - - e.GET("/images/resetimages", func(c echo.Context) error { - - err := s.imageManager.Reset() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - return c.String(http.StatusOK, "OK") - }) - e.GET("/images/remove/:cod", func(c echo.Context) error { - cod := c.Param("cod") - - err := s.imageManager.RemoveImage(cod) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - return c.String(http.StatusOK, "OK") - }) - e.POST("/image-option-toggle", func(c echo.Context) error { // Toggle the imageOption value s.config.Images.Enabled = !s.config.Images.Enabled diff --git a/apps/backend/storage.go b/apps/backend/storage.go index 117993e..cca89da 100644 --- a/apps/backend/storage.go +++ b/apps/backend/storage.go @@ -1,161 +1,205 @@ package main import ( - "backend/config" - "backend/queries" - storagehandlers "backend/storage_handlers" + "bytes" + "encoding/json" "fmt" + "io" "mime/multipart" "net/http" + "net/textproto" "os" "path/filepath" - "slices" "time" - - "github.com/labstack/echo/v4" ) -// Upload saves a file to the storage directory. -func Upload(file *multipart.FileHeader, from_admin bool, s storagehandlers.StorageHandler) (string, error) { - fmt.Printf("Uploading file: %s\n", file.Filename) +var EXAMPLE_STORAGE_URL = "http://localhost:8080" +var EXAMPLE_STORAGE_TOKEN = "test" - newId, err := queries.StorageNewId() +func DeleteFile(id string) error { + req, err := http.NewRequest("DELETE", EXAMPLE_STORAGE_URL+"/delete/"+id+"?token="+EXAMPLE_STORAGE_TOKEN, nil) if err != nil { - return "", err + return fmt.Errorf("failed to create delete request: %w", err) } - - err = s.Add(file, newId) + resp, err := http.DefaultClient.Do(req) if err != nil { - return "", err + return fmt.Errorf("delete request failed: %w", err) } - //if !from_admin , expires in 60 days - var expiresAt *time.Time - if !from_admin { - exp := time.Now().AddDate(0, 2, 0) // 60 days - expiresAt = &exp - } else { - expiresAt = nil - } - - err = queries.StorageUpdate(newId, file.Filename, filepath.Ext(file.Filename), from_admin, expiresAt) - if err != nil { - return "", err - } - - err = queries.StorageDeleteUnused() - if err != nil { - return "", err - } - return newId, nil - -} - -func StorageAuth(c echo.Context) error { - token := c.Request().URL.Query().Get("token") - if len(token) < 1 { - return c.String(http.StatusUnauthorized, "Unauthorized") - } - err := queries.StorageCheckToken(token) - if err != nil { - return err - } - - err = queries.StorageCleanExpiredTokens(token) - if err != nil { - return err + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("delete failed: %s", resp.Status) } return nil } -func RemoveFromDb(ref string) error { - - return queries.StorageDelete(ref) +func GetFile(path string) ([]byte, error) { + panic("unimplemented") } -func RemoveExpired(s storagehandlers.StorageHandler) error { - //get all expired files - items_db, err := queries.StorageGetExpiredFiles() - if err != nil { - return err - } - - for _, item := range items_db { - formattedName := item.Id + item.Ext - err = s.Delete(formattedName) - if err != nil { - return fmt.Errorf("failed to delete file %s: %w", formattedName, err) - } - err = queries.StorageDelete(item.Id) - if err != nil { - return err - } - - } - return nil +type FileMetadata struct { + ID string `json:"id"` + OriginalName string `json:"originalName"` + Size int64 `json:"size"` + MIMEType string `json:"mimeType"` + BlockCount int `json:"blockCount"` + UploadedAt time.Time `json:"uploadedAt"` + ExpiresAt *time.Time `json:"expiresAt"` + Bucket string `json:"bucket,omitempty"` } -func CheckConcruency(s storagehandlers.StorageHandler) error { - items_db, err := queries.StorageGet_min() - if err != nil { - return err - } +// func ListFiles() ([]FileMetadata, error) { +// //get list of files from api +// resp, err := http.Get(EXAMPLE_STORAGE_URL + "/files?token=" + EXAMPLE_STORAGE_TOKEN) +// if err != nil { +// return nil, err +// } +// defer resp.Body.Close() - files, err := s.List() - if err != nil { - return err - } - if len(items_db) != len(files) { - var founds []string - for _, item := range items_db { - formattedName := item.Id + item.Ext - found := slices.Contains(files, formattedName) - if found { - founds = append(founds, formattedName) - } else { - err := queries.StorageDelete(item.Id) - if err != nil { - return err - } - } - } - for _, file := range files { - found := slices.Contains(founds, file) - if !found { - fmt.Printf("File %v not found in db\n", file) - //TODO decidere se eliminarlo +// if resp.StatusCode != http.StatusOK { +// return nil, fmt.Errorf("failed to list files: %s", resp.Status) +// } +// //parse response +// body, err := io.ReadAll(resp.Body) +// if err != nil { +// fmt.Println("Error reading response body:", err) +// return nil, err +// } +// var data []FileMetadata +// err = json.Unmarshal(body, &data) +// if err != nil { +// fmt.Println("Error unmarshalling response body:", err) +// return nil, err +// } - } - } +// return data, nil + +// } + +func ListBucket(bucketName string) ([]FileMetadata, error) { + //get list of files from api + resp, err := http.Get(EXAMPLE_STORAGE_URL + "/bucket/" + bucketName + "?token=" + EXAMPLE_STORAGE_TOKEN) + if err != nil { + return nil, err } - return nil + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to list files: %s", resp.Status) + } + //parse response + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return nil, err + } + var data []FileMetadata + err = json.Unmarshal(body, &data) + if err != nil { + fmt.Println("Error unmarshalling response body:", err) + return nil, err + } + return data, nil } -func TempFolderCleaner(limit int) error { - files, err := os.ReadDir(filepath.Join(config.Cfg.Storage.Path, "/tmp/")) +// type CheckFileResponse struct { +// Available []FileMetadata `json:"available"` +// Unavailable []string `json:"unavailable"` +// } + +// func CheckFiles(ids []string) ([]FileMetadata, []string, error) { + +// //application/json with array of ids as body of post request to /files-check +// jsonData, err := json.Marshal(ids) +// if err != nil { +// return nil, nil, err +// } +// resp, err := http.Post(EXAMPLE_STORAGE_URL+"/files-check?token="+EXAMPLE_STORAGE_TOKEN, +// "application/json", bytes.NewBuffer(jsonData)) +// if err != nil { +// return nil, nil, err +// } +// defer resp.Body.Close() + +// if resp.StatusCode != http.StatusOK { +// return nil, nil, fmt.Errorf("failed to check files: %s", resp.Status) +// } +// //parse response +// body, err := io.ReadAll(resp.Body) +// if err != nil { +// fmt.Println("Error reading response body:", err) +// return nil, nil, err +// } +// var data CheckFileResponse +// err = json.Unmarshal(body, &data) +// if err != nil { +// fmt.Println("Error unmarshalling response body:", err) +// return nil, nil, err +// } +// return data.Available, data.Unavailable, nil + +// } + +func UploadFile(path string) (string, error) { + + file, err := os.Open(path) if err != nil { - return err + return "", fmt.Errorf("failed to open file: %w", err) } - fmt.Println("Temp files: ", len(files)) - if len(files) >= limit { + defer file.Close() - var oldest_Time time.Time - var oldest_File string - for _, file := range files { - file_infos, err := file.Info() - if err != nil { - return err - } - if oldest_Time.IsZero() || file_infos.ModTime().Before(oldest_Time) { - oldest_Time = file_infos.ModTime() - oldest_File = file.Name() - } - } - err = os.Remove(filepath.Join(config.Cfg.Storage.Path, "/tmp/", oldest_File)) - if err != nil { - return err - } + var body bytes.Buffer + writer := multipart.NewWriter(&body) + mimeType := "image/webp" + + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filepath.Base(path))) + h.Set("Content-Type", mimeType) + part, err := writer.CreatePart(h) + if err != nil { + return "", fmt.Errorf("failed to create form file: %w", err) } - return nil + if _, err := io.Copy(part, file); err != nil { + return "", fmt.Errorf("failed to copy file data: %w", err) + } + + err = writer.WriteField("bucket", "images") + if err != nil { + return "", fmt.Errorf("failed to write bucket field: %w", err) + } + + if err := writer.Close(); err != nil { + return "", fmt.Errorf("failed to close writer: %w", err) + } + + req, err := http.NewRequest("POST", EXAMPLE_STORAGE_URL+"/upload?token="+EXAMPLE_STORAGE_TOKEN, &body) + if err != nil { + return "", fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("upload request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("upload failed: %s", resp.Status) + } + //parse response + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read upload response: %w", err) + } + var data map[string]string + err = json.Unmarshal(respBody, &data) + if err != nil { + return "", fmt.Errorf("failed to parse upload response: %w", err) + } + if id, ok := data["id"]; ok { + return id, nil + } + + return "", nil } diff --git a/apps/backend/storage_handlers/fs.go b/apps/backend/storage_handlers/fs.go deleted file mode 100644 index d194532..0000000 --- a/apps/backend/storage_handlers/fs.go +++ /dev/null @@ -1,77 +0,0 @@ -package storagehandlers - -import ( - "backend/config" - "io" - "log" - "mime/multipart" - "os" - "path/filepath" -) - -var _ StorageHandler = (*Fs_Handler)(nil) - -type Fs_Handler struct { -} - -func NewFsHandler() *Fs_Handler { - return &Fs_Handler{} -} -func (f *Fs_Handler) Init() error { - if _, err := os.Stat(config.Cfg.Storage.Path); os.IsNotExist(err) { - return err - } - return nil -} - -func (f *Fs_Handler) Add(file *multipart.FileHeader, fileId string) error { - src, err := file.Open() - if err != nil { - log.Printf("Error opening file: %v\n", err) - return err - } - defer src.Close() - - extension := filepath.Ext(file.Filename) - destPath := filepath.Join(config.Cfg.Storage.Path, fileId+extension) - dst, err := os.Create(destPath) - if err != nil { - log.Println("Error creating file") - return err - } - defer dst.Close() - if _, err = io.Copy(dst, src); err != nil { - log.Println("Error copying file") - return err - } - return nil -} - -func (f *Fs_Handler) Delete(filename string) error { - filePath := filepath.Join(config.Cfg.Storage.Path, filename) - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return err - } - return os.Remove(filePath) -} - -func (f *Fs_Handler) Get(filename string) (string, error) { - path := filepath.Join(config.Cfg.Storage.Path, filename) - - if _, err := os.Stat(path); os.IsNotExist(err) { - return "", err - } - return path, nil -} - -func (f *Fs_Handler) List() ([]string, error) { - files, err := os.ReadDir(config.Cfg.Storage.Path) - if err != nil { - return nil, err - } - var filenames []string - for _, file := range files { - filenames = append(filenames, file.Name()) - } - return filenames, nil -} diff --git a/apps/backend/storage_handlers/go_fs.go b/apps/backend/storage_handlers/go_fs.go deleted file mode 100644 index b6d0599..0000000 --- a/apps/backend/storage_handlers/go_fs.go +++ /dev/null @@ -1,93 +0,0 @@ -package storagehandlers - -import ( - "backend/config" - "bytes" - "fmt" - "io" - "mime/multipart" - "net/http" -) - -var _ StorageHandler = (*GOFS_Handler)(nil) - -type GOFS_Handler struct { - Endpoint string -} - -func NewGoFsHandler() *GOFS_Handler { - - return &GOFS_Handler{ - Endpoint: config.Cfg.Storage.GOFS.Endpoint, - } -} -func (g *GOFS_Handler) Init() error { - resp, err := http.Get(g.Endpoint + "/health") - if err != nil { - return fmt.Errorf("storage server unreachable: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("storage server unhealthy: %s", resp.Status) - } - return nil -} -func (g *GOFS_Handler) Add(file *multipart.FileHeader, fileId string) error { - f, err := file.Open() - if err != nil { - return err - } - defer f.Close() - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - part, err := writer.CreateFormFile("file", file.Filename) - if err != nil { - return err - } - if _, err := io.Copy(part, f); err != nil { - return err - } - _ = writer.WriteField("fileId", fileId) - writer.Close() - - req, err := http.NewRequest("POST", g.Endpoint+"/upload", body) - if err != nil { - return err - } - req.Header.Set("Content-Type", writer.FormDataContentType()) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("upload failed: %s", resp.Status) - } - return nil -} - -func (g *GOFS_Handler) Delete(fileId string) error { - req, err := http.NewRequest("DELETE", g.Endpoint+"/delete/"+fileId, nil) - if err != nil { - return err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("delete failed: %s", resp.Status) - } - return nil -} - -func (g *GOFS_Handler) Get(filename string) (string, error) { - panic("unimplemented") -} - -func (g *GOFS_Handler) List() ([]string, error) { - panic("unimplemented") -} diff --git a/apps/backend/storage_handlers/minio.go b/apps/backend/storage_handlers/minio.go deleted file mode 100644 index a99ce4b..0000000 --- a/apps/backend/storage_handlers/minio.go +++ /dev/null @@ -1,135 +0,0 @@ -package storagehandlers - -import ( - "backend/config" - "context" - "fmt" - "log" - "mime/multipart" - "path/filepath" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" -) - -var ( - useSSL = false - bucketName = "storage" -) -var _ StorageHandler = (*S3_Handler)(nil) - -type S3_Handler struct { - minioClient *minio.Client -} - -func (s *S3_Handler) Init() error { - found, err := s.minioClient.BucketExists(context.Background(), bucketName) - if err != nil { - - log.Println(err) - return err - } - if !found { - err = s.minioClient.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}) - if err != nil { - log.Println(err) - return err - } - log.Println("Bucket created successfully") - } - log.Println("Bucket found") - return nil - -} - -func NewS3Handler() (*S3_Handler, error) { - - minioClient, err := minio.New(config.Cfg.Storage.Minio.URL, &minio.Options{ - Creds: credentials.NewStaticV4(config.Cfg.Storage.Minio.RootUser, config.Cfg.Storage.Minio.Password, ""), - Secure: useSSL, - }) - if err != nil { - log.Fatalln(err) - } - return &S3_Handler{ - minioClient, - }, nil -} - -func (s *S3_Handler) Add(file *multipart.FileHeader, fileId string) error { - ctx := context.Background() - src, err := file.Open() - if err != nil { - log.Printf("Error opening file: %v\n", err) - return err - } - defer src.Close() - - extension := filepath.Ext(file.Filename) - objectName := fileId + extension - - info, err := s.minioClient.PutObject(ctx, bucketName, objectName, src, file.Size, minio.PutObjectOptions{ContentType: "application/octet-stream"}) - if err != nil { - log.Fatalln(err) - return err - } - log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size) - return nil -} - -func (s *S3_Handler) Delete(filename string) error { - opts := minio.RemoveObjectOptions{ - GovernanceBypass: true, - } - err := s.minioClient.RemoveObject(context.Background(), bucketName, filename, opts) - if err != nil { - log.Fatalln(err) - return err - } - return nil -} - -// func (s *S3_Handler) DeleteM(filename string) error { -// opts := minio.RemoveObjectOptions{ -// GovernanceBypass: true, -// } -// err := s.minioClient.RemoveObjects(context.Background(), bucketName, filename, opts) -// if err != nil { -// log.Fatalln(err) -// return err -// } -// return nil -// } - -func (s *S3_Handler) Get(filename string) (string, error) { - localPath := filepath.Join(config.Cfg.Storage.Path, "/tmp/", filename) - - err := s.minioClient.FGetObject(context.Background(), bucketName, filename, localPath, minio.GetObjectOptions{}) - if err != nil { - fmt.Println(err) - return "", fmt.Errorf("error downloading file %s: %w", filename, err) - } - log.Printf("Successfully downloaded %s\n", filename) - return localPath, nil - -} - -func (s *S3_Handler) List() ([]string, error) { - ctx, cancel := context.WithCancel(context.Background()) - - defer cancel() - - objectCh := s.minioClient.ListObjects(ctx, bucketName, minio.ListObjectsOptions{ - Prefix: "", - Recursive: true, - }) - var list []string - for object := range objectCh { - if object.Err != nil { - log.Println(object.Err) - return nil, object.Err - } - list = append(list, object.Key) - } - return list, nil -} diff --git a/apps/backend/storage_handlers/storage_handler.go b/apps/backend/storage_handlers/storage_handler.go deleted file mode 100644 index 792c636..0000000 --- a/apps/backend/storage_handlers/storage_handler.go +++ /dev/null @@ -1,47 +0,0 @@ -package storagehandlers - -import ( - "backend/config" - "backend/typesdefs" - - "errors" - "mime/multipart" -) - -type StorageHandler interface { - Init() error - Add(file *multipart.FileHeader, fileId string) error - Delete(filename string) error - Get(filename string) (string, error) - List() ([]string, error) -} - -func NewStorageHandler(strategy typesdefs.StorageStrategy) (StorageHandler, error) { - if config.Cfg.Storage.Path == "" { - return nil, errors.New("storage path not set") - } - switch strategy { - case typesdefs.FS: - h := NewFsHandler() - err := h.Init() - if err != nil { - return nil, err - } - return h, nil - - case typesdefs.S3: - h, err := NewS3Handler() - if err != nil { - return nil, err - } - err = h.Init() - if err != nil { - return nil, err - } - return h, nil - case typesdefs.GOFS: - return nil, nil - default: - return nil, errors.New("invalid storage strategy") - } -} diff --git a/apps/backend/typesdefs/models.go b/apps/backend/typesdefs/models.go index db59c28..cc043a6 100644 --- a/apps/backend/typesdefs/models.go +++ b/apps/backend/typesdefs/models.go @@ -257,10 +257,8 @@ type AnnuncioParsed struct { Titolo_en *string Unita_condominio *string Numero_vani *string - Url_foto *[]string Url_video *[]string Web *bool - Og_url *string Disponibile_da *string Permanenza *[]int Persone *[]string @@ -309,23 +307,16 @@ type AnnunciDB struct { Stato *string Web *bool Caratteristiche *map[string]interface{} - Url_immagini *[]string Homepage *bool Url_video *[]string Email *string Creato_il *time.Time Modificato_il *time.Time - Og_url *string Disponibile_da *time.Time Permanenza *[]int Persone *[]string } -type FilebrowserData struct { - Data []File - Folder *string -} - type File struct { Name string Size int64 @@ -359,16 +350,32 @@ type Categoria struct { Nome string } -type Storageindex struct { - Id string - Created_at time.Time - Filename string - Ext string - From_admin bool - Expires_at *time.Time +type ImagesRefs struct { + Codice string + Ordine int + Img string + Thumb string + Og_url string } -type MiogestImagesRef_Table struct { +type ImagesToProcess struct { Codice string - Foto *[]string + Url string + Order int +} + +type ProcessedImage struct { + Codice string + Order int + Path string + ThumnailPath string + OGUrl string +} + +type UploadedImage struct { + Codice string + Order int + ImgId string + ThumbId string + OGUrl string } diff --git a/apps/backend/typesdefs/types.go b/apps/backend/typesdefs/types.go deleted file mode 100644 index 8fa1533..0000000 --- a/apps/backend/typesdefs/types.go +++ /dev/null @@ -1,9 +0,0 @@ -package typesdefs - -type StorageStrategy string - -const ( - FS StorageStrategy = "fs" - S3 StorageStrategy = "s3" - GOFS StorageStrategy = "gofs" -) diff --git a/apps/backend/utils/utils.go b/apps/backend/utils/utils.go index c601936..501e863 100644 --- a/apps/backend/utils/utils.go +++ b/apps/backend/utils/utils.go @@ -7,7 +7,6 @@ import ( "log" "net/http" "os" - "path/filepath" "reflect" "strings" ) @@ -21,21 +20,6 @@ func MakeUppercase(s string) string { return "" } -func CheckStringIfPresent(value *string) string { - if value != nil { - return strings.TrimSpace(*value) - } - return "" -} - -func CheckStringIfPresentBool(value string) bool { - if value != "" { - return true - } else { - return false - } -} - func fetchXML(url string) ([]byte, error) { resp, err := http.Get(url) if err != nil { @@ -70,36 +54,6 @@ func GetDataXML[T any](url string) T { return result } -func GetPathsArray(path string) ([]string, error) { - var paths []string - - err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - paths = append(paths, path) - } - return nil - }) - - if err != nil { - return nil, err - } - - return paths, nil -} - -func SafeRemove(path string) error { - err := os.Remove(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - return nil -} func SafeRemoveAll(path string) error { err := os.RemoveAll(path) if err != nil { @@ -111,32 +65,6 @@ func SafeRemoveAll(path string) error { return nil } -func PathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err -} - -func GetPathDirsArray(path string) []string { - var folderNames []string - files, err := os.ReadDir(path) - if err != nil { - log.Fatal(err) - } - - for _, file := range files { - if file.IsDir() { - folderNames = append(folderNames, file.Name()) - } - } - return folderNames -} - func GetStringValue(strPtr *string) string { if strPtr != nil { return *strPtr diff --git a/apps/infoalloggi/emails/onboarding-servizio.tsx b/apps/infoalloggi/emails/onboarding-servizio.tsx index 69c643c..83ff622 100644 --- a/apps/infoalloggi/emails/onboarding-servizio.tsx +++ b/apps/infoalloggi/emails/onboarding-servizio.tsx @@ -65,7 +65,7 @@ const OnboardingServizio = ({ alt={`Annuncio image - ${annuncio.codice}`} className="block w-[240px] rounded-[4px] object-cover object-center" height="140px" - src={`${env.BASE_URL}/go-api/images/get/${annuncio.immagine}`} + src={`${env.BASE_URL}/storage-api/get/${annuncio.immagine}?image=true`} width="100%" /> diff --git a/apps/infoalloggi/next.config.ts b/apps/infoalloggi/next.config.ts index 70d4ada..fadf33c 100644 --- a/apps/infoalloggi/next.config.ts +++ b/apps/infoalloggi/next.config.ts @@ -41,6 +41,7 @@ async function createNextConfig(): Promise { localeDetection: false, }, images: { + unoptimized: true, minimumCacheTTL: 7200, remotePatterns: [ { @@ -50,7 +51,7 @@ async function createNextConfig(): Promise { protocol: "https", }, { - hostname: `${env.BACKENDSERVER_URL}`, + hostname: `${env.STORAGE_URL}`, }, ], }, @@ -63,10 +64,6 @@ async function createNextConfig(): Promise { destination: env.NODE_ENV === "production" ? "/404" : "/api/panel", source: "/api/panel", }, - { - destination: `${env.BACKENDSERVER_URL}/images/get/:slug*`, - source: "/go-api/images/get/:slug*", - }, ]; }, diff --git a/apps/infoalloggi/src/components/annunci_grid.tsx b/apps/infoalloggi/src/components/annunci_grid.tsx index 91fdda8..e8f9ecb 100644 --- a/apps/infoalloggi/src/components/annunci_grid.tsx +++ b/apps/infoalloggi/src/components/annunci_grid.tsx @@ -15,7 +15,7 @@ export const AnnunciGrid = ({ pagedata }: { pagedata: AnnuncioRicerca[] }) => { comune={annuncio.comune} consegna={annuncio.consegna} id={annuncio.id} - immagini={annuncio.url_immagini || undefined} + images={annuncio.images} key={annuncio.codice} mq={annuncio.mq} prezzo={annuncio.prezzo} diff --git a/apps/infoalloggi/src/components/annunci_map.tsx b/apps/infoalloggi/src/components/annunci_map.tsx index 74f10b8..4f843c4 100644 --- a/apps/infoalloggi/src/components/annunci_map.tsx +++ b/apps/infoalloggi/src/components/annunci_map.tsx @@ -118,12 +118,12 @@ const SelectedComp = memo( }} >
- {selected.url_immagini?.[0] ? ( + {selected.images?.[0] ? ( a ) : ( diff --git a/apps/infoalloggi/src/components/annuncio_card.tsx b/apps/infoalloggi/src/components/annuncio_card.tsx index 8f1d323..056e930 100644 --- a/apps/infoalloggi/src/components/annuncio_card.tsx +++ b/apps/infoalloggi/src/components/annuncio_card.tsx @@ -33,12 +33,13 @@ type CardAnnuncioProps = { provincia: string | null; consegna: number | null; camere: number | null; - immagini: string[] | undefined; + //immagini: string[] | undefined; tipo: string | null; stato: string | null; className?: string; videos?: string[] | undefined; updated_at: Date | null; + images: { img: string; thumb: string }[]; }; export const CardAnnuncio = ({ @@ -51,12 +52,13 @@ export const CardAnnuncio = ({ provincia, consegna, camere, - immagini, + //immagini, tipo, stato, className, videos, updated_at, + images, }: CardAnnuncioProps) => { const { t } = useTranslation(); @@ -73,7 +75,7 @@ export const CardAnnuncio = ({ //target="_blank" href={`/annuncio/${codice}`} //duration-700 ease-in-out animate-in fade-in > -
+
{stato === "Trattativa" && (
@@ -86,16 +88,15 @@ export const CardAnnuncio = ({ - {immagini?.map((img, idx) => ( - + {images?.map((img, idx) => ( + @@ -117,7 +118,11 @@ export const CardAnnuncio = ({ @@ -135,11 +140,11 @@ export const CardAnnuncio = ({ >
@@ -274,7 +279,7 @@ export const CarouselAnnuncio = ({ height={400} onClick={() => handleOpenModal(idx)} priority={idx === 0} - src={`${img}?${updated_at?.toString() || new Date().toString()}`} + src={`/storage-api/get/${img}?image=true&${updated_at?.toString() || new Date().toString()}`} width={800} /> @@ -344,7 +353,7 @@ export const CarouselAnnuncio = ({ alt={`carousel-img-${idx}`} className="mx-auto h-[90vh] w-full rounded-md object-contain sm:w-[80vw]" height={1080} - src={`${img}?${updated_at?.toString() || new Date().toString()}`} + src={`/storage-api/get/${img}?image=true&${updated_at?.toString() || new Date().toString()}`} width={1920} /> @@ -362,7 +371,11 @@ export const CarouselAnnuncio = ({ diff --git a/apps/infoalloggi/src/components/servizio/annuncio_card.tsx b/apps/infoalloggi/src/components/servizio/annuncio_card.tsx index a4bbdea..dac0042 100644 --- a/apps/infoalloggi/src/components/servizio/annuncio_card.tsx +++ b/apps/infoalloggi/src/components/servizio/annuncio_card.tsx @@ -37,7 +37,6 @@ type AnnuncioData = Pick< | "prezzo" | "desc_en" | "desc_it" - | "url_immagini" | "titolo_en" | "titolo_it" | "tipo" @@ -45,7 +44,9 @@ type AnnuncioData = Pick< | "stato" | "web" | "updated_at" ->; +> & { + images: Pick<{ img: string; thumb: string }, "img">[]; +}; export const AnnuncioCard = ({ className }: { className?: string }) => { const { data } = useServizioAnnuncio(); @@ -88,8 +89,8 @@ export const BasicAnnuncioCard = ({ height={1080} priority src={ - (data.url_immagini && - `/go-api/images/get/${data.url_immagini[0]}?${data.updated_at?.toString() || new Date().toString()}`) || + (data?.images[0] && + `/storage-api/get/${data.images[0].img}?image=true&${data.updated_at?.toString() || new Date().toISOString()}`) || "/fallback-image.png" } width={1920} @@ -179,14 +180,14 @@ const AnnuncioDettaglio = ({ data }: { data: AnnuncioData }) => {
- {data?.url_immagini?.map((img, idx) => ( - + {data?.images?.map((img, idx) => ( + @@ -201,15 +202,11 @@ const AnnuncioDettaglio = ({ data }: { data: AnnuncioData }) => { >
diff --git a/apps/infoalloggi/src/forms/FormEditAnnuncio.tsx b/apps/infoalloggi/src/forms/FormEditAnnuncio.tsx index 3502d53..a1dcbb2 100644 --- a/apps/infoalloggi/src/forms/FormEditAnnuncio.tsx +++ b/apps/infoalloggi/src/forms/FormEditAnnuncio.tsx @@ -45,12 +45,12 @@ import { cn } from "~/lib/utils"; import { useZodForm } from "~/lib/zodForm"; import { StatusBadge } from "~/pages/area-riservata/admin/edit-annuncio/[id]"; import { useTranslation } from "~/providers/I18nProvider"; -import type { Annunci } from "~/schemas/public/Annunci"; +import type { AnnunciWithImages } from "~/server/controllers/annunci.controller"; import { zAnnuncioId } from "~/server/utils/zod_types"; import { api } from "~/utils/api"; import type { Caratteristiche } from "~/utils/kanel-types"; -export const AnnuncioEditForm = ({ data }: { data: Annunci }) => { +export const AnnuncioEditForm = ({ data }: { data: AnnunciWithImages }) => { const schema = z.object({ accessori: z.string().array().nullable(), @@ -534,11 +534,7 @@ export const AnnuncioEditForm = ({ data }: { data: Annunci }) => { `/go-api/images/get/${v}`, - ) || null - } + immagini={data.images?.map((v) => v.img) || "fallback"} single updated_at={data.updated_at} videos={data.url_video} diff --git a/apps/infoalloggi/src/hooks/storageClienSideHooks.ts b/apps/infoalloggi/src/hooks/storageClienSideHooks.ts index 03825e5..a814b82 100644 --- a/apps/infoalloggi/src/hooks/storageClienSideHooks.ts +++ b/apps/infoalloggi/src/hooks/storageClienSideHooks.ts @@ -21,6 +21,7 @@ export async function uploadFile( if (expiresAt) { formData.append("expires_at", expiresAt); } + formData.append("bucket", "storage"); const response = await fetch(`/storage-api/upload`, { method: "POST", diff --git a/apps/infoalloggi/src/middleware.ts b/apps/infoalloggi/src/middleware.ts index fb52200..0f54dc6 100644 --- a/apps/infoalloggi/src/middleware.ts +++ b/apps/infoalloggi/src/middleware.ts @@ -24,7 +24,7 @@ export const config = { { key: "purpose", type: "header", value: "prefetch" }, ], source: - "/((?!api/trpc|go-api/images|api/tiles|api/auth|_next/static|_next/image|screenshots|site.webmanifest|favicon.ico|favicon|sitemap.xml|robots.txt).*)", + "/((?!api/trpc|storage-api|api/tiles|api/auth|_next/static|_next/image|screenshots|site.webmanifest|favicon.ico|favicon|sitemap.xml|robots.txt).*)", }, ], }; diff --git a/apps/infoalloggi/src/middlewares/api_middleware.ts b/apps/infoalloggi/src/middlewares/api_middleware.ts index 28706fd..251d15b 100644 --- a/apps/infoalloggi/src/middlewares/api_middleware.ts +++ b/apps/infoalloggi/src/middlewares/api_middleware.ts @@ -6,25 +6,48 @@ import { verifyToken } from "~/server/auth/jwt"; export const apisMiddleware = async (req: NextRequest) => { const { pathname, searchParams } = req.nextUrl; console.log("APIs Middleware triggered for:", pathname); - const accessToken = req.cookies.get( - TOKEN_CONFIG.ACCESS_TOKEN_COOKIE_NAME, - )?.value; - if (!accessToken) { - return new NextResponse("Unauthorized", { status: 401 }); + + // Only handle storage API routes + if (!pathname.startsWith("/storage-api/")) { + return null; // Pass to next middleware } - const payload = await verifyToken(accessToken); - if (!payload) { - return new NextResponse("Unauthorized", { status: 401 }); + + const params = new URLSearchParams(searchParams); + const isImageRequest = params.get("image") === "true"; + const isVideoRequest = params.get("video") === "true"; + + // Check if this is a Next.js Image Optimization request + const purpose = req.headers.get("purpose"); + const isNextImageRequest = + purpose === "prefetch" || req.headers.get("x-vercel-id"); + + // For regular requests (not from Next Image Optimization), check auth + if (!isNextImageRequest && !isImageRequest && !isVideoRequest) { + const accessToken = req.cookies.get( + TOKEN_CONFIG.ACCESS_TOKEN_COOKIE_NAME, + )?.value; + + if (!accessToken) { + console.log("No access token found"); + return new NextResponse("Unauthorized", { status: 401 }); + } + + const payload = await verifyToken(accessToken); + if (!payload) { + console.log("Invalid token"); + return new NextResponse("Unauthorized", { status: 401 }); + } } + // Build the storage API URL correctly const slug = pathname.replace("/storage-api/", ""); - // Preserve existing query params and add token - const params = new URLSearchParams(searchParams); params.set("token", env.STORAGE_TOKEN); const storageUrl = `${env.STORAGE_URL}/${slug}?${params.toString()}`; + console.log("Proxying to:", storageUrl); + try { // Forward the request to storage API const headers = new Headers(req.headers); @@ -40,14 +63,53 @@ export const apisMiddleware = async (req: NextRequest) => { duplex: "half", }); + // Handle failed responses if (!response.ok) { - console.error("Storage API error:", response.status); + console.error(`Storage API error: ${response.status} for ${storageUrl}`); + + // Return fallback for image requests + if (isImageRequest || isNextImageRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-image.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + + // Return fallback for video requests + if (isVideoRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-video.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + return new NextResponse("Storage API error", { status: response.status }); } // Get response data const data = await response.arrayBuffer(); + // Check if we got valid data + if (!data || data.byteLength === 0) { + console.error("Received empty response from storage API"); + + if (isImageRequest || isNextImageRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-image.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + + if (isVideoRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-video.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + + return new NextResponse("Empty response", { status: 500 }); + } + // Create response with proper headers const proxyResponse = new NextResponse(data, { status: response.status, @@ -59,16 +121,24 @@ export const apisMiddleware = async (req: NextRequest) => { const contentDisposition = response.headers.get("Content-Disposition"); const contentLength = response.headers.get("Content-Length"); - if (contentType) proxyResponse.headers.set("Content-Type", contentType); - if (contentDisposition) - proxyResponse.headers.set("Content-Disposition", contentDisposition); - if (contentLength) - proxyResponse.headers.set("Content-Length", contentLength); + if (contentType) { + proxyResponse.headers.set("Content-Type", contentType); + } - // Set CORS headers - proxyResponse.headers.set("Access-Control-Allow-Origin", env.BASE_URL); - proxyResponse.headers.set("Access-Control-Allow-Credentials", "true"); - proxyResponse.headers.delete("X-Frame-Options"); // Remove if set by storage API + if (contentDisposition) { + proxyResponse.headers.set("Content-Disposition", contentDisposition); + } + if (contentLength) { + proxyResponse.headers.set("Content-Length", contentLength); + } + + // Set CORS and caching headers + proxyResponse.headers.set("Access-Control-Allow-Origin", "*"); + proxyResponse.headers.set( + "Cache-Control", + "public, max-age=31536000, immutable", + ); + proxyResponse.headers.delete("X-Frame-Options"); proxyResponse.headers.set( "Content-Security-Policy", "frame-ancestors 'self'", @@ -77,6 +147,22 @@ export const apisMiddleware = async (req: NextRequest) => { return proxyResponse; } catch (error) { console.error("Storage proxy error:", error); + + // Return fallback on error + if (isImageRequest || isNextImageRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-image.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + + if (isVideoRequest) { + const url = req.nextUrl.clone(); + url.pathname = "/fallback-video.png"; + url.search = ""; + return NextResponse.rewrite(url); + } + return new NextResponse("Failed to proxy request", { status: 500 }); } }; diff --git a/apps/infoalloggi/src/pages/annuncio/[cod].tsx b/apps/infoalloggi/src/pages/annuncio/[cod].tsx index ae098cc..95ca306 100644 --- a/apps/infoalloggi/src/pages/annuncio/[cod].tsx +++ b/apps/infoalloggi/src/pages/annuncio/[cod].tsx @@ -134,7 +134,7 @@ const AnnuncioView = ({ cod, flag }: Omit) => {
img.img)} updated_at={data.updated_at} videos={data.url_video} /> @@ -202,7 +202,11 @@ const AnnuncioView = ({ cod, flag }: Omit) => { { - return
Test page
; + return ( +
+ Test page + asdad +
+ ); }; export default Test; diff --git a/apps/infoalloggi/src/schemas/public/Annunci.ts b/apps/infoalloggi/src/schemas/public/Annunci.ts index 9bc607a..2a8f137 100644 --- a/apps/infoalloggi/src/schemas/public/Annunci.ts +++ b/apps/infoalloggi/src/schemas/public/Annunci.ts @@ -91,8 +91,6 @@ export default interface AnnunciTable { caratteristiche: ColumnType; - url_immagini: ColumnType; - homepage: ColumnType; url_video: ColumnType; @@ -105,8 +103,6 @@ export default interface AnnunciTable { consegna: ColumnType; - og_url: ColumnType; - disponibile_da: ColumnType; permanenza: ColumnType; diff --git a/apps/infoalloggi/src/schemas/public/ImagesRefs.ts b/apps/infoalloggi/src/schemas/public/ImagesRefs.ts new file mode 100644 index 0000000..e192dfb --- /dev/null +++ b/apps/infoalloggi/src/schemas/public/ImagesRefs.ts @@ -0,0 +1,23 @@ +// @generated +// This file is automatically generated by Kanel. Do not modify manually. + +import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; + +/** Represents the table public.images_refs */ +export default interface ImagesRefsTable { + codice: ColumnType; + + ordine: ColumnType; + + img: ColumnType; + + thumb: ColumnType; + + og_url: ColumnType; +} + +export type ImagesRefs = Selectable; + +export type NewImagesRefs = Insertable; + +export type ImagesRefsUpdate = Updateable; diff --git a/apps/infoalloggi/src/schemas/public/MiogestImagesRef.ts b/apps/infoalloggi/src/schemas/public/MiogestImagesRef.ts deleted file mode 100644 index 65f2c68..0000000 --- a/apps/infoalloggi/src/schemas/public/MiogestImagesRef.ts +++ /dev/null @@ -1,20 +0,0 @@ -// @generated -// This file is automatically generated by Kanel. Do not modify manually. - -import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely'; - -/** Identifier type for public.miogest_images_ref */ -export type MiogestImagesRefCodice = string & { __brand: 'public.miogest_images_ref' }; - -/** Represents the table public.miogest_images_ref */ -export default interface MiogestImagesRefTable { - codice: ColumnType; - - foto: ColumnType; -} - -export type MiogestImagesRef = Selectable; - -export type NewMiogestImagesRef = Insertable; - -export type MiogestImagesRefUpdate = Updateable; diff --git a/apps/infoalloggi/src/schemas/public/PublicSchema.ts b/apps/infoalloggi/src/schemas/public/PublicSchema.ts index 712b41c..339371e 100644 --- a/apps/infoalloggi/src/schemas/public/PublicSchema.ts +++ b/apps/infoalloggi/src/schemas/public/PublicSchema.ts @@ -11,7 +11,6 @@ import type { default as UsersStorageTable } from './UsersStorage'; import type { default as ServizioAnnunciTable } from './ServizioAnnunci'; import type { default as EmailsTable } from './Emails'; import type { default as PaymentsTable } from './Payments'; -import type { default as MiogestImagesRefTable } from './MiogestImagesRef'; import type { default as TempTokensTable } from './TempTokens'; import type { default as EventQueueTable } from './EventQueue'; import type { default as OrdiniTable } from './Ordini'; @@ -22,6 +21,7 @@ import type { default as EtichetteTable } from './Etichette'; import type { default as MessagesTable } from './Messages'; import type { default as UsersAnagraficaTable } from './UsersAnagrafica'; import type { default as BanlistTable } from './Banlist'; +import type { default as ImagesRefsTable } from './ImagesRefs'; import type { default as BannersTable } from './Banners'; import type { default as ServizioTable } from './Servizio'; @@ -46,8 +46,6 @@ export default interface PublicSchema { payments: PaymentsTable; - miogest_images_ref: MiogestImagesRefTable; - temp_tokens: TempTokensTable; event_queue: EventQueueTable; @@ -68,6 +66,8 @@ export default interface PublicSchema { banlist: BanlistTable; + images_refs: ImagesRefsTable; + banners: BannersTable; servizio: ServizioTable; diff --git a/apps/infoalloggi/src/server/api/routers/interests.ts b/apps/infoalloggi/src/server/api/routers/interests.ts index 3273028..6b30fde 100644 --- a/apps/infoalloggi/src/server/api/routers/interests.ts +++ b/apps/infoalloggi/src/server/api/routers/interests.ts @@ -1,4 +1,5 @@ import { TRPCError } from "@trpc/server"; +import { jsonArrayFrom } from "kysely/helpers/postgres"; import z from "zod/v4"; import { env } from "~/env"; import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc"; @@ -89,13 +90,12 @@ export const intrestsRouter = createTRPCRouter({ } const annunci = await db .selectFrom("annunci") - .select([ + .select((eb) => [ "annunci.id", "annunci.codice", "annunci.prezzo", "annunci.desc_en", "annunci.desc_it", - "annunci.url_immagini", "annunci.titolo_en", "annunci.titolo_it", "annunci.tipo", @@ -103,6 +103,13 @@ export const intrestsRouter = createTRPCRouter({ "annunci.stato", "annunci.web", "annunci.updated_at", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .select(["images_refs.img", "images_refs.thumb"]) + .whereRef("images_refs.codice", "=", "annunci.codice") + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) .where("web", "=", true) .where("stato", "!=", "Sospeso") diff --git a/apps/infoalloggi/src/server/controllers/annunci.controller.ts b/apps/infoalloggi/src/server/controllers/annunci.controller.ts index 9e98328..59baaca 100644 --- a/apps/infoalloggi/src/server/controllers/annunci.controller.ts +++ b/apps/infoalloggi/src/server/controllers/annunci.controller.ts @@ -1,14 +1,14 @@ import { TRPCError } from "@trpc/server"; +import { jsonArrayFrom } from "kysely/helpers/postgres"; import { env } from "~/env"; import type { Annunci, AnnunciId, AnnunciUpdate, } from "~/schemas/public/Annunci"; +import type { ImagesRefs } from "~/schemas/public/ImagesRefs"; import type { ServizioServizioId } from "~/schemas/public/Servizio"; import { db } from "~/server/db"; -import { AnnuncioObjectWithImages } from "~/server/services/annunci.service"; -import { createSrcset } from "~/server/services/imageServer"; import { revalidate } from "../utils/revalidationHelper"; // const ratelimit = new RateLimiterHandler({ @@ -16,6 +16,9 @@ import { revalidate } from "../utils/revalidationHelper"; // maxRequests: 10, // analytics: true, // }); +export type AnnunciWithImages = Annunci & { + images: Pick[]; +}; export const getAnnunciListHandler = async (): Promise< Pick< @@ -69,11 +72,20 @@ export const getAnnunciByCod = async ({ cod, }: { cod: string; -}): Promise => { +}): Promise => { try { const annuncio = await db .selectFrom("annunci") .selectAll() + .select((eb) => + jsonArrayFrom( + eb + .selectFrom("images_refs") + .whereRef("images_refs.codice", "=", "annunci.codice") + .select(["img", "thumb"]) + .orderBy("images_refs.ordine", "asc"), + ).as("images"), + ) .where("annunci.web", "=", true) .where("codice", "=", cod) .executeTakeFirst(); @@ -82,7 +94,7 @@ export const getAnnunciByCod = async ({ return null; } - return AnnuncioObjectWithImages(annuncio); + return annuncio; } catch (e) { throw new TRPCError({ cause: (e as Error).cause, @@ -95,7 +107,17 @@ export const getAnnunciMetaByCod = async ({ cod }: { cod: string }) => { try { const annuncio = await db .selectFrom("annunci") - .select(["titolo_it", "desc_it", "og_url", "url_immagini"]) + .select((eb) => [ + "titolo_it", + "desc_it", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .whereRef("images_refs.codice", "=", "annunci.codice") + .select(["img", "thumb"]) + .orderBy("images_refs.ordine", "asc"), + ).as("images"), + ]) .where("annunci.web", "=", true) .where("codice", "=", cod) .executeTakeFirst(); @@ -103,11 +125,9 @@ export const getAnnunciMetaByCod = async ({ cod }: { cod: string }) => { return null; } const ogImage = - annuncio.url_immagini && - annuncio.url_immagini.length > 0 && - annuncio.url_immagini[0] - ? `${env.BASE_URL}/go-api/images/get/${annuncio.url_immagini[0]}` - : annuncio.og_url || ""; + annuncio.images && annuncio.images.length > 0 && annuncio.images[0] + ? `${env.BASE_URL}/storage-api/get/${annuncio.images[0]}?image=true` + : `${env.BASE_URL}/og.jpg`; return { description: annuncio.desc_it || "", @@ -127,11 +147,20 @@ export const getAnnunciById_rawImgUrls = async ({ id, }: { id: AnnunciId; -}): Promise => { +}): Promise => { try { const annuncio = await db .selectFrom("annunci") .selectAll() + .select((eb) => [ + jsonArrayFrom( + eb + .selectFrom("images_refs") + .select(["images_refs.img", "images_refs.thumb"]) + .whereRef("images_refs.codice", "=", "annunci.codice") + .orderBy("images_refs.ordine", "asc"), + ).as("images"), + ]) .where("id", "=", id) .executeTakeFirst(); if (!annuncio) { @@ -167,7 +196,7 @@ export const get_AnnunciPositionsHandler = async ({ try { let query = db .selectFrom("annunci") - .select([ + .select((eb) => [ "id", "codice", "comune", @@ -179,13 +208,19 @@ export const get_AnnunciPositionsHandler = async ({ "tipo", "titolo_it", "titolo_en", - "url_immagini", "modificato_il", "stato", "url_video", "lon_secondario", "lat_secondario", "updated_at", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .whereRef("images_refs.codice", "=", "annunci.codice") + .select(["img", "thumb"]) + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) .where("web", "=", true) @@ -206,11 +241,6 @@ export const get_AnnunciPositionsHandler = async ({ if (!annunci) { return []; } - for (const annuncio of annunci) { - annuncio.url_immagini = annuncio.url_immagini - ? createSrcset(annuncio.url_immagini) - : []; - } return annunci; } catch (e) { @@ -234,12 +264,13 @@ export type AnnuncioRicerca = Pick< | "tipo" | "titolo_it" | "titolo_en" - | "url_immagini" | "modificato_il" | "stato" | "url_video" | "updated_at" ->; +> & { + images: Pick[]; +}; export const getCursor_AnnunciHandler = async ({ page = 0, @@ -259,7 +290,7 @@ export const getCursor_AnnunciHandler = async ({ try { let query = db .selectFrom("annunci") - .select([ + .select((eb) => [ "id", "codice", "comune", @@ -271,11 +302,17 @@ export const getCursor_AnnunciHandler = async ({ "tipo", "titolo_it", "titolo_en", - "url_immagini", "modificato_il", "stato", "url_video", "updated_at", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .whereRef("images_refs.codice", "=", "annunci.codice") + .select(["img", "thumb"]) + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) .where("web", "=", true) @@ -319,12 +356,6 @@ export const getCursor_AnnunciHandler = async ({ annunci.pop(); // Remove the last item if it exceeds the page size } - for (const annuncio of annunci) { - annuncio.url_immagini = annuncio.url_immagini - ? createSrcset(annuncio.url_immagini) - : []; - } - return { annunci, hasMore, diff --git a/apps/infoalloggi/src/server/controllers/servizio.controller.ts b/apps/infoalloggi/src/server/controllers/servizio.controller.ts index 6d22638..2c86402 100644 --- a/apps/infoalloggi/src/server/controllers/servizio.controller.ts +++ b/apps/infoalloggi/src/server/controllers/servizio.controller.ts @@ -5,7 +5,7 @@ import { jsonArrayFrom, jsonObjectFrom } from "kysely/helpers/postgres"; import { z } from "zod/v4"; import type { PurchaseData } from "~/components/acquisto_receipt"; import { env } from "~/env"; -import type { Annunci, AnnunciId } from "~/schemas/public/Annunci"; +import type { AnnunciId } from "~/schemas/public/Annunci"; import OrderTypeEnum from "~/schemas/public/OrderTypeEnum"; import type { OrdiniOrdineId } from "~/schemas/public/Ordini"; import PaymentStatusEnum from "~/schemas/public/PaymentStatusEnum"; @@ -632,13 +632,12 @@ export const getAllServizioAnnunci = async (userId: UsersId) => { .selectFrom("servizio_annunci") .selectAll("servizio_annunci") .innerJoin("annunci", "annunci.id", "servizio_annunci.annunci_id") - .select([ + .select((eb) => [ "annunci.id", "annunci.codice", "annunci.prezzo", "annunci.desc_en", "annunci.desc_it", - "annunci.url_immagini", "annunci.titolo_en", "annunci.titolo_it", "annunci.tipo", @@ -646,6 +645,13 @@ export const getAllServizioAnnunci = async (userId: UsersId) => { "annunci.stato", "annunci.web", "annunci.updated_at", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .select(["images_refs.img", "images_refs.thumb"]) + .whereRef("images_refs.codice", "=", "annunci.codice") + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) //ignora annunci che non sono disponibili .where("annunci.web", "=", true) @@ -1146,11 +1152,17 @@ export const sendServizioEmail = async ({ const annunci = await db .selectFrom("servizio_annunci") .innerJoin("annunci", "annunci.id", "servizio_annunci.annunci_id") - .select([ + .select((eb) => [ "annunci.titolo_it", - "annunci.url_immagini", "annunci.codice", "annunci.prezzo", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .select(["images_refs.img", "images_refs.thumb"]) + .whereRef("images_refs.codice", "=", "annunci.codice") + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) .where("servizio_annunci.servizio_id", "=", servizioId) .execute(); @@ -1161,9 +1173,9 @@ export const sendServizioEmail = async ({ props: { annunci: annunci.map((annuncio) => ({ codice: annuncio.codice, - immagine: annuncio.url_immagini - ? annuncio.url_immagini[0] || null - : null, + immagine: annuncio?.images + ? annuncio.images[0]?.img || "fallback" + : "fallback", prezzo: annuncio.prezzo, titolo: annuncio.titolo_it, })), @@ -1464,23 +1476,6 @@ export const SendContactEmail = async ({ } }; -export type AnnuncioData = Pick< - Annunci, - | "id" - | "codice" - | "prezzo" - | "desc_en" - | "desc_it" - | "url_immagini" - | "titolo_en" - | "titolo_it" - | "tipo" - | "consegna" - | "stato" - | "web" - | "updated_at" ->; - export const getCompatibileAnnunci = async (servizioId: ServizioServizioId) => { try { const servizio = await getServizioById(servizioId); @@ -1493,13 +1488,12 @@ export const getCompatibileAnnunci = async (servizioId: ServizioServizioId) => { let qry = db .selectFrom("annunci") - .select([ + .select((eb) => [ "annunci.id", "annunci.codice", "annunci.prezzo", "annunci.desc_en", "annunci.desc_it", - "annunci.url_immagini", "annunci.titolo_en", "annunci.titolo_it", "annunci.tipo", @@ -1507,6 +1501,13 @@ export const getCompatibileAnnunci = async (servizioId: ServizioServizioId) => { "annunci.stato", "annunci.web", "annunci.updated_at", + jsonArrayFrom( + eb + .selectFrom("images_refs") + .select(["images_refs.img", "images_refs.thumb"]) + .whereRef("images_refs.codice", "=", "annunci.codice") + .orderBy("images_refs.ordine", "asc"), + ).as("images"), ]) .where("web", "=", true) .where("stato", "!=", "Sospeso") diff --git a/apps/infoalloggi/src/server/services/annunci.service.ts b/apps/infoalloggi/src/server/services/annunci.service.ts deleted file mode 100644 index afdee31..0000000 --- a/apps/infoalloggi/src/server/services/annunci.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { createSrcset } from "~/server/services/imageServer"; - -type AnnuncioBase = { - url_immagini: string[] | null; -}; - -export const AnnuncioObjectWithImages = ( - annuncio: T, -): T => { - return { - ...annuncio, - url_immagini: annuncio.url_immagini - ? createSrcset(annuncio.url_immagini) - : [], - }; -}; diff --git a/apps/infoalloggi/src/server/services/imageServer.ts b/apps/infoalloggi/src/server/services/imageServer.ts deleted file mode 100644 index 636f76e..0000000 --- a/apps/infoalloggi/src/server/services/imageServer.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { env } from "~/env"; - -export const createSrcset = (url_immagini: string[]) => { - const finalSrcset = []; - for (const url of url_immagini) { - //finalSrcset.push(`https://${env.BACKENDSERVER_URL}/images/get/${url}`); - finalSrcset.push(`/go-api/images/get/${url}`); - } - return finalSrcset; -}; - -export const createSrc = (immagine: string, ext?: boolean) => { - return `${ext ? "https://infoalloggi.it" : env.NEXT_PUBLIC_BASE_URL}/go-api/images/get/${immagine}`; -}; diff --git a/apps/infoalloggi/src/server/services/storage.service.ts b/apps/infoalloggi/src/server/services/storage.service.ts index 8d743ce..0058f78 100644 --- a/apps/infoalloggi/src/server/services/storage.service.ts +++ b/apps/infoalloggi/src/server/services/storage.service.ts @@ -21,7 +21,7 @@ export type FileMetadataWithAdmin = FileMetadata & { export async function fetchFiles(): Promise { try { const response = await fetch( - `${env.STORAGE_URL}/files?token=${env.STORAGE_TOKEN}`, + `${env.STORAGE_URL}/bucket/storage?token=${env.STORAGE_TOKEN}`, ); if (!response.ok) { console.error("Failed to fetch file list:", response.statusText); diff --git a/ref-docker-compose.yml b/ref-docker-compose.yml index a17e0c9..99c51a9 100644 --- a/ref-docker-compose.yml +++ b/ref-docker-compose.yml @@ -19,24 +19,41 @@ services: timeout: 5s retries: 10 - minio: - image: minio/minio - expose: - - "9000" - - "9001" + # minio: + # image: minio/minio + # expose: + # - "9000" + # - "9001" + # volumes: + # - minio_storage:/data + # networks: + # - dokploy-network + # environment: + # MINIO_ROOT_USER: ${MINIO_ROOT_USER} + # MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + # command: server --console-address ":9001" /data + # healthcheck: + # test: [ "CMD", "curl", "-f", "http://minio:9000/minio/health/live" ] + # interval: 10s + # timeout: 10s + # retries: 5 + storage: + image: "${IMAGE_REGISTRY:-ghcr.io}/${OWNER:-marcopedone}/go_fs:${TAG:-latest}" + ports: + - "8080:8080" + environment: + PORT: 8080 + AUTH_TOKEN: ${STORAGE_TOKEN} volumes: - - minio_storage:/data + - storage_data:/app/storage networks: - dokploy-network - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} - command: server --console-address ":9001" /data healthcheck: - test: [ "CMD", "curl", "-f", "http://minio:9000/minio/health/live" ] + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] interval: 10s - timeout: 10s + timeout: 5s retries: 5 + start_period: 10s tiles: image: eqalpha/keydb:latest @@ -144,6 +161,7 @@ services: JWT_SECRET: ${JWT_SECRET} SKEBBY_USER: ${SKEBBY_USER} SKEBBY_PASS: ${SKEBBY_PASS} + STORAGE_URL: http://storage:8080 STORAGE_TOKEN: ${STORAGE_TOKEN} networks: - dokploy-network @@ -164,7 +182,7 @@ networks: external: true volumes: - minio_storage: + storage_data: images_volume: storage_volume: tiles-data: