package main import ( "backend/config" "backend/queries" "backend/typesdefs" "backend/utils" "fmt" "image" "image/jpeg" "image/png" "io" "log" "net/http" "os" "path/filepath" "slices" "sync" "github.com/labstack/echo/v4" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" ) type ImageProcessing struct { ImagesToProcess []typesdefs.ImagesToProcess ProcessedImages []typesdefs.ProcessedImage UploadedImages []typesdefs.UploadedImage } func NewImageProcessing() *ImageProcessing { return &ImageProcessing{ ImagesToProcess: []typesdefs.ImagesToProcess{}, ProcessedImages: []typesdefs.ProcessedImage{}, UploadedImages: []typesdefs.UploadedImage{}, } } func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.ImagesToProcess) { if annuncio.Foto == nil || len(*annuncio.Foto) == 0 { return } for idx, url := range *annuncio.Foto { *p = append(*p, typesdefs.ImagesToProcess{ Codice: tmp.Codice, Url: url, Order: idx, }) } } func ProcessImagePool(toProcess *[]typesdefs.ImagesToProcess) ([]typesdefs.UploadedImage, error) { var ( ProcessedImages []typesdefs.ProcessedImage UploadedImages []typesdefs.UploadedImage mu sync.Mutex wg sync.WaitGroup ) sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) // clear images folder err := clearImageFolder() if err != nil { log.Fatalf("Failed to initialize image processing: %v", err) } // process images for _, img := range *toProcess { sem <- struct{}{} // acquire semaphore wg.Add(1) go func(img typesdefs.ImagesToProcess) { defer wg.Done() defer func() { <-sem }() // release semaphore imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url))) downloadImage(img.Url, imgPath) webpPath, thumbnailwebpPath, err := Conversion(imgPath) if err != nil { log.Fatalf("Failed to convert image: %v", err) } mu.Lock() ProcessedImages = append(ProcessedImages, typesdefs.ProcessedImage{ Codice: img.Codice, Order: img.Order, Path: webpPath, ThumnailPath: thumbnailwebpPath, OGUrl: img.Url, }) mu.Unlock() }(img) } wg.Wait() uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) var uploadWg sync.WaitGroup for _, img := range ProcessedImages { uploadSem <- struct{}{} // acquire semaphore uploadWg.Add(1) go func(img typesdefs.ProcessedImage) { defer uploadWg.Done() defer func() { <-uploadSem }() // release semaphore imgId, err := UploadFile(img.Path) if err != nil { log.Printf("failed to upload image: %v", err) return } thumbId, err := UploadFile(img.ThumnailPath) if err != nil { log.Printf("failed to upload thumbnail: %v", err) return } mu.Lock() UploadedImages = append(UploadedImages, typesdefs.UploadedImage{ Codice: img.Codice, Order: img.Order, ImgId: imgId, ThumbId: thumbId, OGUrl: img.OGUrl, }) mu.Unlock() }(img) } uploadWg.Wait() return UploadedImages, nil } func clearImageFolder() error { err := utils.SafeRemoveAll(config.Cfg.Images.Path) if err != nil { return err } err = os.Mkdir(config.Cfg.Images.Path, 0755) if err != nil { return err } return nil } func downloadImage(url, filename string) error { response, err := http.Get(url) if err != nil { return fmt.Errorf("failed to download image: %v", err) } defer response.Body.Close() file, err := os.Create(filename) if err != nil { return fmt.Errorf("failed to create image file: %v", err) } defer file.Close() _, err = io.Copy(file, response.Body) if err != nil { return fmt.Errorf("failed to save image file: %v", err) } return nil } func Conversion(input string) (string, string, error) { var empty string var width uint = 1920 output := filepath.Base(input) outputdir := filepath.Dir(input) ext := filepath.Ext(output) output = output[0 : len(output)-len(ext)] // Remove file extension // Add .webp extension log.Println("Converting", input, "to", output+".webp") f, err := os.Open(input) if err != nil { return empty, empty, err } defer f.Close() var img image.Image switch ext { case ".webp": log.Println("Image already in WebP format") img, err = webpbin.Decode(f) if err != nil { return empty, empty, err } case ".png", ".PNG": // Convert from PNG img, err = png.Decode(f) if err != nil { return empty, empty, err } case ".jpeg", ".jpg", ".JPG", ".JPEG": // Convert from JPEG img, err = jpeg.Decode(f) if err != nil { return empty, empty, err } default: return empty, empty, fmt.Errorf("error: Unsupported file type: %s", ext) } err = image_creation(img, width, output, outputdir) if err != nil { return empty, empty, err } err = thumbnail_creation(img, output, outputdir) if err != nil { return empty, empty, err } fmt.Println("Conversion completed successfully") return fmt.Sprintf("%s/%s.webp", outputdir, output), fmt.Sprintf("%s/thumbnail-%s.webp", outputdir, output), nil } func image_creation(img image.Image, width uint, output string, outputdir string) error { // Resize the image resizedImg := resize.Resize(width, 0, img, resize.Lanczos3) // Create the output file outputPath := fmt.Sprintf("%s/%s.webp", outputdir, output) log.Println("Creating", outputPath) out, err := os.Create(outputPath) if err != nil { return err } defer out.Close() // Encode the resized image to the output file err = webpbin.Encode(out, resizedImg) if err != nil { return err } return nil } func thumbnail_creation(img image.Image, output string, outputdir string) error { // Resize the image resizedImg := resize.Resize(32, 0, img, resize.Lanczos3) // Create the output file outputPath := fmt.Sprintf("%s/thumbnail-%s.webp", outputdir, output) log.Println("Creating", outputPath) out, err := os.Create(outputPath) if err != nil { return err } defer out.Close() // Encode the resized image to the output file err = webpbin.Encode(out, resizedImg) if err != nil { return err } return nil } func FallbackImage(c echo.Context) error { // Serve a fallback image if the requested image is not found fallbackPath := "fallback.webp" // Path to your fallback image if _, err := os.Stat(fallbackPath); os.IsNotExist(err) { return c.String(http.StatusNotFound, "Fallback image not found") } return c.File(fallbackPath) } type imgRef struct { Og_url string Img string Thumb string } func CodiciToProcessImages(annunci *typesdefs.AnnunciXML) ([]string, error) { currentBucketState, err := ListBucket("images") if err != nil { return nil, fmt.Errorf("failed to list bucket: %w", err) } //make a list of bucket ids bucketIds := []string{} for _, item := range currentBucketState { bucketIds = append(bucketIds, item.ID) } db_images, err := queries.GetImagesFromDB() if err != nil { return nil, err } dbImagesMap := make(map[string][]imgRef) // Convert slice to map for faster lookup for _, ref := range db_images { dbImagesMap[ref.Codice] = append(dbImagesMap[ref.Codice], imgRef{ Og_url: ref.Og_url, Img: ref.Img, Thumb: ref.Thumb, }) } var codiciToProcess []string for _, annuncio := range annunci.Annuncio { if annuncio.Codice != nil && annuncio.Foto != nil { refFoto, existsInDb := dbImagesMap[*annuncio.Codice] if !existsInDb { codiciToProcess = append(codiciToProcess, *annuncio.Codice) continue } needProcess := false for _, fotoUrl := range *annuncio.Foto { found := false for _, ref := range refFoto { if fotoUrl == ref.Og_url { if slices.Contains(bucketIds, ref.Img) && slices.Contains(bucketIds, ref.Thumb) { found = true break } else { needProcess = true break } } } if !found { needProcess = true break } if needProcess { break } } if needProcess { codiciToProcess = append(codiciToProcess, *annuncio.Codice) } } } return codiciToProcess, nil } func ForceClearImages(codici []string) error { var idsToDelete []string for _, codice := range codici { refs, err := queries.GetCodiceImagesFromDB(codice) if err != nil { return fmt.Errorf("failed to get images by codice %s: %w", codice, err) } for _, ref := range refs { idsToDelete = append(idsToDelete, ref.Img) idsToDelete = append(idsToDelete, ref.Thumb) } } err := DeleteIdsFromStorage(idsToDelete) if err != nil { return fmt.Errorf("failed to delete images from storage: %w", err) } for _, codice := range codici { err := queries.ClearImagesRefsByCodice(codice) if err != nil { return fmt.Errorf("failed to clear images by codice %s: %w", codice, err) } } return nil } func DeleteIdsFromStorage(ids []string) error { for _, id := range ids { err := DeleteFile(id) if err != nil { return fmt.Errorf("failed to delete file with id %s: %w", id, err) } } return nil }