package main import ( "backend/config" "backend/queries" "backend/typesdefs" "backend/utils" "fmt" "image" "image/jpeg" "image/png" "io" "log" "net/http" "os" "path/filepath" "strings" "sync" "time" "github.com/labstack/echo/v4" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" "github.com/schollz/progressbar/v3" ) func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) { if annuncio.Foto == nil || len(*annuncio.Foto) == 0 { return } for idx, url := range *annuncio.Foto { *p = append(*p, typesdefs.MediaToProcess{ Codice: tmp.Codice, Url: url, Order: idx, }) } } func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.UploadedMedia, error) { if len(*toProcess) == 0 { log.Println("No new images to process.") return []typesdefs.UploadedMedia{}, nil } var ( ProcessedImages []typesdefs.ProcessedMedia UploadedImages []typesdefs.UploadedMedia mu sync.Mutex wg sync.WaitGroup ) sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) totalImages := len(*toProcess) bar := progressbar.Default(int64(totalImages), "Processing images") // clear images folder err := clearImageFolder() if err != nil { log.Fatalf("Failed to initialize image processing: %v", err) } // process images for _, img := range *toProcess { sem <- struct{}{} // acquire semaphore wg.Add(1) go func(img typesdefs.MediaToProcess) { defer wg.Done() defer func() { <-sem }() // release semaphore imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url))) err := downloadImage(img.Url, imgPath) if err != nil { log.Printf("Failed to download image %s: %v", img.Url, err) return // Exit goroutine on download failure } webpPath, thumbnailwebpPath, err := Conversion(imgPath) if err != nil { log.Fatalf("Failed to convert image: %v", err) } mu.Lock() ProcessedImages = append(ProcessedImages, typesdefs.ProcessedMedia{ Codice: img.Codice, Order: img.Order, Path: webpPath, ThumnailPath: thumbnailwebpPath, OGUrl: img.Url, }) mu.Unlock() bar.Add(1) }(img) } wg.Wait() bar.Finish() fmt.Println() totalToUpload := len(ProcessedImages) uploadBar := progressbar.Default(int64(totalToUpload), "Uploading images") uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) var uploadWg sync.WaitGroup for _, img := range ProcessedImages { uploadSem <- struct{}{} // acquire semaphore uploadWg.Add(1) go func(img typesdefs.ProcessedMedia) { defer uploadWg.Done() defer func() { <-uploadSem }() // release semaphore imgId, err := UploadFile(img.Path, "image/webp", "images") if err != nil { log.Printf("failed to upload image: %v", err) return } thumbId, err := UploadFile(img.ThumnailPath, "image/webp", "images") if err != nil { log.Printf("failed to upload thumbnail: %v", err) return } mu.Lock() UploadedImages = append(UploadedImages, typesdefs.UploadedMedia{ Codice: img.Codice, Order: img.Order, MediaId: imgId, ThumbId: thumbId, OGUrl: img.OGUrl, }) uploadBar.Add(1) mu.Unlock() }(img) } uploadWg.Wait() uploadBar.Finish() fmt.Println() return UploadedImages, nil } func clearImageFolder() error { err := utils.SafeRemoveAll(config.Cfg.Images.Path) if err != nil { return err } err = os.Mkdir(config.Cfg.Images.Path, 0755) if err != nil { return err } return nil } func downloadImage(url, filename string) error { client := &http.Client{ Timeout: 2 * time.Minute, // Adjust based on expected image sizes } response, err := client.Get(url) if err != nil { return fmt.Errorf("failed to download image: %w", err) } defer response.Body.Close() // Check HTTP status if response.StatusCode != http.StatusOK { return fmt.Errorf("failed to download image: HTTP %d", response.StatusCode) } // Optional: Validate Content-Type contentType := response.Header.Get("Content-Type") if !strings.HasPrefix(contentType, "image/") { log.Printf("Warning: unexpected content type %s for %s", contentType, url) } file, err := os.Create(filename) if err != nil { return fmt.Errorf("failed to create image file: %w", err) } defer file.Close() _, err = io.Copy(file, response.Body) if err != nil { return fmt.Errorf("failed to save image file: %w", err) } return nil } func Conversion(input string) (string, string, error) { var empty string var width uint = 1920 output := filepath.Base(input) outputdir := filepath.Dir(input) ext := filepath.Ext(output) output = output[0 : len(output)-len(ext)] // Remove file extension // Add .webp extension //log.Println("Converting", input, "to", output+".webp") f, err := os.Open(input) if err != nil { return empty, empty, err } defer f.Close() var img image.Image switch ext { case ".webp": log.Println("Image already in WebP format") img, err = webpbin.Decode(f) if err != nil { return empty, empty, err } case ".png", ".PNG": // Convert from PNG img, err = png.Decode(f) if err != nil { return empty, empty, err } case ".jpeg", ".jpg", ".JPG", ".JPEG": // Convert from JPEG img, err = jpeg.Decode(f) if err != nil { return empty, empty, err } default: return empty, empty, fmt.Errorf("error: Unsupported file type: %s", ext) } err = image_creation(img, width, output, outputdir) if err != nil { return empty, empty, fmt.Errorf("error in image creation: %w", err) } err = thumbnail_creation(img, output, outputdir) if err != nil { return empty, empty, fmt.Errorf("error in thumbnail creation: %w", err) } //fmt.Println("Conversion completed successfully") return fmt.Sprintf("%s/%s.webp", outputdir, output), fmt.Sprintf("%s/thumbnail-%s.webp", outputdir, output), nil } func image_creation(img image.Image, width uint, output string, outputdir string) error { // Resize the image resizedImg := resize.Resize(width, 0, img, resize.Lanczos3) // Create the output file outputPath := fmt.Sprintf("%s/%s.webp", outputdir, output) //log.Println("Creating", outputPath) out, err := os.Create(outputPath) if err != nil { return err } defer out.Close() // Encode the resized image to the output file err = webpbin.Encode(out, resizedImg) if err != nil { return err } return nil } func thumbnail_creation(img image.Image, output string, outputdir string) error { // Resize the image resizedImg := resize.Resize(32, 0, img, resize.Lanczos3) // Create the output file outputPath := fmt.Sprintf("%s/thumbnail-%s.webp", outputdir, output) //log.Println("Creating", outputPath) out, err := os.Create(outputPath) if err != nil { return err } defer out.Close() // Encode the resized image to the output file err = webpbin.Encode(out, resizedImg) if err != nil { return err } return nil } func FallbackImage(c echo.Context) error { // Serve a fallback image if the requested image is not found fallbackPath := "fallback.webp" // Path to your fallback image if _, err := os.Stat(fallbackPath); os.IsNotExist(err) { return c.String(http.StatusNotFound, "Fallback image not found") } return c.File(fallbackPath) } type imgRef struct { Og_url string Img string Thumb string } func CodiciToProcessImages(annunci *typesdefs.AnnunciXML) ([]string, error) { currentBucketState, err := ListBucket("images") if err != nil { return nil, fmt.Errorf("failed to list bucket: %w", err) } //make a list of bucket ids bucketIdsMap := make(map[string]bool) for _, item := range currentBucketState { bucketIdsMap[item.ID] = true } db_images, err := queries.GetImagesFromDB() if err != nil { return nil, err } dbImagesMap := make(map[string][]imgRef) for _, ref := range db_images { dbImagesMap[ref.Codice] = append(dbImagesMap[ref.Codice], imgRef{ Og_url: ref.Og_url, Img: ref.Img, Thumb: ref.Thumb, }) } var codiciToProcess []string for _, annuncio := range annunci.Annuncio { if annuncio.Codice != nil && annuncio.Foto != nil { // Check if codice exists in DB refFotos, existsInDb := dbImagesMap[*annuncio.Codice] if !existsInDb { codiciToProcess = append(codiciToProcess, *annuncio.Codice) continue } needProcess := false for _, fotoUrl := range *annuncio.Foto { found := false // Check if the fotoUrl exists in the DB refs for _, ref := range refFotos { if fotoUrl == ref.Og_url { // check if the corresponding img and thumb are in the bucket if bucketIdsMap[ref.Img] && bucketIdsMap[ref.Thumb] { found = true break } } } if !found { needProcess = true break } } if needProcess { codiciToProcess = append(codiciToProcess, *annuncio.Codice) } } } return codiciToProcess, nil } func ForceClearImages(codici []string) error { var idsToDelete []string for _, codice := range codici { refs, err := queries.GetCodiceImagesFromDB(codice) if err != nil { return fmt.Errorf("failed to get images by codice %s: %w", codice, err) } for _, ref := range refs { idsToDelete = append(idsToDelete, ref.Img) idsToDelete = append(idsToDelete, ref.Thumb) } } err := DeleteIdsFromStorage(idsToDelete) if err != nil { return fmt.Errorf("failed to delete images from storage: %w", err) } for _, codice := range codici { err := queries.ClearImagesRefsByCodice(codice) if err != nil { return fmt.Errorf("failed to clear images by codice %s: %w", codice, err) } } return nil } // Removes stranded images (in storage but not in DB) func ReconcileImages() error { currentBucketState, err := ListBucket("images") if err != nil { return fmt.Errorf("failed to list bucket: %w", err) } //make a list of bucket ids bucketIdsMap := make(map[string]bool) for _, item := range currentBucketState { bucketIdsMap[item.ID] = true } db_images, err := queries.GetImagesFromDB() if err != nil { return fmt.Errorf("failed to get images from DB: %w", err) } knownStorageIds := make(map[string]bool) for _, ref := range db_images { knownStorageIds[ref.Img] = true knownStorageIds[ref.Thumb] = true } // reconcile: delete storage files not referenced in DB var strandedIds []string for _, item := range currentBucketState { if !knownStorageIds[item.ID] { strandedIds = append(strandedIds, item.ID) } } if len(strandedIds) > 0 { log.Printf("Reconciliation: found %d stranded files, deleting...", len(strandedIds)) if err := DeleteIdsFromStorage(strandedIds); err != nil { log.Printf("Warning: failed to delete stranded files: %v", err) // non-fatal } } else { log.Println("Reconciliation: no stranded files found.") } return nil }