From 364f4a77b382eb7b204890e4cd2f26f3b0a5ac09 Mon Sep 17 00:00:00 2001 From: Marco Pedone Date: Fri, 22 May 2026 15:49:28 +0200 Subject: [PATCH] mediarefs --- .../model/{images_refs.go => media_refs.go} | 13 +- .../postgres/public/model/videos_refs.go | 20 - .../postgres/public/table/images_refs.go | 90 --- .../postgres/public/table/media_refs.go | 93 ++++ .../postgres/public/table/table_use_schema.go | 3 +- .../postgres/public/table/videos_refs.go | 90 --- apps/backend/images.go | 255 +++------ apps/backend/jet.sh | 2 +- apps/backend/miogest_handler.go | 514 ------------------ apps/backend/miogest_parsing.go | 182 +++++++ apps/backend/queries/annunci.go | 48 +- apps/backend/queries/images.go | 69 --- apps/backend/queries/media.go | 51 ++ apps/backend/queries/queries.go | 3 +- apps/backend/queries/storage.go | 50 +- apps/backend/queries/videos.go | 65 --- apps/backend/server.go | 137 +---- apps/backend/typesdefs/models.go | 22 +- apps/backend/update.go | 359 ++++++++++++ apps/backend/video.go | 257 ++------- apps/db/migrations/57_mediatable.up.sql | 68 +++ .../src/components/annuncio_card.tsx | 102 ++-- .../src/forms/FormEditAnnuncio.tsx | 27 +- apps/infoalloggi/src/pages/annuncio/[cod].tsx | 18 +- .../src/schemas/public/ImagesRefs.ts | 46 -- .../src/schemas/public/MediaRefs.ts | 51 ++ .../src/schemas/public/PublicSchema.ts | 9 +- .../src/schemas/public/VideosRefs.ts | 46 -- .../src/server/api/routers/annunci.ts | 9 +- .../src/server/api/routers/revalidation.ts | 2 +- .../server/controllers/annunci.controller.ts | 65 +-- .../server/controllers/servizio.controller.ts | 13 +- .../src/server/services/interests.service.ts | 2 +- apps/infoalloggi/src/utils/kysely-helper.ts | 46 +- 34 files changed, 1154 insertions(+), 1673 deletions(-) rename apps/backend/gen/postgres/postgres/public/model/{images_refs.go => media_refs.go} (59%) delete mode 100644 apps/backend/gen/postgres/postgres/public/model/videos_refs.go delete mode 100644 apps/backend/gen/postgres/postgres/public/table/images_refs.go create mode 100644 apps/backend/gen/postgres/postgres/public/table/media_refs.go delete mode 100644 apps/backend/gen/postgres/postgres/public/table/videos_refs.go delete mode 100644 apps/backend/miogest_handler.go create mode 100644 apps/backend/miogest_parsing.go delete mode 100644 apps/backend/queries/images.go create mode 100644 apps/backend/queries/media.go delete mode 100644 apps/backend/queries/videos.go create mode 100644 apps/backend/update.go create mode 100644 apps/db/migrations/57_mediatable.up.sql delete mode 100644 apps/infoalloggi/src/schemas/public/ImagesRefs.ts create mode 100644 apps/infoalloggi/src/schemas/public/MediaRefs.ts delete mode 100644 apps/infoalloggi/src/schemas/public/VideosRefs.ts diff --git a/apps/backend/gen/postgres/postgres/public/model/images_refs.go b/apps/backend/gen/postgres/postgres/public/model/media_refs.go similarity index 59% rename from apps/backend/gen/postgres/postgres/public/model/images_refs.go rename to apps/backend/gen/postgres/postgres/public/model/media_refs.go index 3634307..ca6b4d9 100644 --- a/apps/backend/gen/postgres/postgres/public/model/images_refs.go +++ b/apps/backend/gen/postgres/postgres/public/model/media_refs.go @@ -11,10 +11,11 @@ import ( "github.com/google/uuid" ) -type ImagesRefs struct { - Codice string - Ordine int32 - Img uuid.UUID - Thumb uuid.UUID - OgURL string +type MediaRefs struct { + Codice string + Ordine int32 + StorageID uuid.UUID + OgURL string + MediaType string + IsThumbnail bool } diff --git a/apps/backend/gen/postgres/postgres/public/model/videos_refs.go b/apps/backend/gen/postgres/postgres/public/model/videos_refs.go deleted file mode 100644 index 7bc2e4d..0000000 --- a/apps/backend/gen/postgres/postgres/public/model/videos_refs.go +++ /dev/null @@ -1,20 +0,0 @@ -// -// Code generated by go-jet DO NOT EDIT. -// -// WARNING: Changes to this file may cause incorrect behavior -// and will be lost if the code is regenerated -// - -package model - -import ( - "github.com/google/uuid" -) - -type VideosRefs struct { - Codice string - Ordine int32 - Video uuid.UUID - Thumb uuid.UUID - OgURL string -} diff --git a/apps/backend/gen/postgres/postgres/public/table/images_refs.go b/apps/backend/gen/postgres/postgres/public/table/images_refs.go deleted file mode 100644 index 1202fc0..0000000 --- a/apps/backend/gen/postgres/postgres/public/table/images_refs.go +++ /dev/null @@ -1,90 +0,0 @@ -// -// Code generated by go-jet DO NOT EDIT. -// -// WARNING: Changes to this file may cause incorrect behavior -// and will be lost if the code is regenerated -// - -package table - -import ( - "github.com/go-jet/jet/v2/postgres" -) - -var ImagesRefs = newImagesRefsTable("public", "images_refs", "") - -type imagesRefsTable struct { - postgres.Table - - // Columns - Codice postgres.ColumnString - Ordine postgres.ColumnInteger - Img postgres.ColumnString - Thumb postgres.ColumnString - OgURL postgres.ColumnString - - AllColumns postgres.ColumnList - MutableColumns postgres.ColumnList - DefaultColumns postgres.ColumnList -} - -type ImagesRefsTable struct { - imagesRefsTable - - EXCLUDED imagesRefsTable -} - -// AS creates new ImagesRefsTable with assigned alias -func (a ImagesRefsTable) AS(alias string) *ImagesRefsTable { - return newImagesRefsTable(a.SchemaName(), a.TableName(), alias) -} - -// Schema creates new ImagesRefsTable with assigned schema name -func (a ImagesRefsTable) FromSchema(schemaName string) *ImagesRefsTable { - return newImagesRefsTable(schemaName, a.TableName(), a.Alias()) -} - -// WithPrefix creates new ImagesRefsTable with assigned table prefix -func (a ImagesRefsTable) WithPrefix(prefix string) *ImagesRefsTable { - return newImagesRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) -} - -// WithSuffix creates new ImagesRefsTable with assigned table suffix -func (a ImagesRefsTable) WithSuffix(suffix string) *ImagesRefsTable { - return newImagesRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) -} - -func newImagesRefsTable(schemaName, tableName, alias string) *ImagesRefsTable { - return &ImagesRefsTable{ - imagesRefsTable: newImagesRefsTableImpl(schemaName, tableName, alias), - EXCLUDED: newImagesRefsTableImpl("", "excluded", ""), - } -} - -func newImagesRefsTableImpl(schemaName, tableName, alias string) imagesRefsTable { - var ( - CodiceColumn = postgres.StringColumn("codice") - OrdineColumn = postgres.IntegerColumn("ordine") - ImgColumn = postgres.StringColumn("img") - ThumbColumn = postgres.StringColumn("thumb") - OgURLColumn = postgres.StringColumn("og_url") - allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, ImgColumn, ThumbColumn, OgURLColumn} - mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, ImgColumn, ThumbColumn, OgURLColumn} - defaultColumns = postgres.ColumnList{} - ) - - return imagesRefsTable{ - Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), - - //Columns - Codice: CodiceColumn, - Ordine: OrdineColumn, - Img: ImgColumn, - Thumb: ThumbColumn, - OgURL: OgURLColumn, - - AllColumns: allColumns, - MutableColumns: mutableColumns, - DefaultColumns: defaultColumns, - } -} diff --git a/apps/backend/gen/postgres/postgres/public/table/media_refs.go b/apps/backend/gen/postgres/postgres/public/table/media_refs.go new file mode 100644 index 0000000..6be17a5 --- /dev/null +++ b/apps/backend/gen/postgres/postgres/public/table/media_refs.go @@ -0,0 +1,93 @@ +// +// Code generated by go-jet DO NOT EDIT. +// +// WARNING: Changes to this file may cause incorrect behavior +// and will be lost if the code is regenerated +// + +package table + +import ( + "github.com/go-jet/jet/v2/postgres" +) + +var MediaRefs = newMediaRefsTable("public", "media_refs", "") + +type mediaRefsTable struct { + postgres.Table + + // Columns + Codice postgres.ColumnString + Ordine postgres.ColumnInteger + StorageID postgres.ColumnString + OgURL postgres.ColumnString + MediaType postgres.ColumnString + IsThumbnail postgres.ColumnBool + + AllColumns postgres.ColumnList + MutableColumns postgres.ColumnList + DefaultColumns postgres.ColumnList +} + +type MediaRefsTable struct { + mediaRefsTable + + EXCLUDED mediaRefsTable +} + +// AS creates new MediaRefsTable with assigned alias +func (a MediaRefsTable) AS(alias string) *MediaRefsTable { + return newMediaRefsTable(a.SchemaName(), a.TableName(), alias) +} + +// Schema creates new MediaRefsTable with assigned schema name +func (a MediaRefsTable) FromSchema(schemaName string) *MediaRefsTable { + return newMediaRefsTable(schemaName, a.TableName(), a.Alias()) +} + +// WithPrefix creates new MediaRefsTable with assigned table prefix +func (a MediaRefsTable) WithPrefix(prefix string) *MediaRefsTable { + return newMediaRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) +} + +// WithSuffix creates new MediaRefsTable with assigned table suffix +func (a MediaRefsTable) WithSuffix(suffix string) *MediaRefsTable { + return newMediaRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) +} + +func newMediaRefsTable(schemaName, tableName, alias string) *MediaRefsTable { + return &MediaRefsTable{ + mediaRefsTable: newMediaRefsTableImpl(schemaName, tableName, alias), + EXCLUDED: newMediaRefsTableImpl("", "excluded", ""), + } +} + +func newMediaRefsTableImpl(schemaName, tableName, alias string) mediaRefsTable { + var ( + CodiceColumn = postgres.StringColumn("codice") + OrdineColumn = postgres.IntegerColumn("ordine") + StorageIDColumn = postgres.StringColumn("storage_id") + OgURLColumn = postgres.StringColumn("og_url") + MediaTypeColumn = postgres.StringColumn("media_type") + IsThumbnailColumn = postgres.BoolColumn("is_thumbnail") + allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, StorageIDColumn, OgURLColumn, MediaTypeColumn, IsThumbnailColumn} + mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, StorageIDColumn, OgURLColumn, MediaTypeColumn, IsThumbnailColumn} + defaultColumns = postgres.ColumnList{IsThumbnailColumn} + ) + + return mediaRefsTable{ + Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), + + //Columns + Codice: CodiceColumn, + Ordine: OrdineColumn, + StorageID: StorageIDColumn, + OgURL: OgURLColumn, + MediaType: MediaTypeColumn, + IsThumbnail: IsThumbnailColumn, + + AllColumns: allColumns, + MutableColumns: mutableColumns, + DefaultColumns: defaultColumns, + } +} diff --git a/apps/backend/gen/postgres/postgres/public/table/table_use_schema.go b/apps/backend/gen/postgres/postgres/public/table/table_use_schema.go index 6c5ed5b..7a93655 100644 --- a/apps/backend/gen/postgres/postgres/public/table/table_use_schema.go +++ b/apps/backend/gen/postgres/postgres/public/table/table_use_schema.go @@ -11,7 +11,6 @@ package table // this method only once at the beginning of the program. func UseSchema(schema string) { Annunci = Annunci.FromSchema(schema) - ImagesRefs = ImagesRefs.FromSchema(schema) + MediaRefs = MediaRefs.FromSchema(schema) Storage = Storage.FromSchema(schema) - VideosRefs = VideosRefs.FromSchema(schema) } diff --git a/apps/backend/gen/postgres/postgres/public/table/videos_refs.go b/apps/backend/gen/postgres/postgres/public/table/videos_refs.go deleted file mode 100644 index 2cbc78c..0000000 --- a/apps/backend/gen/postgres/postgres/public/table/videos_refs.go +++ /dev/null @@ -1,90 +0,0 @@ -// -// Code generated by go-jet DO NOT EDIT. -// -// WARNING: Changes to this file may cause incorrect behavior -// and will be lost if the code is regenerated -// - -package table - -import ( - "github.com/go-jet/jet/v2/postgres" -) - -var VideosRefs = newVideosRefsTable("public", "videos_refs", "") - -type videosRefsTable struct { - postgres.Table - - // Columns - Codice postgres.ColumnString - Ordine postgres.ColumnInteger - Video postgres.ColumnString - Thumb postgres.ColumnString - OgURL postgres.ColumnString - - AllColumns postgres.ColumnList - MutableColumns postgres.ColumnList - DefaultColumns postgres.ColumnList -} - -type VideosRefsTable struct { - videosRefsTable - - EXCLUDED videosRefsTable -} - -// AS creates new VideosRefsTable with assigned alias -func (a VideosRefsTable) AS(alias string) *VideosRefsTable { - return newVideosRefsTable(a.SchemaName(), a.TableName(), alias) -} - -// Schema creates new VideosRefsTable with assigned schema name -func (a VideosRefsTable) FromSchema(schemaName string) *VideosRefsTable { - return newVideosRefsTable(schemaName, a.TableName(), a.Alias()) -} - -// WithPrefix creates new VideosRefsTable with assigned table prefix -func (a VideosRefsTable) WithPrefix(prefix string) *VideosRefsTable { - return newVideosRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName()) -} - -// WithSuffix creates new VideosRefsTable with assigned table suffix -func (a VideosRefsTable) WithSuffix(suffix string) *VideosRefsTable { - return newVideosRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName()) -} - -func newVideosRefsTable(schemaName, tableName, alias string) *VideosRefsTable { - return &VideosRefsTable{ - videosRefsTable: newVideosRefsTableImpl(schemaName, tableName, alias), - EXCLUDED: newVideosRefsTableImpl("", "excluded", ""), - } -} - -func newVideosRefsTableImpl(schemaName, tableName, alias string) videosRefsTable { - var ( - CodiceColumn = postgres.StringColumn("codice") - OrdineColumn = postgres.IntegerColumn("ordine") - VideoColumn = postgres.StringColumn("video") - ThumbColumn = postgres.StringColumn("thumb") - OgURLColumn = postgres.StringColumn("og_url") - allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, VideoColumn, ThumbColumn, OgURLColumn} - mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, VideoColumn, ThumbColumn, OgURLColumn} - defaultColumns = postgres.ColumnList{} - ) - - return videosRefsTable{ - Table: postgres.NewTable(schemaName, tableName, alias, allColumns...), - - //Columns - Codice: CodiceColumn, - Ordine: OrdineColumn, - Video: VideoColumn, - Thumb: ThumbColumn, - OgURL: OgURLColumn, - - AllColumns: allColumns, - MutableColumns: mutableColumns, - DefaultColumns: defaultColumns, - } -} diff --git a/apps/backend/images.go b/apps/backend/images.go index 5547f97..fa59443 100644 --- a/apps/backend/images.go +++ b/apps/backend/images.go @@ -2,6 +2,7 @@ package main import ( "backend/config" + models "backend/gen/postgres/postgres/public/model" "backend/queries" "backend/typesdefs" "backend/utils" @@ -18,7 +19,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" ) @@ -37,25 +37,26 @@ func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Ann } } -func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.UploadedMedia, error) { +func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess, isDryRun bool) error { if len(*toProcess) == 0 { log.Println("No new images to process.") - return []typesdefs.UploadedMedia{}, nil + return nil } - var ( - ProcessedImages []typesdefs.ProcessedMedia - UploadedImages []typesdefs.UploadedMedia - mu sync.Mutex - wg sync.WaitGroup - ) + var wg sync.WaitGroup + sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) totalImages := len(*toProcess) log.Printf("Starting processing of %d images with concurrency limit %d...", totalImages, config.Cfg.Images.ConcurrentLimit) + // clear images folder - err := clearImageFolder() - if err != nil { - log.Fatalf("Failed to initialize image processing: %v", err) + if isDryRun { + log.Println("Dry run. Clearing images folder...") + } else { + err := clearImageFolder() + if err != nil { + log.Fatalf("Failed to initialize image processing: %v", err) + } } // process images @@ -65,69 +66,73 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload go func(img typesdefs.MediaToProcess) { defer wg.Done() defer func() { <-sem }() // release semaphore + if isDryRun { + log.Printf("Dry run: image processing: %s_%d", img.Codice, img.Order) + return + } + log.Printf("%s_%d: elaborating", img.Codice, img.Order) imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url))) + + log.Printf("%s_%d: downloading", img.Codice, img.Order) err := downloadImage(img.Url, imgPath) if err != nil { log.Printf("Failed to download image %s: %v", img.Url, err) return // Exit goroutine on download failure } + + log.Printf("%s_%d: conversion", img.Codice, img.Order) webpPath, thumbnailwebpPath, err := Conversion(imgPath) if err != nil { log.Fatalf("Failed to convert image: %v", err) } - mu.Lock() - ProcessedImages = append(ProcessedImages, typesdefs.ProcessedMedia{ - Codice: img.Codice, - Order: img.Order, - Path: webpPath, - ThumnailPath: thumbnailwebpPath, - OGUrl: img.Url, + + log.Printf("%s_%d: upload", img.Codice, img.Order) + imgId, err := queries.UploadFile(webpPath, "image/webp", "images") + if err != nil { + log.Printf("failed to upload image: %v", err) + return + } + thumbId, err := queries.UploadFile(thumbnailwebpPath, "image/webp", "images") + if err != nil { + log.Printf("failed to upload thumbnail: %v", err) + return + } + + log.Printf("%s_%d: images_refs upd", img.Codice, img.Order) + err = queries.SetMediaToDB(models.MediaRefs{ + Codice: img.Codice, + Ordine: int32(img.Order), + OgURL: img.Url, + MediaType: "image", + StorageID: imgId, + IsThumbnail: false, }) - mu.Unlock() + if err != nil { + log.Printf("failed to set media to db: %v", err) + return + + } + err = queries.SetMediaToDB(models.MediaRefs{ + Codice: img.Codice, + Ordine: int32(img.Order), + OgURL: img.Url, + MediaType: "image", + StorageID: thumbId, + IsThumbnail: true, + }) + if err != nil { + log.Printf("failed to set media to db: %v", err) + return + } + + log.Printf("%s_%d: done", img.Codice, img.Order) }(img) } wg.Wait() log.Printf("Processed all images") fmt.Println() - totalToUpload := len(ProcessedImages) - log.Printf("Starting upload of %d images with concurrency limit %d...", totalToUpload, config.Cfg.Images.ConcurrentLimit) - uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) - var uploadWg sync.WaitGroup - for _, img := range ProcessedImages { - uploadSem <- struct{}{} // acquire semaphore - uploadWg.Add(1) - go func(img typesdefs.ProcessedMedia) { - defer uploadWg.Done() - defer func() { <-uploadSem }() // release semaphore - - imgId, err := queries.UploadFile(img.Path, "image/webp", "images") - if err != nil { - log.Printf("failed to upload image: %v", err) - return - } - thumbId, err := queries.UploadFile(img.ThumnailPath, "image/webp", "images") - if err != nil { - log.Printf("failed to upload thumbnail: %v", err) - return - } - mu.Lock() - UploadedImages = append(UploadedImages, typesdefs.UploadedMedia{ - Codice: img.Codice, - Order: img.Order, - MediaId: imgId, - ThumbId: thumbId, - OGUrl: img.OGUrl, - }) - - mu.Unlock() - }(img) - } - uploadWg.Wait() - log.Printf("Uploaded all images") - fmt.Println() - - return UploadedImages, nil + return nil } func clearImageFolder() error { @@ -271,137 +276,3 @@ func thumbnail_creation(img image.Image, output string, outputdir string) error } return nil } - -type imgRef struct { - Og_url string - Img uuid.UUID - Thumb uuid.UUID -} - -func CodiciToProcessImages(annunci *typesdefs.AnnunciXML) ([]string, error) { - currentBucketState, err := queries.GetStorageByTag("images") - if err != nil { - return nil, fmt.Errorf("failed to list bucket: %w", err) - } - //make a list of bucket ids - bucketIdsMap := make(map[uuid.UUID]bool) - for _, item := range currentBucketState { - bucketIdsMap[item.ID] = true - } - - db_images, err := queries.GetImagesFromDB() - if err != nil { - return nil, err - } - - dbImagesMap := make(map[string][]imgRef) - for _, ref := range db_images { - dbImagesMap[ref.Codice] = append(dbImagesMap[ref.Codice], imgRef{ - Og_url: ref.OgURL, - Img: ref.Img, - Thumb: ref.Thumb, - }) - } - - var codiciToProcess []string - for _, annuncio := range annunci.Annuncio { - - if annuncio.Codice != nil && annuncio.Foto != nil { - // Check if codice exists in DB - refFotos, existsInDb := dbImagesMap[*annuncio.Codice] - if !existsInDb { - codiciToProcess = append(codiciToProcess, *annuncio.Codice) - continue - } - needProcess := false - for _, fotoUrl := range *annuncio.Foto { - found := false - // Check if the fotoUrl exists in the DB refs - for _, ref := range refFotos { - if fotoUrl == ref.Og_url { - // check if the corresponding img and thumb are in the bucket - if bucketIdsMap[ref.Img] && bucketIdsMap[ref.Thumb] { - found = true - break - } - } - } - if !found { - needProcess = true - break - } - } - if needProcess { - codiciToProcess = append(codiciToProcess, *annuncio.Codice) - } - } - } - - return codiciToProcess, nil - -} - -func ForceClearImages(codici []string) error { - var idsToDelete uuid.UUIDs - for _, codice := range codici { - refs, err := queries.GetCodiceImagesFromDB(codice) - if err != nil { - return fmt.Errorf("failed to get images by codice %s: %w", codice, err) - } - for _, ref := range refs { - idsToDelete = append(idsToDelete, ref.Img) - idsToDelete = append(idsToDelete, ref.Thumb) - } - - } - err := queries.DeleteMultFile(idsToDelete) - if err != nil { - return fmt.Errorf("failed to delete images from storage: %w", err) - } - for _, codice := range codici { - err := queries.ClearImagesRefsByCodice(codice) - if err != nil { - return fmt.Errorf("failed to clear images by codice %s: %w", codice, err) - } - } - return nil -} - -// Removes stranded images (in storage but not in DB) -func ReconcileImages() error { - currentBucketState, err := queries.GetStorageByTag("images") - if err != nil { - return fmt.Errorf("failed to list bucket: %w", err) - } - //make a list of bucket ids - bucketIdsMap := make(map[uuid.UUID]bool) - for _, item := range currentBucketState { - bucketIdsMap[item.ID] = true - } - db_images, err := queries.GetImagesFromDB() - if err != nil { - return fmt.Errorf("failed to get images from DB: %w", err) - } - knownStorageIds := make(map[uuid.UUID]bool) - for _, ref := range db_images { - knownStorageIds[ref.Img] = true - knownStorageIds[ref.Thumb] = true - } - // reconcile: delete storage files not referenced in DB - var strandedIds uuid.UUIDs - for _, item := range currentBucketState { - if !knownStorageIds[item.ID] { - strandedIds = append(strandedIds, item.ID) - } - } - if len(strandedIds) > 0 { - log.Printf("Reconciliation: found %d stranded files, deleting...", len(strandedIds)) - if err := queries.DeleteMultFile(strandedIds); err != nil { - log.Printf("Warning: failed to delete stranded files: %v", err) // non-fatal - } - } else { - log.Println("Reconciliation: no stranded files found.") - } - return nil - -} diff --git a/apps/backend/jet.sh b/apps/backend/jet.sh index 297d498..6de1287 100644 --- a/apps/backend/jet.sh +++ b/apps/backend/jet.sh @@ -1,3 +1,3 @@ #!/bin/bash -jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,images_refs,videos_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum +jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum diff --git a/apps/backend/miogest_handler.go b/apps/backend/miogest_handler.go deleted file mode 100644 index 7184485..0000000 --- a/apps/backend/miogest_handler.go +++ /dev/null @@ -1,514 +0,0 @@ -package main - -import ( - "backend/config" - "backend/parser" - "backend/typesdefs" - "backend/utils" - "fmt" - "log" - "reflect" - "regexp" - "slices" - "strconv" - "strings" - "time" -) - -type MiogestHandler struct { - Caratterisiche typesdefs.ListaCaratteristiche - Categorie []typesdefs.Categoria - AnnunciXML typesdefs.AnnunciXML - AnnunciParsed typesdefs.AnnunciParsed - annunci_cache_expires time.Time - vars_cache_expires time.Time -} - -func NewMiogestHandler() *MiogestHandler { - return &MiogestHandler{ - Caratterisiche: typesdefs.ListaCaratteristiche{}, - Categorie: nil, - AnnunciXML: typesdefs.AnnunciXML{}, - AnnunciParsed: typesdefs.AnnunciParsed{}, - annunci_cache_expires: time.Now(), - vars_cache_expires: time.Now(), - } -} - -// ANNUNCI FROM MIOGEST -func (m *MiogestHandler) GetAnnunci() (typesdefs.AnnunciXML, error) { - if len(m.AnnunciXML.Annuncio) == 0 || time.Now().After(m.annunci_cache_expires) { - err := m.GetAnnunciFromMiogest() - if err != nil { - return typesdefs.AnnunciXML{}, err - } - } - return m.AnnunciXML, nil -} - -func (m *MiogestHandler) GetAnnunciFromMiogest() error { - maxRetries := 3 - baseDelay := 10 * time.Second - - for attempt := range maxRetries { - var err error - m.AnnunciXML, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml") - - if err == nil { - // Success - validate we got data - if len(m.AnnunciXML.Annuncio) > 0 { - m.annunci_cache_expires = time.Now().Add(1 * time.Hour) - return nil - } - log.Printf("Warning: Successfully fetched XML but got 0 annunci") - } - - // Log the specific error type - if err != nil { - log.Printf("Failed to get annunci (attempt %d/%d): %v", attempt+1, maxRetries, err) - } - - // Failed - check if we should retry - if attempt < maxRetries-1 { - delay := baseDelay * time.Duration(1< bestPriority { - bestPriority = p - best = i - } - } - return best, true -} - -func processLocatore(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { - idx, ok := selectTargetCliente(annuncio) - if !ok { - log.Printf("Annuncio %s has no client information", utils.GetStringValue(annuncio.Codice)) - return - } - trg := annuncio.Clienti[idx] - - tmp.Email = parser.ParseEmail(trg.Email1) - - var number string - for _, tel := range []*string{trg.Tel1, trg.Tel2, trg.Tel3} { - if tel != nil { - cleanedTel := re.ReplaceAllString(*tel, "") - if cleanedTel != "" { - number = cleanedTel - break - } - } - } - tmp.Numero = &number - if trg.Cognome != nil && trg.Nome != nil { - rawLoc := fmt.Sprintf("%s %s", utils.MakeUppercase(*trg.Cognome), utils.MakeUppercase(*trg.Nome)) - rawLoc = strings.TrimSpace(rawLoc) - rawLoc = regexp.MustCompile(`\s+`).ReplaceAllString(rawLoc, " ") // Replace multiple spaces with a single space - tmp.Locatore = strUpd(rawLoc) - } - tmp.Idlocatore = trg.Id - tmp.Tipo_locatore = strUpd(flatClienteTipologia(trg.Tipologie)) -} - -func processIndirizzo(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { - tmp.Indirizzo = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Indirizzo))) - tmp.Civico = annuncio.Civico - tmp.Comune = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Comune))) - tmp.Cap = annuncio.Cap - tmp.Provincia = annuncio.Provincia - tmp.Regione = annuncio.Regione - tmp.Lat = annuncio.Latitudine - tmp.Lon = annuncio.Longitudine - tmp.Indirizzo_secondario = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.IndirizzoSecondario))) - tmp.Civico_secondario = annuncio.CivicoSecondario - tmp.Lat_secondario = annuncio.LatitudineSecondario - tmp.Lon_secondario = annuncio.LongitudineSecondario -} - -func processCategorie(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, m *MiogestHandler) { - if annuncio.Categoria == nil { - return - } - resultArray := make([]string, 0) - cats := *annuncio.Categoria - defaultCategorie := m.GetCategorie() - for _, cat := range cats { - for _, catdef := range defaultCategorie { - if cat == catdef.Id { - resultArray = append(resultArray, catdef.Nome) - } - } - } - if tmp.Categorie == nil { - tmp.Categorie = &[]string{} - } - *tmp.Categorie = resultArray -} - -func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { - tmpanno := utils.GetStringValue(annuncio.Anno) - if tmpanno != "0" && tmpanno != "" { - tmp.Anno = &tmpanno - } - - tmp.Prezzo = parser.ParsePrezzo(annuncio.Prezzo) - tmp.Classe = annuncio.Classe - if annuncio.Mq != nil { - replaced := strings.ReplaceAll(*annuncio.Mq, ",", ".") - if s, err := strconv.ParseFloat(replaced, 64); err == nil { - tmp.Mq = &s - } - - } - tmp.Piano = annuncio.Piano - tmp.Piano_palazzo = annuncio.PianiCondominio - tmp.Unita_condominio = annuncio.UnitaCondominio - tmp.Numero_vani = annuncio.Vani - tmp.Numero_camere = annuncio.Camere - tmp.Numero_bagni = annuncio.Bagni - tmp.Numero_balconi = annuncio.Balconi - tmp.Numero_terrazzi = annuncio.Terrazzi - tmp.Numero_box = annuncio.Box - tmp.Numero_postiauto = annuncio.PostiAuto -} - -func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, m *MiogestHandler) map[string]any { - if reflect.TypeFor[*typesdefs.AnnuncioXML]().Kind() != reflect.Pointer || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct { - return nil - } - tmpCaratteristiche := make(map[string]any) - val := reflect.ValueOf(wrkrecord).Elem() - numField := val.NumField() - for i := range numField { - if strings.HasPrefix(val.Type().Field(i).Name, "Scheda_") { - nameField := val.Type().Field(i).Name - valueField := val.Field(i) - if valueField.IsValid() && !valueField.IsNil() { - tmpCaratteristiche[nameField] = valueField.Interface() - } else { - tmpCaratteristiche[nameField] = nil - } - } - } - for _, scheda := range m.GetCaratteristiche().Caratteristiche { - if _, ok := tmpCaratteristiche[scheda.Tagxml]; !ok { - tmpCaratteristiche[scheda.Tagxml] = nil - } - } - return tmpCaratteristiche -} diff --git a/apps/backend/miogest_parsing.go b/apps/backend/miogest_parsing.go new file mode 100644 index 0000000..2843c90 --- /dev/null +++ b/apps/backend/miogest_parsing.go @@ -0,0 +1,182 @@ +package main + +import ( + "backend/parser" + "backend/typesdefs" + "backend/utils" + "fmt" + "log" + "reflect" + "regexp" + "slices" + "strconv" + "strings" +) + +var re = regexp.MustCompile(`[^+\d]`) + +func boolUpd(b bool) *bool { + return &b +} + +func strUpd(s string) *string { + return &s +} + +func flatClienteTipologia(t []string) string { + if len(t) == 0 { + return "" + } + if slices.Contains(t, "Proprietario") { + return "proprietario" + } else if slices.Contains(t, "Delegato") { + return "delegato" + } else if slices.Contains(t, "Cliente") { + return "inquilino" + } + return "" +} +func tipologiaPriority(tipologie []string) int { + if slices.Contains(tipologie, "Delegato") { + return 3 + } + if slices.Contains(tipologie, "Cliente") { + return 2 + } + if slices.Contains(tipologie, "Proprietario") { + return 1 + } + return 0 +} +func selectTargetCliente(annuncio *typesdefs.AnnuncioXML) (int, bool) { + if len(annuncio.Clienti) == 0 { + return 0, false + } + best := 0 + bestPriority := tipologiaPriority(annuncio.Clienti[0].Tipologie) + for i := 1; i < len(annuncio.Clienti); i++ { + if p := tipologiaPriority(annuncio.Clienti[i].Tipologie); p > bestPriority { + bestPriority = p + best = i + } + } + return best, true +} + +func processLocatore(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { + idx, ok := selectTargetCliente(annuncio) + if !ok { + log.Printf("Annuncio %s has no client information", utils.GetStringValue(annuncio.Codice)) + return + } + trg := annuncio.Clienti[idx] + + tmp.Email = parser.ParseEmail(trg.Email1) + + var number string + for _, tel := range []*string{trg.Tel1, trg.Tel2, trg.Tel3} { + if tel != nil { + cleanedTel := re.ReplaceAllString(*tel, "") + if cleanedTel != "" { + number = cleanedTel + break + } + } + } + tmp.Numero = &number + if trg.Cognome != nil && trg.Nome != nil { + rawLoc := fmt.Sprintf("%s %s", utils.MakeUppercase(*trg.Cognome), utils.MakeUppercase(*trg.Nome)) + rawLoc = strings.TrimSpace(rawLoc) + rawLoc = regexp.MustCompile(`\s+`).ReplaceAllString(rawLoc, " ") // Replace multiple spaces with a single space + tmp.Locatore = strUpd(rawLoc) + } + tmp.Idlocatore = trg.Id + tmp.Tipo_locatore = strUpd(flatClienteTipologia(trg.Tipologie)) +} + +func processIndirizzo(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { + tmp.Indirizzo = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Indirizzo))) + tmp.Civico = annuncio.Civico + tmp.Comune = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Comune))) + tmp.Cap = annuncio.Cap + tmp.Provincia = annuncio.Provincia + tmp.Regione = annuncio.Regione + tmp.Lat = annuncio.Latitudine + tmp.Lon = annuncio.Longitudine + tmp.Indirizzo_secondario = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.IndirizzoSecondario))) + tmp.Civico_secondario = annuncio.CivicoSecondario + tmp.Lat_secondario = annuncio.LatitudineSecondario + tmp.Lon_secondario = annuncio.LongitudineSecondario +} + +func processCategorie(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, def_cats *[]typesdefs.Categoria) { + if annuncio.Categoria == nil { + return + } + resultArray := make([]string, 0) + cats := *annuncio.Categoria + for _, cat := range cats { + for _, catdef := range *def_cats { + if cat == catdef.Id { + resultArray = append(resultArray, catdef.Nome) + } + } + } + if tmp.Categorie == nil { + tmp.Categorie = &[]string{} + } + *tmp.Categorie = resultArray +} + +func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { + tmpanno := utils.GetStringValue(annuncio.Anno) + if tmpanno != "0" && tmpanno != "" { + tmp.Anno = &tmpanno + } + + tmp.Prezzo = parser.ParsePrezzo(annuncio.Prezzo) + tmp.Classe = annuncio.Classe + if annuncio.Mq != nil { + replaced := strings.ReplaceAll(*annuncio.Mq, ",", ".") + if s, err := strconv.ParseFloat(replaced, 64); err == nil { + tmp.Mq = &s + } + + } + tmp.Piano = annuncio.Piano + tmp.Piano_palazzo = annuncio.PianiCondominio + tmp.Unita_condominio = annuncio.UnitaCondominio + tmp.Numero_vani = annuncio.Vani + tmp.Numero_camere = annuncio.Camere + tmp.Numero_bagni = annuncio.Bagni + tmp.Numero_balconi = annuncio.Balconi + tmp.Numero_terrazzi = annuncio.Terrazzi + tmp.Numero_box = annuncio.Box + tmp.Numero_postiauto = annuncio.PostiAuto +} + +func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, caratt *typesdefs.ListaCaratteristiche) map[string]any { + if reflect.TypeFor[*typesdefs.AnnuncioXML]().Kind() != reflect.Pointer || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct { + return nil + } + tmpCaratteristiche := make(map[string]any) + val := reflect.ValueOf(wrkrecord).Elem() + numField := val.NumField() + for i := range numField { + if strings.HasPrefix(val.Type().Field(i).Name, "Scheda_") { + nameField := val.Type().Field(i).Name + valueField := val.Field(i) + if valueField.IsValid() && !valueField.IsNil() { + tmpCaratteristiche[nameField] = valueField.Interface() + } else { + tmpCaratteristiche[nameField] = nil + } + } + } + for _, scheda := range caratt.Caratteristiche { + if _, ok := tmpCaratteristiche[scheda.Tagxml]; !ok { + tmpCaratteristiche[scheda.Tagxml] = nil + } + } + return tmpCaratteristiche +} diff --git a/apps/backend/queries/annunci.go b/apps/backend/queries/annunci.go index d8f5660..ce5e074 100644 --- a/apps/backend/queries/annunci.go +++ b/apps/backend/queries/annunci.go @@ -5,7 +5,7 @@ import ( . "backend/gen/postgres/postgres/public/table" "backend/typesdefs" "fmt" - "slices" + "log" "time" . "github.com/go-jet/jet/v2/postgres" @@ -22,26 +22,22 @@ func AnnunciGetActive() ([]string, error) { return annunci, nil } -func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) error { +func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool, isDryRun bool) error { var err error if resetAllBeforeUpdate { //SET ALL WEB TO FALSE stmt := Annunci.UPDATE(Annunci.Web, Annunci.Homepage, Annunci.Stato).SET(false, false, "Sospeso").WHERE(Annunci.Codice.IS_NOT_NULL()) - _, err = stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to set all web to false: %w", err) + if isDryRun { + log.Printf("Dry run: Setting all web to false") + } else { + _, err = stmt.Exec(postgres) + if err != nil { + return fmt.Errorf("failed to set all web to false: %w", err) + } } } - //GET ALL CODICI - var codici []string - stmt := SELECT(Annunci.Codice).DISTINCT(Annunci.Codice).FROM(Annunci) - err = stmt.Query(postgres, &codici) - if err != nil { - return fmt.Errorf("failed to get all codici: %w", err) - } - var to_update []models.Annunci var to_insert []models.Annunci for _, annuncio := range annunci.Annuncio { value := models.Annunci{ @@ -97,35 +93,19 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e TipoLocatore: annuncio.Tipo_locatore, MediaUpdatedAt: Ptr(time.Now()), } - if slices.Contains(codici, annuncio.Codice) { - to_update = append(to_update, value) - - } else { - to_insert = append(to_insert, value) - } - + to_insert = append(to_insert, value) } if len(to_insert) > 0 { - stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert) - _, err = stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to insert/update annunci batch: %w", err) - } - } - - if len(to_update) > 0 { - for _, upd := range to_update { - stmt := Annunci.UPDATE(Annunci.MutableColumns).MODEL(upd).WHERE(Annunci.Codice.EQ(Text(upd.Codice))) + stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert).ON_CONFLICT(Annunci.Codice).DO_UPDATE(SET(Annunci.MutableColumns.SET(Annunci.EXCLUDED.MutableColumns))) + if isDryRun { + log.Printf("Dry run: would have inserted %d records\n", len(to_insert)) + } else { _, err = stmt.Exec(postgres) if err != nil { return fmt.Errorf("failed to insert/update annunci batch: %w", err) } } } - - if err != nil { - return fmt.Errorf("failed to insert/update annunci batch: %w", err) - } fmt.Println("Annunci insert/update done") return nil } diff --git a/apps/backend/queries/images.go b/apps/backend/queries/images.go deleted file mode 100644 index 9c6353d..0000000 --- a/apps/backend/queries/images.go +++ /dev/null @@ -1,69 +0,0 @@ -package queries - -import ( - models "backend/gen/postgres/postgres/public/model" - . "backend/gen/postgres/postgres/public/table" - "backend/typesdefs" - "fmt" - "log" - - . "github.com/go-jet/jet/v2/postgres" -) - -func SetImagesToDB(pool *[]typesdefs.UploadedMedia) error { - if len(*pool) == 0 { - log.Println("No new images to save to DB.") - return nil - } - - var to_insert []models.ImagesRefs - - for _, data := range *pool { - - to_insert = append(to_insert, models.ImagesRefs{ - Codice: data.Codice, - Ordine: int32(data.Order), - Img: data.MediaId, - Thumb: data.ThumbId, - OgURL: data.OGUrl, - }) - } - stmt := ImagesRefs.INSERT(ImagesRefs.AllColumns).MODELS(to_insert).ON_CONFLICT(ImagesRefs.Codice, ImagesRefs.Ordine).DO_UPDATE(SET(ImagesRefs.Img.SET(ImagesRefs.EXCLUDED.Img), ImagesRefs.Thumb.SET(ImagesRefs.EXCLUDED.Thumb))) - - _, err := stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to insert/update images batch: %w", err) - } - - return nil -} - -func GetImagesFromDB() ([]models.ImagesRefs, error) { - var images []models.ImagesRefs - stmt := SELECT(ImagesRefs.AllColumns).FROM(ImagesRefs).ORDER_BY(ImagesRefs.Ordine.ASC()) - err := stmt.Query(postgres, &images) - if err != nil { - return nil, fmt.Errorf("failed to get images: %w", err) - } - return images, nil -} - -func GetCodiceImagesFromDB(codice string) ([]models.ImagesRefs, error) { - var images []models.ImagesRefs - stmt := SELECT(ImagesRefs.AllColumns).FROM(ImagesRefs).WHERE(ImagesRefs.Codice.EQ(Text(codice))).ORDER_BY(ImagesRefs.Ordine.ASC()) - err := stmt.Query(postgres, &images) - if err != nil { - return nil, fmt.Errorf("failed to get images: %w", err) - } - return images, nil -} - -func ClearImagesRefsByCodice(codice string) error { - stmt := ImagesRefs.DELETE().WHERE(ImagesRefs.Codice.EQ(Text(codice))) - _, err := stmt.Exec(postgres) - - if err != nil { - return fmt.Errorf("failed to clear images refs by codice: %w", err) - } - return nil -} diff --git a/apps/backend/queries/media.go b/apps/backend/queries/media.go new file mode 100644 index 0000000..3372aba --- /dev/null +++ b/apps/backend/queries/media.go @@ -0,0 +1,51 @@ +package queries + +import ( + models "backend/gen/postgres/postgres/public/model" + . "backend/gen/postgres/postgres/public/table" + "fmt" + + . "github.com/go-jet/jet/v2/postgres" +) + +func SetMediaToDB(to_insert models.MediaRefs) error { + + stmt := MediaRefs.INSERT(MediaRefs.AllColumns).MODEL(to_insert).ON_CONFLICT(MediaRefs.Codice, MediaRefs.Ordine, MediaRefs.IsThumbnail, MediaRefs.MediaType).DO_UPDATE(SET(MediaRefs.StorageID.SET(MediaRefs.EXCLUDED.StorageID), MediaRefs.OgURL.SET(MediaRefs.EXCLUDED.OgURL))) + + _, err := stmt.Exec(postgres) + if err != nil { + return fmt.Errorf("failed to insert/update media_refs: %w", err) + } + + return nil +} + +func GetMediaFromDB() ([]models.MediaRefs, error) { + var media []models.MediaRefs + stmt := SELECT(MediaRefs.AllColumns).FROM(MediaRefs).ORDER_BY(MediaRefs.Ordine.ASC()) + err := stmt.Query(postgres, &media) + if err != nil { + return nil, fmt.Errorf("failed to get media: %w", err) + } + return media, nil +} + +// func GetCodiceMediaFromDB(codice string) ([]models.MediaRefs, error) { +// var media []models.MediaRefs +// stmt := SELECT(MediaRefs.AllColumns).FROM(MediaRefs).WHERE(MediaRefs.Codice.EQ(Text(codice))).ORDER_BY(MediaRefs.Ordine.ASC()) +// err := stmt.Query(postgres, &media) +// if err != nil { +// return nil, fmt.Errorf("failed to get media: %w", err) +// } +// return media, nil +// } + +// func ClearMediaRefsByCodice(codice string) error { +// stmt := MediaRefs.DELETE().WHERE(MediaRefs.Codice.EQ(Text(codice))) +// _, err := stmt.Exec(postgres) + +// if err != nil { +// return fmt.Errorf("failed to clear media_refs by codice: %w", err) +// } +// return nil +// } diff --git a/apps/backend/queries/queries.go b/apps/backend/queries/queries.go index 4fd329c..15a03dc 100644 --- a/apps/backend/queries/queries.go +++ b/apps/backend/queries/queries.go @@ -3,6 +3,7 @@ package queries import ( "database/sql" "fmt" + "log" "os" "github.com/joho/godotenv" @@ -19,7 +20,7 @@ func init() { connStr := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", os.Getenv("POSTGRES_USER"), os.Getenv("POSTGRES_PASSWORD"), os.Getenv("PGHOST"), os.Getenv("PGPORT"), os.Getenv("POSTGRES_DB")) postgres, err = sql.Open("postgres", connStr) if err != nil { - panic(err) + log.Fatalf("Error connecting to database: %v", err) } } diff --git a/apps/backend/queries/storage.go b/apps/backend/queries/storage.go index ee075c1..fb8920f 100644 --- a/apps/backend/queries/storage.go +++ b/apps/backend/queries/storage.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" - . "github.com/go-jet/jet/v2/postgres" + //. "github.com/go-jet/jet/v2/postgres" "github.com/google/uuid" ) @@ -21,32 +21,32 @@ import ( // return nil // } -func DeleteMultFile(ids uuid.UUIDs) error { - if len(ids) == 0 { - return nil - } - var sqlIDs []Expression - for _, id := range ids { - sqlIDs = append(sqlIDs, UUID(id)) - } - stmt := Storage.DELETE().WHERE(Storage.ID.IN(sqlIDs...)) - _, err := stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to delete files: %w", err) - } - return nil -} +// func DeleteMultFile(ids uuid.UUIDs) error { +// if len(ids) == 0 { +// return nil +// } +// var sqlIDs []Expression +// for _, id := range ids { +// sqlIDs = append(sqlIDs, UUID(id)) +// } +// stmt := Storage.DELETE().WHERE(Storage.ID.IN(sqlIDs...)) +// _, err := stmt.Exec(postgres) +// if err != nil { +// return fmt.Errorf("failed to delete files: %w", err) +// } +// return nil +// } -func GetStorageByTag(tag string) ([]models.Storage, error) { - var rows []models.Storage - stmt := SELECT(Storage.AllColumns).FROM(Storage).WHERE(Storage.Tag.EQ(Text(tag))) - err := stmt.Query(postgres, &rows) - if err != nil { - return nil, fmt.Errorf("failed to get storage by tag: %w", err) - } - return rows, nil +// func GetStorageByTag(tag string) ([]models.Storage, error) { +// var rows []models.Storage +// stmt := SELECT(Storage.AllColumns).FROM(Storage).WHERE(Storage.Tag.EQ(Text(tag))) +// err := stmt.Query(postgres, &rows) +// if err != nil { +// return nil, fmt.Errorf("failed to get storage by tag: %w", err) +// } +// return rows, nil -} +// } func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) { diff --git a/apps/backend/queries/videos.go b/apps/backend/queries/videos.go deleted file mode 100644 index 7856add..0000000 --- a/apps/backend/queries/videos.go +++ /dev/null @@ -1,65 +0,0 @@ -package queries - -import ( - models "backend/gen/postgres/postgres/public/model" - . "backend/gen/postgres/postgres/public/table" - "backend/typesdefs" - "fmt" - "log" - - . "github.com/go-jet/jet/v2/postgres" -) - -func SetVideosToDB(pool *[]typesdefs.UploadedMedia) error { - if len(*pool) == 0 { - log.Println("No new videos to save to DB.") - return nil - } - var to_insert []models.VideosRefs - - for _, data := range *pool { - to_insert = append(to_insert, models.VideosRefs{ - Codice: data.Codice, - Ordine: int32(data.Order), - Video: data.MediaId, - Thumb: data.ThumbId, - OgURL: data.OGUrl, - }) - - } - stmt := VideosRefs.INSERT(VideosRefs.AllColumns).MODELS(to_insert).ON_CONFLICT(VideosRefs.Codice, VideosRefs.Ordine).DO_UPDATE(SET(VideosRefs.Video.SET(VideosRefs.EXCLUDED.Video), VideosRefs.Thumb.SET(VideosRefs.EXCLUDED.Thumb))) - _, err := stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to insert video refs: %w", err) - } - return nil -} - -func GetVideosFromDB() ([]models.VideosRefs, error) { - var videos []models.VideosRefs - stmt := VideosRefs.SELECT(VideosRefs.AllColumns).FROM(VideosRefs).ORDER_BY(VideosRefs.Ordine.ASC()) - err := stmt.Query(postgres, &videos) - if err != nil { - return nil, fmt.Errorf("failed to get videos: %w", err) - } - return videos, nil -} - -func GetCodiceVideosFromDB(codice string) ([]models.VideosRefs, error) { - var videos []models.VideosRefs - stmt := VideosRefs.SELECT(VideosRefs.AllColumns).FROM(VideosRefs).WHERE(VideosRefs.Codice.EQ(Text(codice))).ORDER_BY(VideosRefs.Ordine.ASC()) - err := stmt.Query(postgres, &videos) - if err != nil { - return nil, fmt.Errorf("failed to get videos: %w", err) - } - return videos, nil -} - -func ClearVideosRefsByCodice(codice string) error { - stmt := VideosRefs.DELETE().WHERE(VideosRefs.Codice.EQ(Text(codice))) - _, err := stmt.Exec(postgres) - if err != nil { - return fmt.Errorf("failed to clear videos refs by codice: %w", err) - } - return nil -} diff --git a/apps/backend/server.go b/apps/backend/server.go index 643fcc4..7a39555 100644 --- a/apps/backend/server.go +++ b/apps/backend/server.go @@ -3,34 +3,25 @@ package main import ( "backend/config" "backend/queries" - "backend/typesdefs" "fmt" "io" "log" - "math/rand" "net/http" "os" "strconv" "strings" "text/template" - "time" "github.com/labstack/echo/v4" ) type Server struct { - miogest *MiogestHandler - config *config.Config + config *config.Config } func NewServer(cfg *config.Config) (*Server, error) { - miogest := NewMiogestHandler() - if miogest == nil { - return nil, fmt.Errorf("error connecting to miogest") - } return &Server{ - miogest: miogest, - config: cfg, + config: cfg, }, nil } @@ -78,126 +69,18 @@ func (s *Server) SetupRoutes() *echo.Echo { return c.JSONPretty(http.StatusOK, fmt.Sprintf("DB Connection: %v", h), " ") }) - // MIOGEST ROUTES - e.GET("/cats", func(c echo.Context) error { - var data = s.miogest.GetCaratteristiche() - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/categ", func(c echo.Context) error { - var data = s.miogest.GetCategorie() - return c.JSONPretty(http.StatusOK, data, " ") - }) - - e.GET("/miogest", func(c echo.Context) error { - data, err := s.miogest.GetAnnunci() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/miogest-sample", func(c echo.Context) error { - xml, err := s.miogest.GetAnnunci() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - rand := rand.Intn(len(xml.Annuncio)) - data := xml.Annuncio[rand] - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/parse", func(c echo.Context) error { - start := time.Now() - var data, err = s.miogest.GetAnnunciParsed(nil, nil, false) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - end := time.Now() - fmt.Println("time taken:", end.Sub(start).String()) - return c.JSONPretty(http.StatusOK, data, " ") - }) e.GET("/update", func(c echo.Context) error { - forceMediaStale := false - forceMediaStr := c.QueryParam("forcemedia") - if forceMediaStr == "true" { - forceMediaStale = true - } - imgPool := []typesdefs.MediaToProcess{} - videoPool := []typesdefs.MediaToProcess{} - - data, err := s.miogest.GetAnnunciParsed(&imgPool, &videoPool, forceMediaStale) + //da inizializzare on server startup non qui + u := NewUpdater(UpdaterConfig{ + isDryRun: c.QueryParam("dryrun") == "true", + codFilter: c.QueryParam("cod"), + forceMediaRefresh: c.QueryParam("forcemedia") == "true", + }) + err := u.Update() if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } - err = queries.AnnunciInsert(data, true) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - imgRefs, err := ProcessImagePool(&imgPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.SetImagesToDB(&imgRefs) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - videoRefs, err := ProcessVideoPool(&videoPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.SetVideosToDB(&videoRefs) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - /* - err = invalidateAllAnnunciFB() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - */ - log.Println("Update completed") - return c.String(http.StatusOK, "Updated") - }) - e.GET("/update-cod/:cod", func(c echo.Context) error { - - cod := c.Param("cod") - imgPool := []typesdefs.MediaToProcess{} - videoPool := []typesdefs.MediaToProcess{} - data, err := s.miogest.GetAnnuncioParsed(cod, &imgPool, &videoPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.AnnunciInsert(data, false) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - imgRefs, err := ProcessImagePool(&imgPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.SetImagesToDB(&imgRefs) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - videoRefs, err := ProcessVideoPool(&videoPool) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - err = queries.SetVideosToDB(&videoRefs) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - /* - url := fmt.Sprintf("https://www.infoalloggi.it/annuncio/%s", cod) - err = invalidateFacebookCache(url) - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - */ - log.Printf("Update %s completed", cod) - return c.String(http.StatusOK, "Updated") + return c.String(http.StatusOK, "update") }) //IMAGE ROUTES diff --git a/apps/backend/typesdefs/models.go b/apps/backend/typesdefs/models.go index 0c82f55..62fbe66 100644 --- a/apps/backend/typesdefs/models.go +++ b/apps/backend/typesdefs/models.go @@ -2,8 +2,6 @@ package typesdefs import ( "time" - - "github.com/google/uuid" ) type AnnunciXML struct { @@ -128,7 +126,9 @@ type AnnuncioXML struct { Url *string `xml:"Url"` } `xml:"AMVideo"` } `xml:"AMMedias"` - Dettaglio *string `xml:"Dettaglio"` + Dettaglio *string `xml:"Dettaglio"` + ProcessingImages bool + ProcessingVideos bool } type AnnunciParsed struct { @@ -218,19 +218,3 @@ type MediaToProcess struct { Url string Order int } - -type ProcessedMedia struct { - Codice string - Order int - Path string - ThumnailPath string - OGUrl string -} - -type UploadedMedia struct { - Codice string - Order int - MediaId uuid.UUID - ThumbId uuid.UUID - OGUrl string -} diff --git a/apps/backend/update.go b/apps/backend/update.go new file mode 100644 index 0000000..b4a36c7 --- /dev/null +++ b/apps/backend/update.go @@ -0,0 +1,359 @@ +package main + +import ( + "backend/config" + models "backend/gen/postgres/postgres/public/model" + "backend/parser" + "backend/queries" + "backend/typesdefs" + "backend/utils" + "fmt" + "log" + "slices" + "strings" + "time" +) + +type UpdaterConfig struct { + // Dry run flag to simulate the update process without making changes. + isDryRun bool + // Filter by codice filter for specific updates. If empty, all records are updated. + codFilter string + // Force media refresh flag to re-download and upload media + forceMediaRefresh bool +} + +type Updater struct { + isDryRun bool + codFilter string + forceMediaRefresh bool + caratterisiche typesdefs.ListaCaratteristiche + categorie []typesdefs.Categoria + imgPool []typesdefs.MediaToProcess + videoPool []typesdefs.MediaToProcess + annunci typesdefs.AnnunciParsed +} + +func NewUpdater(props UpdaterConfig) *Updater { + + u := Updater{ + isDryRun: props.isDryRun, + codFilter: props.codFilter, + forceMediaRefresh: props.forceMediaRefresh, + } + fmt.Printf("updater cfg: %+v\n", props) + u.init_Updater() + return &u +} + +func (u *Updater) init_Updater() { + var err error + u.caratterisiche, err = genCaratteristiche() + if err != nil { + log.Fatalf("Error generating carateristiche: %v", err) + } + + u.categorie, err = genCategorie() + if err != nil { + log.Fatalf("Error generating categorie: %v", err) + } +} + +func genCaratteristiche() (typesdefs.ListaCaratteristiche, error) { + var values typesdefs.ListaCaratteristiche + //https://www.miogest.com/apps/revo.aspx?tipo=schede + var result, err = utils.GetXMLFromFile[typesdefs.DefaultCaratteristiche]("./caratteristiche.xml") + if err != nil { + return values, err + + } + for _, tag := range result.Scheda { + var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml} + values.Caratteristiche = append(values.Caratteristiche, c) + } + return values, nil +} + +func genCategorie() ([]typesdefs.Categoria, error) { + var values []typesdefs.Categoria + //https://www.miogest.com/apps/revo.aspx?tipo=categorie + var result, err = utils.GetXMLFromFile[typesdefs.DefaultCategories]("./categorie.xml") + if err != nil { + return values, err + } + for _, elem := range result.Cat { + var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome} + values = append(values, c) + } + return values, nil +} + +func (u *Updater) Update() error { + var err error + err = u.parseAnnunci() + if err != nil { + return err + } + err = queries.AnnunciInsert(u.annunci, u.codFilter == "", u.isDryRun) + if err != nil { + return err + } + err = ProcessImagePool(&u.imgPool, u.isDryRun) + if err != nil { + return err + } + + err = ProcessVideoPool(&u.videoPool, u.isDryRun) + if err != nil { + return err + } + log.Println("Update completed") + + return nil +} + +func (u *Updater) parseAnnunci() error { + var err error + + xml, err := getAnnunciMiogest() + if err != nil { + return err + } + + // filter by codFilter if provided + if u.codFilter != "" { + idx := slices.IndexFunc(xml.Annuncio, func(a typesdefs.AnnuncioXML) bool { + return *a.Codice == u.codFilter + }) + if idx == -1 { + return fmt.Errorf("Codice Filtering: Annuncio con codice %s not found", u.codFilter) + } + xml.Annuncio = []typesdefs.AnnuncioXML{xml.Annuncio[idx]} + } + u.setupMediaWorkload(&xml) + + for i := range xml.Annuncio { + u.annunci.Annuncio = append(u.annunci.Annuncio, u.extractAnnuncioData(&xml.Annuncio[i])) + } + + return err +} + +func getAnnunciMiogest() (typesdefs.AnnunciXML, error) { + var result typesdefs.AnnunciXML + var err error + result, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml") + if err != nil { + return result, fmt.Errorf("Error fetching data from Miogest: %w", err) + } + if len(result.Annuncio) == 0 { + return result, fmt.Errorf("No annunci found in the XML data") + } + for i, d := range result.Annuncio { + sanified := strings.ReplaceAll(*d.Codice, " ", "") + result.Annuncio[i].Codice = &sanified + } + + return result, nil + +} + +func (u *Updater) setupMediaWorkload(xml *typesdefs.AnnunciXML) { + + if u.forceMediaRefresh && (config.Cfg.Images.Enabled || config.Cfg.Videos.Enabled) { + for i := range xml.Annuncio { + xml.Annuncio[i].ProcessingImages = config.Cfg.Images.Enabled + xml.Annuncio[i].ProcessingVideos = config.Cfg.Videos.Enabled + } + return + } + + db_media, err := queries.GetMediaFromDB() + if err != nil { + log.Fatalf("failed to get media from db: %v", err) + return + } + for i, a := range xml.Annuncio { + if a.Codice == nil { + continue + } + var in_db []models.MediaRefs + for _, v := range db_media { + if v.Codice == *a.Codice { + in_db = append(in_db, v) + } + } + + fmt.Printf("codice: %s, in_bd: %d\n", *a.Codice, len(in_db)) + + if len(in_db) == 0 { + // not found in db + if config.Cfg.Images.Enabled { + xml.Annuncio[i].ProcessingImages = true + } + if config.Cfg.Videos.Enabled { + xml.Annuncio[i].ProcessingVideos = true + } + continue + } + if config.Cfg.Images.Enabled { + + img_process := false + for _, fotoUrl := range *a.Foto { + found_img := false + found_thumb := false + + for _, ref := range in_db { + if found_img && found_thumb { + break + } + fmt.Printf("%s -> %s\n", fotoUrl, ref.OgURL) + if fotoUrl == ref.OgURL { + fmt.Println("match") + if ref.IsThumbnail { + found_thumb = true + continue + } + found_img = true + } + } + if !found_img || !found_thumb { + fmt.Printf("not found img %s\n", fotoUrl) + img_process = true + break + } + } + + xml.Annuncio[i].ProcessingImages = img_process + } + if config.Cfg.Videos.Enabled { + vid_process := false + for _, media := range a.AMMedias.AMVideo { + + if media.Url == nil { + continue + } + if !strings.Contains(*media.Url, "video.miogest") { + continue + } + + found_vid := false + found_thumb := false + for _, ref := range in_db { + if found_vid && found_thumb { + break + } + if *media.Url == ref.OgURL { + if ref.IsThumbnail { + found_thumb = true + continue + } + found_vid = true + } + } + if !found_vid || !found_thumb { + + vid_process = true + break + } + + } + xml.Annuncio[i].ProcessingImages = vid_process + } + } + + if u.codFilter != "" { + for i := range xml.Annuncio { + + if config.Cfg.Images.Enabled { + xml.Annuncio[i].ProcessingImages = true + } + if config.Cfg.Videos.Enabled { + xml.Annuncio[i].ProcessingImages = true + } + } + } + +} + +func (u *Updater) extractAnnuncioData(annuncio *typesdefs.AnnuncioXML) typesdefs.AnnuncioParsed { + tmp := typesdefs.AnnuncioParsed{} + + tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice)) + processLocatore(&tmp, annuncio) + processIndirizzo(&tmp, annuncio) + + tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2) + tmp.Consegna = parser.ParseConsegna(annuncio.Consegna) + processCategorie(&tmp, annuncio, &u.categorie) + processDatiImmobile(&tmp, annuncio) + tmp.Stato = annuncio.Stato + + tmp.Homepage = boolUpd(false) + if annuncio.HomePage != nil { + if *annuncio.HomePage == "si" { + + tmp.Homepage = boolUpd(true) + } + } + layout := "20060102150405" + if annuncio.Creato != nil { + t, err := time.Parse(layout, *annuncio.Creato) + if err != nil { + tmp.Creato_il = nil + } else { + tmp.Creato_il = &t + } + } + if annuncio.Modifica != nil { + t, err := time.Parse(layout, *annuncio.Modifica) + if err != nil { + tmp.Modificato_il = nil + } else { + tmp.Modificato_il = &t + } + } + tmp.Titolo_it = annuncio.Titolo + tmp.Titolo_en = annuncio.TitoloEn + tmp.Desc_it = annuncio.Descrizione + tmp.Desc_en = annuncio.DescrizioneEn + + if annuncio.Dettaglio != nil { + disp, parm, pers, err := parser.ParseDettaglio(*annuncio.Dettaglio, tmp.Consegna) + if err == nil { + tmp.Disponibile_da = disp + tmp.Permanenza = parm + tmp.Persone = pers + + } + } + + tmp.Web = boolUpd(true) + + if annuncio.ProcessingVideos { + AddToVideoProcessing(&tmp, annuncio, &u.videoPool) + } + + if annuncio.Video != nil { + ext_videos := []string{} + for _, v := range *annuncio.Video { + if strings.Contains(v, "youtu") { + ext_videos = append(ext_videos, v) + } + tmp.External_videos = &ext_videos + } + } else { + tmp.External_videos = &[]string{} + } + + if annuncio.Accessori != nil { + tmp.Accessori = annuncio.Accessori + } + tmpcaratt := parseCaratteristiche(annuncio, &u.caratterisiche) + tmp.Caratteristiche = &tmpcaratt + + if annuncio.ProcessingImages { + AddToImageProcessing(&tmp, annuncio, &u.imgPool) + } + return tmp +} diff --git a/apps/backend/video.go b/apps/backend/video.go index c826f41..a9999dc 100644 --- a/apps/backend/video.go +++ b/apps/backend/video.go @@ -2,6 +2,7 @@ package main import ( "backend/config" + models "backend/gen/postgres/postgres/public/model" "backend/queries" "backend/typesdefs" "backend/utils" @@ -17,7 +18,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/nfnt/resize" "github.com/nickalie/go-webpbin" ) @@ -45,25 +45,25 @@ func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Ann } } -func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.UploadedMedia, error) { +func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess, isDryRun bool) error { if len(*toProcess) == 0 { log.Println("No new videos to process.") - return []typesdefs.UploadedMedia{}, nil + return nil } - var ( - ProcessedVideos []typesdefs.ProcessedMedia - UploadedVideos []typesdefs.UploadedMedia - mu sync.Mutex - wg sync.WaitGroup - ) + var wg sync.WaitGroup + totalToProcess := len(*toProcess) log.Printf("Processing %d videos with concurrency limit %d", totalToProcess, config.Cfg.Videos.ConcurrentLimit) sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit) // clear videos folder - err := clearVideoFolder() - if err != nil { - log.Fatalf("Failed to initialize image processing: %v", err) + if isDryRun { + log.Println("Dry run. Clearing videos folder.") + } else { + err := clearVideoFolder() + if err != nil { + log.Fatalf("Failed to initialize image processing: %v", err) + } } // process videos @@ -73,71 +73,73 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload go func(video typesdefs.MediaToProcess) { defer wg.Done() defer func() { <-sem }() // release semaphore + if isDryRun { + log.Printf("Dry run. video processing: %s_%d", video.Codice, video.Order) + return + } + + log.Printf("%s_%d: elaborating", video.Codice, video.Order) videoPath := filepath.Join(config.Cfg.Videos.Path, fmt.Sprintf("%s_%d%s", video.Codice, video.Order, filepath.Ext(video.Url))) + + log.Printf("%s_%d: download", video.Codice, video.Order) err := downloadVideo(video.Url, videoPath) if err != nil { log.Printf("Failed to download video %s: %v", video.Url, err) return // Exit goroutine on download failure } - // Transcode and generate thumbnail + + log.Printf("%s_%d: conversion", video.Codice, video.Order) mp4Path, thumbnailPath, err := VideoConversion(videoPath) if err != nil { log.Printf("Failed to convert video %s: %v", videoPath, err) return } - mu.Lock() - ProcessedVideos = append(ProcessedVideos, typesdefs.ProcessedMedia{ - Codice: video.Codice, - Order: video.Order, - Path: mp4Path, - ThumnailPath: thumbnailPath, - OGUrl: video.Url, - }) - mu.Unlock() - }(video) - } - wg.Wait() - log.Printf("Video processing completed for %d videos. Starting upload...", len(ProcessedVideos)) - fmt.Println() - totalToUpload := len(ProcessedVideos) - log.Printf("Starting upload of %d videos with concurrency limit %d...", totalToUpload, config.Cfg.Videos.ConcurrentLimit) - uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit) - var uploadWg sync.WaitGroup - for _, video := range ProcessedVideos { - uploadSem <- struct{}{} // acquire semaphore - uploadWg.Add(1) - go func(video typesdefs.ProcessedMedia) { - defer uploadWg.Done() - defer func() { <-uploadSem }() // release semaphore - - videoId, err := queries.UploadFile(video.Path, "video/mp4", "videos") + log.Printf("%s_%d: upload", video.Codice, video.Order) + videoId, err := queries.UploadFile(mp4Path, "video/mp4", "videos") if err != nil { log.Printf("failed to upload video: %v", err) return } - thumbId, err := queries.UploadFile(video.ThumnailPath, "image/webp", "videos") + thumbId, err := queries.UploadFile(thumbnailPath, "image/webp", "videos") if err != nil { log.Printf("failed to upload thumbnail: %v", err) return } - mu.Lock() - UploadedVideos = append(UploadedVideos, typesdefs.UploadedMedia{ - Codice: video.Codice, - Order: video.Order, - MediaId: videoId, - ThumbId: thumbId, - OGUrl: video.OGUrl, - }) - mu.Unlock() + log.Printf("%s_%d: videos_refs upd", video.Codice, video.Order) + err = queries.SetMediaToDB(models.MediaRefs{ + Codice: video.Codice, + Ordine: int32(video.Order), + OgURL: video.Url, + MediaType: "video", + StorageID: videoId, + IsThumbnail: false, + }) + if err != nil { + log.Printf("failed to set media to db: %v", err) + return + + } + err = queries.SetMediaToDB(models.MediaRefs{ + Codice: video.Codice, + Ordine: int32(video.Order), + OgURL: video.Url, + MediaType: "video", + StorageID: thumbId, + IsThumbnail: true, + }) + if err != nil { + log.Printf("failed to set media to db: %v", err) + return + } + + log.Printf("%s_%d: done", video.Codice, video.Order) }(video) } - uploadWg.Wait() - log.Printf("Upload completed for %d videos.", len(UploadedVideos)) - fmt.Println() - - return UploadedVideos, nil + wg.Wait() + log.Println("Uploaded all videos.") + return nil } func clearVideoFolder() error { @@ -323,150 +325,3 @@ func extractThumbnail(input, output string) error { return nil } - -type videRef struct { - Og_url string - Video uuid.UUID - Thumb uuid.UUID -} - -func CodiciToProcessVideos(annunci *typesdefs.AnnunciXML) ([]string, error) { - currentBucketState, err := queries.GetStorageByTag("videos") - if err != nil { - return nil, fmt.Errorf("failed to list bucket: %w", err) - } - - //make a list of bucket ids - bucketIdsMap := make(map[uuid.UUID]bool) - for _, item := range currentBucketState { - bucketIdsMap[item.ID] = true - } - - db_videos, err := queries.GetVideosFromDB() - if err != nil { - return nil, err - } - - dbVideosMap := make(map[string][]videRef) // Convert slice to map for faster lookup - for _, ref := range db_videos { - dbVideosMap[ref.Codice] = append(dbVideosMap[ref.Codice], videRef{ - Og_url: ref.OgURL, - Video: ref.Video, - Thumb: ref.Thumb, - }) - } - - var codiciToProcess []string - for _, annuncio := range annunci.Annuncio { - - if annuncio.Codice != nil && annuncio.AMMedias != nil { - mediaUrls := []string{} - for _, media := range annuncio.AMMedias.AMVideo { - if media.Url != nil { - if strings.Contains(*media.Url, "video.miogest") { - mediaUrls = append(mediaUrls, *media.Url) - } - } - } - if len(mediaUrls) == 0 { - continue - } - - refVideos, existsInDb := dbVideosMap[*annuncio.Codice] - if !existsInDb { - codiciToProcess = append(codiciToProcess, *annuncio.Codice) - continue - } - needProcess := false - - for _, media := range annuncio.AMMedias.AMVideo { - if media.Url == nil { - continue - } - if !strings.Contains(*media.Url, "video.miogest") { - continue - } - - found := false - for _, ref := range refVideos { - if *media.Url == ref.Og_url { - if bucketIdsMap[ref.Video] && bucketIdsMap[ref.Thumb] { - found = true - break - } - } - } - if !found { - needProcess = true - break - } - - } - if needProcess { - codiciToProcess = append(codiciToProcess, *annuncio.Codice) - } - } - } - - return codiciToProcess, nil - -} - -func ForceClearVideos(codici []string) error { - var idsToDelete uuid.UUIDs - for _, codice := range codici { - refs, err := queries.GetCodiceVideosFromDB(codice) - if err != nil { - return fmt.Errorf("failed to get videos by codice %s: %w", codice, err) - } - for _, ref := range refs { - idsToDelete = append(idsToDelete, ref.Video) - idsToDelete = append(idsToDelete, ref.Thumb) - } - - } - err := queries.DeleteMultFile(idsToDelete) - if err != nil { - return fmt.Errorf("failed to delete videos from storage: %w", err) - } - for _, codice := range codici { - err := queries.ClearVideosRefsByCodice(codice) - if err != nil { - return fmt.Errorf("failed to clear videos by codice %s: %w", codice, err) - } - } - return nil -} - -// ReconcileVideos removes stranded videos (in storage but not in DB) -func ReconcileVideos() error { - currentBucketState, err := queries.GetStorageByTag("videos") - if err != nil { - return fmt.Errorf("failed to list bucket: %w", err) - } - //make a list of bucket ids - bucketIdsMap := make(map[uuid.UUID]bool) - for _, item := range currentBucketState { - bucketIdsMap[item.ID] = true - } - db_videos, err := queries.GetVideosFromDB() - if err != nil { - return fmt.Errorf("failed to get videos from DB: %w", err) - } - var strandedIds uuid.UUIDs - for _, ref := range db_videos { - if !bucketIdsMap[ref.Video] { - strandedIds = append(strandedIds, ref.Video) - } - if !bucketIdsMap[ref.Thumb] { - strandedIds = append(strandedIds, ref.Thumb) - } - } - if len(strandedIds) > 0 { - err := queries.DeleteMultFile(strandedIds) - if err != nil { - return fmt.Errorf("failed to delete stranded videos from storage: %w", err) - } - } - return nil -} diff --git a/apps/db/migrations/57_mediatable.up.sql b/apps/db/migrations/57_mediatable.up.sql new file mode 100644 index 0000000..717e690 --- /dev/null +++ b/apps/db/migrations/57_mediatable.up.sql @@ -0,0 +1,68 @@ +-- Media Refs +CREATE TABLE IF NOT EXISTS public.media_refs ( + codice TEXT NOT NULL, + ordine INTEGER NOT NULL, + storage_id UUID NOT NULL, + og_url TEXT NOT NULL, + media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video')), + is_thumbnail BOOLEAN NOT NULL DEFAULT FALSE +); +-- Indexes +CREATE INDEX IF NOT EXISTS idx_media_refs_codice ON public.media_refs (codice); +CREATE INDEX IF NOT EXISTS idx_media_refs_storage_id ON public.media_refs (storage_id); +-- Unique Codice and Ordine +DO $$ BEGIN IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'media_refs_unique' +) THEN +ALTER TABLE public.media_refs +ADD CONSTRAINT media_refs_unique UNIQUE (codice, ordine, is_thumbnail, media_type); +END IF; +END $$; + +-- Foreign Key to Annunci +DO $$ BEGIN IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'media_refs_annuncio' +) THEN +ALTER TABLE public.media_refs +ADD CONSTRAINT media_refs_annuncio FOREIGN KEY (codice) REFERENCES public.annunci (codice) ON UPDATE CASCADE ON DELETE CASCADE; +END IF; +END $$; + +--Foreign Key to Storage +DO $$ BEGIN IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'media_refs_storage' +) THEN +ALTER TABLE public.media_refs +ADD CONSTRAINT media_refs_storage FOREIGN KEY (storage_id) REFERENCES public.storage (id) ON UPDATE CASCADE ON DELETE CASCADE; +END IF; +END $$; + + +DO $$ +BEGIN + IF EXISTS ( + SELECT FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'media_refs' + ) THEN + EXECUTE ' + DELETE FROM public.storage s + WHERE s.tag IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM public.media_refs m + WHERE s.id = m.storage_id + ); + '; + END IF; +END $$; + +-- Drop Tables if they exist +DROP TABLE IF EXISTS public.images_refs; +DROP TABLE IF EXISTS public.videos_refs; \ No newline at end of file diff --git a/apps/infoalloggi/src/components/annuncio_card.tsx b/apps/infoalloggi/src/components/annuncio_card.tsx index e3e5199..07043be 100644 --- a/apps/infoalloggi/src/components/annuncio_card.tsx +++ b/apps/infoalloggi/src/components/annuncio_card.tsx @@ -30,9 +30,11 @@ import { camereTesti, handleConsegna } from "~/lib/annuncio_details"; import { getStorageUrl } from "~/lib/storage_utils"; import { cn, formatCurrency } from "~/lib/utils"; import { useTranslation } from "~/providers/I18nProvider"; -import type { ImagesRefs } from "~/schemas/public/ImagesRefs"; -import type { VideosRefs } from "~/schemas/public/VideosRefs"; -import type { AnnuncioRicerca } from "~/server/controllers/annunci.controller"; +import type { + AnnuncioRicerca, + ImgMedia, + VideoMedia, +} from "~/server/controllers/annunci.controller"; import { Badge } from "./ui/badge"; import { Skeleton } from "./ui/skeleton"; import { VideoPlayer } from "./videoPlayer"; @@ -217,13 +219,17 @@ export const CardAnnuncio = ({ {t.card.alt_immagine} 0 && images[0] ? ( {t.card.alt_immagine}[]; - videos: Pick[]; + immagini: ImgMedia[]; + videos: VideoMedia[]; updated_at: Date | null; }) => { const [openModal, setOpenModal] = useState(false); @@ -484,13 +498,17 @@ export const CarouselAnnuncio = ({ > idx @@ -561,13 +579,17 @@ export const CarouselAnnuncio = ({ > { /> - removeMedia({ type: "img", id: img.img }) - } + onConfirm={async () => removeMedia(img.img)} title="Rimuovi media" >