Marco/infoalloggi-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 6 Wiki Activity Actions

mediarefs

Browse source
This commit is contained in:
Marco Pedone 2026-05-22 15:49:28 +02:00
parent bd13a81a9b
commit 364f4a77b3
34 changed files with 1154 additions and 1673 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
apps/backend/gen/postgres/postgres/public/model/images_refs.go → apps/backend/gen/postgres/postgres/public/model/media_refs.go
View file

@ -11,10 +11,11 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
) )
type ImagesRefs struct { type MediaRefs struct {
Codice string Codice string
Ordine int32 Ordine int32
Img uuid.UUID StorageID uuid.UUID
Thumb uuid.UUID OgURL string
OgURL string MediaType string
IsThumbnail bool
} }

20
apps/backend/gen/postgres/postgres/public/model/videos_refs.go
View file

@ -1,20 +0,0 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package model
import (
"github.com/google/uuid"
)
type VideosRefs struct {
Codice string
Ordine int32
Video uuid.UUID
Thumb uuid.UUID
OgURL string
}

90
apps/backend/gen/postgres/postgres/public/table/images_refs.go
View file

@ -1,90 +0,0 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var ImagesRefs = newImagesRefsTable("public", "images_refs", "")
type imagesRefsTable struct {
postgres.Table
// Columns
Codice postgres.ColumnString
Ordine postgres.ColumnInteger
Img postgres.ColumnString
Thumb postgres.ColumnString
OgURL postgres.ColumnString
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type ImagesRefsTable struct {
imagesRefsTable
EXCLUDED imagesRefsTable
}
// AS creates new ImagesRefsTable with assigned alias
func (a ImagesRefsTable) AS(alias string) *ImagesRefsTable {
return newImagesRefsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new ImagesRefsTable with assigned schema name
func (a ImagesRefsTable) FromSchema(schemaName string) *ImagesRefsTable {
return newImagesRefsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new ImagesRefsTable with assigned table prefix
func (a ImagesRefsTable) WithPrefix(prefix string) *ImagesRefsTable {
return newImagesRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new ImagesRefsTable with assigned table suffix
func (a ImagesRefsTable) WithSuffix(suffix string) *ImagesRefsTable {
return newImagesRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newImagesRefsTable(schemaName, tableName, alias string) *ImagesRefsTable {
return &ImagesRefsTable{
imagesRefsTable: newImagesRefsTableImpl(schemaName, tableName, alias),
EXCLUDED: newImagesRefsTableImpl("", "excluded", ""),
}
}
func newImagesRefsTableImpl(schemaName, tableName, alias string) imagesRefsTable {
var (
CodiceColumn = postgres.StringColumn("codice")
OrdineColumn = postgres.IntegerColumn("ordine")
ImgColumn = postgres.StringColumn("img")
ThumbColumn = postgres.StringColumn("thumb")
OgURLColumn = postgres.StringColumn("og_url")
allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, ImgColumn, ThumbColumn, OgURLColumn}
mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, ImgColumn, ThumbColumn, OgURLColumn}
defaultColumns = postgres.ColumnList{}
)
return imagesRefsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
Codice: CodiceColumn,
Ordine: OrdineColumn,
Img: ImgColumn,
Thumb: ThumbColumn,
OgURL: OgURLColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}

93
apps/backend/gen/postgres/postgres/public/table/media_refs.go Normal file
View file

@ -0,0 +1,93 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var MediaRefs = newMediaRefsTable("public", "media_refs", "")
type mediaRefsTable struct {
postgres.Table
// Columns
Codice postgres.ColumnString
Ordine postgres.ColumnInteger
StorageID postgres.ColumnString
OgURL postgres.ColumnString
MediaType postgres.ColumnString
IsThumbnail postgres.ColumnBool
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type MediaRefsTable struct {
mediaRefsTable
EXCLUDED mediaRefsTable
}
// AS creates new MediaRefsTable with assigned alias
func (a MediaRefsTable) AS(alias string) *MediaRefsTable {
return newMediaRefsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new MediaRefsTable with assigned schema name
func (a MediaRefsTable) FromSchema(schemaName string) *MediaRefsTable {
return newMediaRefsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new MediaRefsTable with assigned table prefix
func (a MediaRefsTable) WithPrefix(prefix string) *MediaRefsTable {
return newMediaRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new MediaRefsTable with assigned table suffix
func (a MediaRefsTable) WithSuffix(suffix string) *MediaRefsTable {
return newMediaRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newMediaRefsTable(schemaName, tableName, alias string) *MediaRefsTable {
return &MediaRefsTable{
mediaRefsTable: newMediaRefsTableImpl(schemaName, tableName, alias),
EXCLUDED: newMediaRefsTableImpl("", "excluded", ""),
}
}
func newMediaRefsTableImpl(schemaName, tableName, alias string) mediaRefsTable {
var (
CodiceColumn = postgres.StringColumn("codice")
OrdineColumn = postgres.IntegerColumn("ordine")
StorageIDColumn = postgres.StringColumn("storage_id")
OgURLColumn = postgres.StringColumn("og_url")
MediaTypeColumn = postgres.StringColumn("media_type")
IsThumbnailColumn = postgres.BoolColumn("is_thumbnail")
allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, StorageIDColumn, OgURLColumn, MediaTypeColumn, IsThumbnailColumn}
mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, StorageIDColumn, OgURLColumn, MediaTypeColumn, IsThumbnailColumn}
defaultColumns = postgres.ColumnList{IsThumbnailColumn}
)
return mediaRefsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
Codice: CodiceColumn,
Ordine: OrdineColumn,
StorageID: StorageIDColumn,
OgURL: OgURLColumn,
MediaType: MediaTypeColumn,
IsThumbnail: IsThumbnailColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}

3
apps/backend/gen/postgres/postgres/public/table/table_use_schema.go
View file

@ -11,7 +11,6 @@ package table
// this method only once at the beginning of the program. // this method only once at the beginning of the program.
func UseSchema(schema string) { func UseSchema(schema string) {
Annunci = Annunci.FromSchema(schema) Annunci = Annunci.FromSchema(schema)
ImagesRefs = ImagesRefs.FromSchema(schema) MediaRefs = MediaRefs.FromSchema(schema)
Storage = Storage.FromSchema(schema) Storage = Storage.FromSchema(schema)
VideosRefs = VideosRefs.FromSchema(schema)
} }

90
apps/backend/gen/postgres/postgres/public/table/videos_refs.go
View file

@ -1,90 +0,0 @@
//
// Code generated by go-jet DO NOT EDIT.
//
// WARNING: Changes to this file may cause incorrect behavior
// and will be lost if the code is regenerated
//
package table
import (
"github.com/go-jet/jet/v2/postgres"
)
var VideosRefs = newVideosRefsTable("public", "videos_refs", "")
type videosRefsTable struct {
postgres.Table
// Columns
Codice postgres.ColumnString
Ordine postgres.ColumnInteger
Video postgres.ColumnString
Thumb postgres.ColumnString
OgURL postgres.ColumnString
AllColumns postgres.ColumnList
MutableColumns postgres.ColumnList
DefaultColumns postgres.ColumnList
}
type VideosRefsTable struct {
videosRefsTable
EXCLUDED videosRefsTable
}
// AS creates new VideosRefsTable with assigned alias
func (a VideosRefsTable) AS(alias string) *VideosRefsTable {
return newVideosRefsTable(a.SchemaName(), a.TableName(), alias)
}
// Schema creates new VideosRefsTable with assigned schema name
func (a VideosRefsTable) FromSchema(schemaName string) *VideosRefsTable {
return newVideosRefsTable(schemaName, a.TableName(), a.Alias())
}
// WithPrefix creates new VideosRefsTable with assigned table prefix
func (a VideosRefsTable) WithPrefix(prefix string) *VideosRefsTable {
return newVideosRefsTable(a.SchemaName(), prefix+a.TableName(), a.TableName())
}
// WithSuffix creates new VideosRefsTable with assigned table suffix
func (a VideosRefsTable) WithSuffix(suffix string) *VideosRefsTable {
return newVideosRefsTable(a.SchemaName(), a.TableName()+suffix, a.TableName())
}
func newVideosRefsTable(schemaName, tableName, alias string) *VideosRefsTable {
return &VideosRefsTable{
videosRefsTable: newVideosRefsTableImpl(schemaName, tableName, alias),
EXCLUDED: newVideosRefsTableImpl("", "excluded", ""),
}
}
func newVideosRefsTableImpl(schemaName, tableName, alias string) videosRefsTable {
var (
CodiceColumn = postgres.StringColumn("codice")
OrdineColumn = postgres.IntegerColumn("ordine")
VideoColumn = postgres.StringColumn("video")
ThumbColumn = postgres.StringColumn("thumb")
OgURLColumn = postgres.StringColumn("og_url")
allColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, VideoColumn, ThumbColumn, OgURLColumn}
mutableColumns = postgres.ColumnList{CodiceColumn, OrdineColumn, VideoColumn, ThumbColumn, OgURLColumn}
defaultColumns = postgres.ColumnList{}
)
return videosRefsTable{
Table: postgres.NewTable(schemaName, tableName, alias, allColumns...),
//Columns
Codice: CodiceColumn,
Ordine: OrdineColumn,
Video: VideoColumn,
Thumb: ThumbColumn,
OgURL: OgURLColumn,
AllColumns: allColumns,
MutableColumns: mutableColumns,
DefaultColumns: defaultColumns,
}
}

255
apps/backend/images.go
View file

@ -2,6 +2,7 @@ package main
import ( import (
"backend/config" "backend/config"
models "backend/gen/postgres/postgres/public/model"
"backend/queries" "backend/queries"
"backend/typesdefs" "backend/typesdefs"
"backend/utils" "backend/utils"
@ -18,7 +19,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/google/uuid"
"github.com/nfnt/resize" "github.com/nfnt/resize"
"github.com/nickalie/go-webpbin" "github.com/nickalie/go-webpbin"
) )
@ -37,25 +37,26 @@ func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Ann
} }
} }
func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.UploadedMedia, error) { func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess, isDryRun bool) error {
if len(*toProcess) == 0 { if len(*toProcess) == 0 {
log.Println("No new images to process.") log.Println("No new images to process.")
return []typesdefs.UploadedMedia{}, nil return nil
} }
var ( var wg sync.WaitGroup
ProcessedImages []typesdefs.ProcessedMedia
UploadedImages []typesdefs.UploadedMedia
mu sync.Mutex
wg sync.WaitGroup
)
sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
totalImages := len(*toProcess) totalImages := len(*toProcess)
log.Printf("Starting processing of %d images with concurrency limit %d...", totalImages, config.Cfg.Images.ConcurrentLimit) log.Printf("Starting processing of %d images with concurrency limit %d...", totalImages, config.Cfg.Images.ConcurrentLimit)
// clear images folder // clear images folder
err := clearImageFolder() if isDryRun {
if err != nil { log.Println("Dry run. Clearing images folder...")
log.Fatalf("Failed to initialize image processing: %v", err) } else {
err := clearImageFolder()
if err != nil {
log.Fatalf("Failed to initialize image processing: %v", err)
}
} }
// process images // process images
@ -65,69 +66,73 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
go func(img typesdefs.MediaToProcess) { go func(img typesdefs.MediaToProcess) {
defer wg.Done() defer wg.Done()
defer func() { <-sem }() // release semaphore defer func() { <-sem }() // release semaphore
if isDryRun {
log.Printf("Dry run: image processing: %s_%d", img.Codice, img.Order)
return
}
log.Printf("%s_%d: elaborating", img.Codice, img.Order)
imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url))) imgPath := filepath.Join(config.Cfg.Images.Path, fmt.Sprintf("%s_%d%s", img.Codice, img.Order, filepath.Ext(img.Url)))
log.Printf("%s_%d: downloading", img.Codice, img.Order)
err := downloadImage(img.Url, imgPath) err := downloadImage(img.Url, imgPath)
if err != nil { if err != nil {
log.Printf("Failed to download image %s: %v", img.Url, err) log.Printf("Failed to download image %s: %v", img.Url, err)
return // Exit goroutine on download failure return // Exit goroutine on download failure
} }
log.Printf("%s_%d: conversion", img.Codice, img.Order)
webpPath, thumbnailwebpPath, err := Conversion(imgPath) webpPath, thumbnailwebpPath, err := Conversion(imgPath)
if err != nil { if err != nil {
log.Fatalf("Failed to convert image: %v", err) log.Fatalf("Failed to convert image: %v", err)
} }
mu.Lock()
ProcessedImages = append(ProcessedImages, typesdefs.ProcessedMedia{ log.Printf("%s_%d: upload", img.Codice, img.Order)
Codice: img.Codice, imgId, err := queries.UploadFile(webpPath, "image/webp", "images")
Order: img.Order, if err != nil {
Path: webpPath, log.Printf("failed to upload image: %v", err)
ThumnailPath: thumbnailwebpPath, return
OGUrl: img.Url, }
thumbId, err := queries.UploadFile(thumbnailwebpPath, "image/webp", "images")
if err != nil {
log.Printf("failed to upload thumbnail: %v", err)
return
}
log.Printf("%s_%d: images_refs upd", img.Codice, img.Order)
err = queries.SetMediaToDB(models.MediaRefs{
Codice: img.Codice,
Ordine: int32(img.Order),
OgURL: img.Url,
MediaType: "image",
StorageID: imgId,
IsThumbnail: false,
}) })
mu.Unlock() if err != nil {
log.Printf("failed to set media to db: %v", err)
return
}
err = queries.SetMediaToDB(models.MediaRefs{
Codice: img.Codice,
Ordine: int32(img.Order),
OgURL: img.Url,
MediaType: "image",
StorageID: thumbId,
IsThumbnail: true,
})
if err != nil {
log.Printf("failed to set media to db: %v", err)
return
}
log.Printf("%s_%d: done", img.Codice, img.Order)
}(img) }(img)
} }
wg.Wait() wg.Wait()
log.Printf("Processed all images") log.Printf("Processed all images")
fmt.Println() fmt.Println()
totalToUpload := len(ProcessedImages) return nil
log.Printf("Starting upload of %d images with concurrency limit %d...", totalToUpload, config.Cfg.Images.ConcurrentLimit)
uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
var uploadWg sync.WaitGroup
for _, img := range ProcessedImages {
uploadSem <- struct{}{} // acquire semaphore
uploadWg.Add(1)
go func(img typesdefs.ProcessedMedia) {
defer uploadWg.Done()
defer func() { <-uploadSem }() // release semaphore
imgId, err := queries.UploadFile(img.Path, "image/webp", "images")
if err != nil {
log.Printf("failed to upload image: %v", err)
return
}
thumbId, err := queries.UploadFile(img.ThumnailPath, "image/webp", "images")
if err != nil {
log.Printf("failed to upload thumbnail: %v", err)
return
}
mu.Lock()
UploadedImages = append(UploadedImages, typesdefs.UploadedMedia{
Codice: img.Codice,
Order: img.Order,
MediaId: imgId,
ThumbId: thumbId,
OGUrl: img.OGUrl,
})
mu.Unlock()
}(img)
}
uploadWg.Wait()
log.Printf("Uploaded all images")
fmt.Println()
return UploadedImages, nil
} }
func clearImageFolder() error { func clearImageFolder() error {
@ -271,137 +276,3 @@ func thumbnail_creation(img image.Image, output string, outputdir string) error
} }
return nil return nil
} }
type imgRef struct {
Og_url string
Img uuid.UUID
Thumb uuid.UUID
}
func CodiciToProcessImages(annunci *typesdefs.AnnunciXML) ([]string, error) {
currentBucketState, err := queries.GetStorageByTag("images")
if err != nil {
return nil, fmt.Errorf("failed to list bucket: %w", err)
}
//make a list of bucket ids
bucketIdsMap := make(map[uuid.UUID]bool)
for _, item := range currentBucketState {
bucketIdsMap[item.ID] = true
}
db_images, err := queries.GetImagesFromDB()
if err != nil {
return nil, err
}
dbImagesMap := make(map[string][]imgRef)
for _, ref := range db_images {
dbImagesMap[ref.Codice] = append(dbImagesMap[ref.Codice], imgRef{
Og_url: ref.OgURL,
Img: ref.Img,
Thumb: ref.Thumb,
})
}
var codiciToProcess []string
for _, annuncio := range annunci.Annuncio {
if annuncio.Codice != nil && annuncio.Foto != nil {
// Check if codice exists in DB
refFotos, existsInDb := dbImagesMap[*annuncio.Codice]
if !existsInDb {
codiciToProcess = append(codiciToProcess, *annuncio.Codice)
continue
}
needProcess := false
for _, fotoUrl := range *annuncio.Foto {
found := false
// Check if the fotoUrl exists in the DB refs
for _, ref := range refFotos {
if fotoUrl == ref.Og_url {
// check if the corresponding img and thumb are in the bucket
if bucketIdsMap[ref.Img] && bucketIdsMap[ref.Thumb] {
found = true
break
}
}
}
if !found {
needProcess = true
break
}
}
if needProcess {
codiciToProcess = append(codiciToProcess, *annuncio.Codice)
}
}
}
return codiciToProcess, nil
}
func ForceClearImages(codici []string) error {
var idsToDelete uuid.UUIDs
for _, codice := range codici {
refs, err := queries.GetCodiceImagesFromDB(codice)
if err != nil {
return fmt.Errorf("failed to get images by codice %s: %w", codice, err)
}
for _, ref := range refs {
idsToDelete = append(idsToDelete, ref.Img)
idsToDelete = append(idsToDelete, ref.Thumb)
}
}
err := queries.DeleteMultFile(idsToDelete)
if err != nil {
return fmt.Errorf("failed to delete images from storage: %w", err)
}
for _, codice := range codici {
err := queries.ClearImagesRefsByCodice(codice)
if err != nil {
return fmt.Errorf("failed to clear images by codice %s: %w", codice, err)
}
}
return nil
}
// Removes stranded images (in storage but not in DB)
func ReconcileImages() error {
currentBucketState, err := queries.GetStorageByTag("images")
if err != nil {
return fmt.Errorf("failed to list bucket: %w", err)
}
//make a list of bucket ids
bucketIdsMap := make(map[uuid.UUID]bool)
for _, item := range currentBucketState {
bucketIdsMap[item.ID] = true
}
db_images, err := queries.GetImagesFromDB()
if err != nil {
return fmt.Errorf("failed to get images from DB: %w", err)
}
knownStorageIds := make(map[uuid.UUID]bool)
for _, ref := range db_images {
knownStorageIds[ref.Img] = true
knownStorageIds[ref.Thumb] = true
}
// reconcile: delete storage files not referenced in DB
var strandedIds uuid.UUIDs
for _, item := range currentBucketState {
if !knownStorageIds[item.ID] {
strandedIds = append(strandedIds, item.ID)
}
}
if len(strandedIds) > 0 {
log.Printf("Reconciliation: found %d stranded files, deleting...", len(strandedIds))
if err := queries.DeleteMultFile(strandedIds); err != nil {
log.Printf("Warning: failed to delete stranded files: %v", err) // non-fatal
}
} else {
log.Println("Reconciliation: no stranded files found.")
}
return nil
}

2
apps/backend/jet.sh
View file

@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,images_refs,videos_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum jet -dsn=postgresql://postgres:rootpost@localhost:5433/postgres?sslmode=disable -schema=public -path=./gen/postgres -tables=storage,annunci,media_refs -ignore-enums=bantype,genericstatusenum,ordertypeenum,paymentstatusenum,statusconfermaenum,tipologiaposizioneenum

514
apps/backend/miogest_handler.go
View file

@ -1,514 +0,0 @@
package main
import (
"backend/config"
"backend/parser"
"backend/typesdefs"
"backend/utils"
"fmt"
"log"
"reflect"
"regexp"
"slices"
"strconv"
"strings"
"time"
)
type MiogestHandler struct {
Caratterisiche typesdefs.ListaCaratteristiche
Categorie []typesdefs.Categoria
AnnunciXML typesdefs.AnnunciXML
AnnunciParsed typesdefs.AnnunciParsed
annunci_cache_expires time.Time
vars_cache_expires time.Time
}
func NewMiogestHandler() *MiogestHandler {
return &MiogestHandler{
Caratterisiche: typesdefs.ListaCaratteristiche{},
Categorie: nil,
AnnunciXML: typesdefs.AnnunciXML{},
AnnunciParsed: typesdefs.AnnunciParsed{},
annunci_cache_expires: time.Now(),
vars_cache_expires: time.Now(),
}
}
// ANNUNCI FROM MIOGEST
func (m *MiogestHandler) GetAnnunci() (typesdefs.AnnunciXML, error) {
if len(m.AnnunciXML.Annuncio) == 0 || time.Now().After(m.annunci_cache_expires) {
err := m.GetAnnunciFromMiogest()
if err != nil {
return typesdefs.AnnunciXML{}, err
}
}
return m.AnnunciXML, nil
}
func (m *MiogestHandler) GetAnnunciFromMiogest() error {
maxRetries := 3
baseDelay := 10 * time.Second
for attempt := range maxRetries {
var err error
m.AnnunciXML, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml")
if err == nil {
// Success - validate we got data
if len(m.AnnunciXML.Annuncio) > 0 {
m.annunci_cache_expires = time.Now().Add(1 * time.Hour)
return nil
}
log.Printf("Warning: Successfully fetched XML but got 0 annunci")
}
// Log the specific error type
if err != nil {
log.Printf("Failed to get annunci (attempt %d/%d): %v", attempt+1, maxRetries, err)
}
// Failed - check if we should retry
if attempt < maxRetries-1 {
delay := baseDelay * time.Duration(1<<attempt) // Exponential backoff
log.Printf("Failed to get annunci (attempt %d/%d): %v. Retrying in %v...",
attempt+1, maxRetries, err, delay)
time.Sleep(delay)
} else {
log.Printf("Failed to get annunci after %d attempts: %v", maxRetries, err)
return fmt.Errorf("failed to get annunci after %d attempts: %w", maxRetries, err)
}
}
return nil
}
// CATEGORIE AND CARATTERISTICHE FROM MIOGEST
func (m *MiogestHandler) InitDefaults() {
m.GenerateCaratteristiche()
m.GenerateCategorie()
}
func (m *MiogestHandler) GetCaratteristiche() typesdefs.ListaCaratteristiche {
if len(m.Caratterisiche.Caratteristiche) == 0 || time.Now().After(m.vars_cache_expires) {
m.GenerateCaratteristiche()
}
return m.Caratterisiche
}
func (m *MiogestHandler) GetCategorie() []typesdefs.Categoria {
if len(m.Categorie) == 0 || time.Now().After(m.vars_cache_expires) {
m.GenerateCategorie()
}
return m.Categorie
}
func (m *MiogestHandler) GenerateCaratteristiche() {
//https://www.miogest.com/apps/revo.aspx?tipo=schede
var result, err = utils.GetXMLFromFile[typesdefs.DefaultCaratteristiche]("./caratteristiche.xml")
if err != nil {
log.Printf("Failed to get caratteristiche: %v", err)
return
}
var array typesdefs.ListaCaratteristiche
for _, tag := range result.Scheda {
var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml}
array.Caratteristiche = append(array.Caratteristiche, c)
}
m.Caratterisiche = array
m.vars_cache_expires = time.Now().Add(1 * time.Hour)
}
func (m *MiogestHandler) GenerateCategorie() {
//https://www.miogest.com/apps/revo.aspx?tipo=categorie
var result, err = utils.GetXMLFromFile[typesdefs.DefaultCategories]("./categorie.xml")
if err != nil {
log.Printf("Failed to get categorie: %v", err)
return
}
var listaCategorie []typesdefs.Categoria
for _, elem := range result.Cat {
var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome}
listaCategorie = append(listaCategorie, c)
}
m.Categorie = listaCategorie
m.vars_cache_expires = time.Now().Add(1 * time.Hour)
}
// ANNUNCI PARSED
func (m *MiogestHandler) GetAnnunciParsed(imgPool *[]typesdefs.MediaToProcess, videoPool *[]typesdefs.MediaToProcess, forceMediaStale bool) (typesdefs.AnnunciParsed, error) {
// salvo il parsing nello struct nel caso voglio implementare un caching e non fare il parsing ad ogni chiamata
var err error
m.AnnunciParsed, err = m.ParseAnnunci("", imgPool, videoPool, forceMediaStale)
return m.AnnunciParsed, err
}
// versione per singolo annuncio
func (m *MiogestHandler) GetAnnuncioParsed(codFilter string, imgPool *[]typesdefs.MediaToProcess, videoPool *[]typesdefs.MediaToProcess) (typesdefs.AnnunciParsed, error) {
data, err := m.ParseAnnunci(codFilter, imgPool, videoPool, false)
if err != nil {
return typesdefs.AnnunciParsed{}, err
}
if len(data.Annuncio) == 0 {
return typesdefs.AnnunciParsed{}, fmt.Errorf("No Annuncio found for codice %s", codFilter)
}
return data, nil
}
func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.MediaToProcess, videoPool *[]typesdefs.MediaToProcess, forceMediaStale bool) (typesdefs.AnnunciParsed, error) {
annunci, err := m.GetAnnunci()
if err != nil {
log.Fatalf("Failed to get annunci for parsing: %v", err.Error())
return typesdefs.AnnunciParsed{}, err
}
if len(annunci.Annuncio) == 0 {
return typesdefs.AnnunciParsed{}, fmt.Errorf("No Annunci in update")
}
if codFilter != "" {
idx := slices.IndexFunc(annunci.Annuncio, func(a typesdefs.AnnuncioXML) bool {
return *a.Codice == codFilter
})
if idx == -1 {
return typesdefs.AnnunciParsed{}, fmt.Errorf("Codice %s not found", codFilter)
}
annunci.Annuncio = []typesdefs.AnnuncioXML{annunci.Annuncio[idx]}
}
image_codes_to_process := []string{}
if config.Cfg.Images.Enabled {
var err error
if forceMediaStale {
// process all images
image_codes_to_process = make([]string, 0, len(annunci.Annuncio))
for _, annuncio := range annunci.Annuncio {
if annuncio.Codice != nil {
image_codes_to_process = append(image_codes_to_process, *annuncio.Codice)
}
}
} else {
// process only new or updated images
image_codes_to_process, err = CodiciToProcessImages(&annunci)
if err != nil {
log.Fatalf("Failed to find images to update: %v", err.Error())
}
}
if codFilter != "" {
// if processing single codice, ensure it's in the list
if !slices.Contains(image_codes_to_process, codFilter) {
image_codes_to_process = append(image_codes_to_process, codFilter)
}
}
err = ReconcileImages()
err = ForceClearImages(image_codes_to_process)
if err != nil {
log.Fatalf("Failed to clear images for codice %s: %v", codFilter, err.Error())
}
}
video_codes_to_process := []string{}
if config.Cfg.Videos.Enabled {
var err error
if forceMediaStale {
// process all videos
video_codes_to_process = make([]string, 0, len(annunci.Annuncio))
for _, annuncio := range annunci.Annuncio {
if annuncio.Codice != nil {
video_codes_to_process = append(video_codes_to_process, *annuncio.Codice)
}
}
} else {
// process only new or updated videos
video_codes_to_process, err = CodiciToProcessVideos(&annunci)
if err != nil {
log.Fatalf("Failed to find videos to update: %v", err.Error())
}
}
if codFilter != "" {
// if processing single codice, ensure it's in the list
if !slices.Contains(video_codes_to_process, codFilter) {
video_codes_to_process = append(video_codes_to_process, codFilter)
}
}
err = ReconcileVideos()
err = ForceClearVideos(video_codes_to_process)
if err != nil {
log.Fatalf("Failed to clear videos for codice %s: %v", codFilter, err.Error())
}
}
totalAnnunci := len(annunci.Annuncio)
log.Printf("Processing %d annunci (images to update: %d, videos to update: %d)", totalAnnunci, len(image_codes_to_process), len(video_codes_to_process))
var AnnunciArray typesdefs.AnnunciParsed
AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio))
m.InitDefaults()
for i := range annunci.Annuncio {
imgToUpdate := slices.Contains(image_codes_to_process, *annunci.Annuncio[i].Codice)
updateVideos := slices.Contains(video_codes_to_process, *annunci.Annuncio[i].Codice)
result := extractData_update(&annunci.Annuncio[i], imgToUpdate, updateVideos, m, imgPool, videoPool)
AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result)
}
log.Printf("Processed %d annunci", len(AnnunciArray.Annuncio))
fmt.Println()
return AnnunciArray, nil
}
var re = regexp.MustCompile(`[^+\d]`)
func boolUpd(b bool) *bool {
return &b
}
func strUpd(s string) *string {
return &s
}
func extractData_update(annuncio *typesdefs.AnnuncioXML, updateImages bool, updateVideos bool, m *MiogestHandler, imgPool *[]typesdefs.MediaToProcess, videoPool *[]typesdefs.MediaToProcess) typesdefs.AnnuncioParsed {
var tmp typesdefs.AnnuncioParsed = typesdefs.AnnuncioParsed{}
tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice))
processLocatore(&tmp, annuncio)
processIndirizzo(&tmp, annuncio)
tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2)
tmp.Consegna = parser.ParseConsegna(annuncio.Consegna)
processCategorie(&tmp, annuncio, m)
processDatiImmobile(&tmp, annuncio)
tmp.Stato = annuncio.Stato
tmp.Homepage = boolUpd(false)
if annuncio.HomePage != nil {
if *annuncio.HomePage == "si" {
tmp.Homepage = boolUpd(true)
}
}
layout := "20060102150405"
if annuncio.Creato != nil {
t, err := time.Parse(layout, *annuncio.Creato)
if err != nil {
tmp.Creato_il = nil
} else {
tmp.Creato_il = &t
}
}
if annuncio.Modifica != nil {
t, err := time.Parse(layout, *annuncio.Modifica)
if err != nil {
tmp.Modificato_il = nil
} else {
tmp.Modificato_il = &t
}
}
tmp.Titolo_it = annuncio.Titolo
tmp.Titolo_en = annuncio.TitoloEn
tmp.Desc_it = annuncio.Descrizione
tmp.Desc_en = annuncio.DescrizioneEn
if annuncio.Dettaglio != nil {
disp, parm, pers, err := parser.ParseDettaglio(*annuncio.Dettaglio, tmp.Consegna)
if err == nil {
tmp.Disponibile_da = disp
tmp.Permanenza = parm
tmp.Persone = pers
}
}
tmp.Web = boolUpd(true)
if updateVideos {
AddToVideoProcessing(&tmp, annuncio, videoPool)
}
if annuncio.Video != nil {
ext_videos := []string{}
for _, v := range *annuncio.Video {
if strings.Contains(v, "youtu") {
ext_videos = append(ext_videos, v)
}
tmp.External_videos = &ext_videos
}
} else {
tmp.External_videos = &[]string{}
}
if annuncio.Accessori != nil {
tmp.Accessori = annuncio.Accessori
}
tmpcaratt := parseCaratteristiche(annuncio, m)
tmp.Caratteristiche = &tmpcaratt
if updateImages {
AddToImageProcessing(&tmp, annuncio, imgPool)
}
return tmp
}
func flatClienteTipologia(t []string) string {
if len(t) == 0 {
return ""
}
if slices.Contains(t, "Proprietario") {
return "proprietario"
} else if slices.Contains(t, "Delegato") {
return "delegato"
} else if slices.Contains(t, "Cliente") {
return "inquilino"
}
return ""
}
func tipologiaPriority(tipologie []string) int {
if slices.Contains(tipologie, "Delegato") {
return 3
}
if slices.Contains(tipologie, "Cliente") {
return 2
}
if slices.Contains(tipologie, "Proprietario") {
return 1
}
return 0
}
func selectTargetCliente(annuncio *typesdefs.AnnuncioXML) (int, bool) {
if len(annuncio.Clienti) == 0 {
return 0, false
}
best := 0
bestPriority := tipologiaPriority(annuncio.Clienti[0].Tipologie)
for i := 1; i < len(annuncio.Clienti); i++ {
if p := tipologiaPriority(annuncio.Clienti[i].Tipologie); p > bestPriority {
bestPriority = p
best = i
}
}
return best, true
}
func processLocatore(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
idx, ok := selectTargetCliente(annuncio)
if !ok {
log.Printf("Annuncio %s has no client information", utils.GetStringValue(annuncio.Codice))
return
}
trg := annuncio.Clienti[idx]
tmp.Email = parser.ParseEmail(trg.Email1)
var number string
for _, tel := range []*string{trg.Tel1, trg.Tel2, trg.Tel3} {
if tel != nil {
cleanedTel := re.ReplaceAllString(*tel, "")
if cleanedTel != "" {
number = cleanedTel
break
}
}
}
tmp.Numero = &number
if trg.Cognome != nil && trg.Nome != nil {
rawLoc := fmt.Sprintf("%s %s", utils.MakeUppercase(*trg.Cognome), utils.MakeUppercase(*trg.Nome))
rawLoc = strings.TrimSpace(rawLoc)
rawLoc = regexp.MustCompile(`\s+`).ReplaceAllString(rawLoc, " ") // Replace multiple spaces with a single space
tmp.Locatore = strUpd(rawLoc)
}
tmp.Idlocatore = trg.Id
tmp.Tipo_locatore = strUpd(flatClienteTipologia(trg.Tipologie))
}
func processIndirizzo(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
tmp.Indirizzo = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Indirizzo)))
tmp.Civico = annuncio.Civico
tmp.Comune = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Comune)))
tmp.Cap = annuncio.Cap
tmp.Provincia = annuncio.Provincia
tmp.Regione = annuncio.Regione
tmp.Lat = annuncio.Latitudine
tmp.Lon = annuncio.Longitudine
tmp.Indirizzo_secondario = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.IndirizzoSecondario)))
tmp.Civico_secondario = annuncio.CivicoSecondario
tmp.Lat_secondario = annuncio.LatitudineSecondario
tmp.Lon_secondario = annuncio.LongitudineSecondario
}
func processCategorie(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, m *MiogestHandler) {
if annuncio.Categoria == nil {
return
}
resultArray := make([]string, 0)
cats := *annuncio.Categoria
defaultCategorie := m.GetCategorie()
for _, cat := range cats {
for _, catdef := range defaultCategorie {
if cat == catdef.Id {
resultArray = append(resultArray, catdef.Nome)
}
}
}
if tmp.Categorie == nil {
tmp.Categorie = &[]string{}
}
*tmp.Categorie = resultArray
}
func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
tmpanno := utils.GetStringValue(annuncio.Anno)
if tmpanno != "0" && tmpanno != "" {
tmp.Anno = &tmpanno
}
tmp.Prezzo = parser.ParsePrezzo(annuncio.Prezzo)
tmp.Classe = annuncio.Classe
if annuncio.Mq != nil {
replaced := strings.ReplaceAll(*annuncio.Mq, ",", ".")
if s, err := strconv.ParseFloat(replaced, 64); err == nil {
tmp.Mq = &s
}
}
tmp.Piano = annuncio.Piano
tmp.Piano_palazzo = annuncio.PianiCondominio
tmp.Unita_condominio = annuncio.UnitaCondominio
tmp.Numero_vani = annuncio.Vani
tmp.Numero_camere = annuncio.Camere
tmp.Numero_bagni = annuncio.Bagni
tmp.Numero_balconi = annuncio.Balconi
tmp.Numero_terrazzi = annuncio.Terrazzi
tmp.Numero_box = annuncio.Box
tmp.Numero_postiauto = annuncio.PostiAuto
}
func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, m *MiogestHandler) map[string]any {
if reflect.TypeFor[*typesdefs.AnnuncioXML]().Kind() != reflect.Pointer || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct {
return nil
}
tmpCaratteristiche := make(map[string]any)
val := reflect.ValueOf(wrkrecord).Elem()
numField := val.NumField()
for i := range numField {
if strings.HasPrefix(val.Type().Field(i).Name, "Scheda_") {
nameField := val.Type().Field(i).Name
valueField := val.Field(i)
if valueField.IsValid() && !valueField.IsNil() {
tmpCaratteristiche[nameField] = valueField.Interface()
} else {
tmpCaratteristiche[nameField] = nil
}
}
}
for _, scheda := range m.GetCaratteristiche().Caratteristiche {
if _, ok := tmpCaratteristiche[scheda.Tagxml]; !ok {
tmpCaratteristiche[scheda.Tagxml] = nil
}
}
return tmpCaratteristiche
}

182
apps/backend/miogest_parsing.go Normal file
View file

@ -0,0 +1,182 @@
package main
import (
"backend/parser"
"backend/typesdefs"
"backend/utils"
"fmt"
"log"
"reflect"
"regexp"
"slices"
"strconv"
"strings"
)
var re = regexp.MustCompile(`[^+\d]`)
func boolUpd(b bool) *bool {
return &b
}
func strUpd(s string) *string {
return &s
}
func flatClienteTipologia(t []string) string {
if len(t) == 0 {
return ""
}
if slices.Contains(t, "Proprietario") {
return "proprietario"
} else if slices.Contains(t, "Delegato") {
return "delegato"
} else if slices.Contains(t, "Cliente") {
return "inquilino"
}
return ""
}
func tipologiaPriority(tipologie []string) int {
if slices.Contains(tipologie, "Delegato") {
return 3
}
if slices.Contains(tipologie, "Cliente") {
return 2
}
if slices.Contains(tipologie, "Proprietario") {
return 1
}
return 0
}
func selectTargetCliente(annuncio *typesdefs.AnnuncioXML) (int, bool) {
if len(annuncio.Clienti) == 0 {
return 0, false
}
best := 0
bestPriority := tipologiaPriority(annuncio.Clienti[0].Tipologie)
for i := 1; i < len(annuncio.Clienti); i++ {
if p := tipologiaPriority(annuncio.Clienti[i].Tipologie); p > bestPriority {
bestPriority = p
best = i
}
}
return best, true
}
func processLocatore(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
idx, ok := selectTargetCliente(annuncio)
if !ok {
log.Printf("Annuncio %s has no client information", utils.GetStringValue(annuncio.Codice))
return
}
trg := annuncio.Clienti[idx]
tmp.Email = parser.ParseEmail(trg.Email1)
var number string
for _, tel := range []*string{trg.Tel1, trg.Tel2, trg.Tel3} {
if tel != nil {
cleanedTel := re.ReplaceAllString(*tel, "")
if cleanedTel != "" {
number = cleanedTel
break
}
}
}
tmp.Numero = &number
if trg.Cognome != nil && trg.Nome != nil {
rawLoc := fmt.Sprintf("%s %s", utils.MakeUppercase(*trg.Cognome), utils.MakeUppercase(*trg.Nome))
rawLoc = strings.TrimSpace(rawLoc)
rawLoc = regexp.MustCompile(`\s+`).ReplaceAllString(rawLoc, " ") // Replace multiple spaces with a single space
tmp.Locatore = strUpd(rawLoc)
}
tmp.Idlocatore = trg.Id
tmp.Tipo_locatore = strUpd(flatClienteTipologia(trg.Tipologie))
}
func processIndirizzo(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
tmp.Indirizzo = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Indirizzo)))
tmp.Civico = annuncio.Civico
tmp.Comune = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Comune)))
tmp.Cap = annuncio.Cap
tmp.Provincia = annuncio.Provincia
tmp.Regione = annuncio.Regione
tmp.Lat = annuncio.Latitudine
tmp.Lon = annuncio.Longitudine
tmp.Indirizzo_secondario = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.IndirizzoSecondario)))
tmp.Civico_secondario = annuncio.CivicoSecondario
tmp.Lat_secondario = annuncio.LatitudineSecondario
tmp.Lon_secondario = annuncio.LongitudineSecondario
}
func processCategorie(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, def_cats *[]typesdefs.Categoria) {
if annuncio.Categoria == nil {
return
}
resultArray := make([]string, 0)
cats := *annuncio.Categoria
for _, cat := range cats {
for _, catdef := range *def_cats {
if cat == catdef.Id {
resultArray = append(resultArray, catdef.Nome)
}
}
}
if tmp.Categorie == nil {
tmp.Categorie = &[]string{}
}
*tmp.Categorie = resultArray
}
func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) {
tmpanno := utils.GetStringValue(annuncio.Anno)
if tmpanno != "0" && tmpanno != "" {
tmp.Anno = &tmpanno
}
tmp.Prezzo = parser.ParsePrezzo(annuncio.Prezzo)
tmp.Classe = annuncio.Classe
if annuncio.Mq != nil {
replaced := strings.ReplaceAll(*annuncio.Mq, ",", ".")
if s, err := strconv.ParseFloat(replaced, 64); err == nil {
tmp.Mq = &s
}
}
tmp.Piano = annuncio.Piano
tmp.Piano_palazzo = annuncio.PianiCondominio
tmp.Unita_condominio = annuncio.UnitaCondominio
tmp.Numero_vani = annuncio.Vani
tmp.Numero_camere = annuncio.Camere
tmp.Numero_bagni = annuncio.Bagni
tmp.Numero_balconi = annuncio.Balconi
tmp.Numero_terrazzi = annuncio.Terrazzi
tmp.Numero_box = annuncio.Box
tmp.Numero_postiauto = annuncio.PostiAuto
}
func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, caratt *typesdefs.ListaCaratteristiche) map[string]any {
if reflect.TypeFor[*typesdefs.AnnuncioXML]().Kind() != reflect.Pointer || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct {
return nil
}
tmpCaratteristiche := make(map[string]any)
val := reflect.ValueOf(wrkrecord).Elem()
numField := val.NumField()
for i := range numField {
if strings.HasPrefix(val.Type().Field(i).Name, "Scheda_") {
nameField := val.Type().Field(i).Name
valueField := val.Field(i)
if valueField.IsValid() && !valueField.IsNil() {
tmpCaratteristiche[nameField] = valueField.Interface()
} else {
tmpCaratteristiche[nameField] = nil
}
}
}
for _, scheda := range caratt.Caratteristiche {
if _, ok := tmpCaratteristiche[scheda.Tagxml]; !ok {
tmpCaratteristiche[scheda.Tagxml] = nil
}
}
return tmpCaratteristiche
}

48
apps/backend/queries/annunci.go
View file

@ -5,7 +5,7 @@ import (
. "backend/gen/postgres/postgres/public/table" . "backend/gen/postgres/postgres/public/table"
"backend/typesdefs" "backend/typesdefs"
"fmt" "fmt"
"slices" "log"
"time" "time"
. "github.com/go-jet/jet/v2/postgres" . "github.com/go-jet/jet/v2/postgres"
@ -22,26 +22,22 @@ func AnnunciGetActive() ([]string, error) {
return annunci, nil return annunci, nil
} }
func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) error { func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool, isDryRun bool) error {
var err error var err error
if resetAllBeforeUpdate { if resetAllBeforeUpdate {
//SET ALL WEB TO FALSE //SET ALL WEB TO FALSE
stmt := Annunci.UPDATE(Annunci.Web, Annunci.Homepage, Annunci.Stato).SET(false, false, "Sospeso").WHERE(Annunci.Codice.IS_NOT_NULL()) stmt := Annunci.UPDATE(Annunci.Web, Annunci.Homepage, Annunci.Stato).SET(false, false, "Sospeso").WHERE(Annunci.Codice.IS_NOT_NULL())
_, err = stmt.Exec(postgres) if isDryRun {
if err != nil { log.Printf("Dry run: Setting all web to false")
return fmt.Errorf("failed to set all web to false: %w", err) } else {
_, err = stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to set all web to false: %w", err)
}
} }
} }
//GET ALL CODICI
var codici []string
stmt := SELECT(Annunci.Codice).DISTINCT(Annunci.Codice).FROM(Annunci)
err = stmt.Query(postgres, &codici)
if err != nil {
return fmt.Errorf("failed to get all codici: %w", err)
}
var to_update []models.Annunci
var to_insert []models.Annunci var to_insert []models.Annunci
for _, annuncio := range annunci.Annuncio { for _, annuncio := range annunci.Annuncio {
value := models.Annunci{ value := models.Annunci{
@ -97,35 +93,19 @@ func AnnunciInsert(annunci typesdefs.AnnunciParsed, resetAllBeforeUpdate bool) e
TipoLocatore: annuncio.Tipo_locatore, TipoLocatore: annuncio.Tipo_locatore,
MediaUpdatedAt: Ptr(time.Now()), MediaUpdatedAt: Ptr(time.Now()),
} }
if slices.Contains(codici, annuncio.Codice) { to_insert = append(to_insert, value)
to_update = append(to_update, value)
} else {
to_insert = append(to_insert, value)
}
} }
if len(to_insert) > 0 { if len(to_insert) > 0 {
stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert) stmt := Annunci.INSERT(Annunci.MutableColumns).MODELS(to_insert).ON_CONFLICT(Annunci.Codice).DO_UPDATE(SET(Annunci.MutableColumns.SET(Annunci.EXCLUDED.MutableColumns)))
_, err = stmt.Exec(postgres) if isDryRun {
if err != nil { log.Printf("Dry run: would have inserted %d records\n", len(to_insert))
return fmt.Errorf("failed to insert/update annunci batch: %w", err) } else {
}
}
if len(to_update) > 0 {
for _, upd := range to_update {
stmt := Annunci.UPDATE(Annunci.MutableColumns).MODEL(upd).WHERE(Annunci.Codice.EQ(Text(upd.Codice)))
_, err = stmt.Exec(postgres) _, err = stmt.Exec(postgres)
if err != nil { if err != nil {
return fmt.Errorf("failed to insert/update annunci batch: %w", err) return fmt.Errorf("failed to insert/update annunci batch: %w", err)
} }
} }
} }
if err != nil {
return fmt.Errorf("failed to insert/update annunci batch: %w", err)
}
fmt.Println("Annunci insert/update done") fmt.Println("Annunci insert/update done")
return nil return nil
} }

69
apps/backend/queries/images.go
View file

@ -1,69 +0,0 @@
package queries
import (
models "backend/gen/postgres/postgres/public/model"
. "backend/gen/postgres/postgres/public/table"
"backend/typesdefs"
"fmt"
"log"
. "github.com/go-jet/jet/v2/postgres"
)
func SetImagesToDB(pool *[]typesdefs.UploadedMedia) error {
if len(*pool) == 0 {
log.Println("No new images to save to DB.")
return nil
}
var to_insert []models.ImagesRefs
for _, data := range *pool {
to_insert = append(to_insert, models.ImagesRefs{
Codice: data.Codice,
Ordine: int32(data.Order),
Img: data.MediaId,
Thumb: data.ThumbId,
OgURL: data.OGUrl,
})
}
stmt := ImagesRefs.INSERT(ImagesRefs.AllColumns).MODELS(to_insert).ON_CONFLICT(ImagesRefs.Codice, ImagesRefs.Ordine).DO_UPDATE(SET(ImagesRefs.Img.SET(ImagesRefs.EXCLUDED.Img), ImagesRefs.Thumb.SET(ImagesRefs.EXCLUDED.Thumb)))
_, err := stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to insert/update images batch: %w", err)
}
return nil
}
func GetImagesFromDB() ([]models.ImagesRefs, error) {
var images []models.ImagesRefs
stmt := SELECT(ImagesRefs.AllColumns).FROM(ImagesRefs).ORDER_BY(ImagesRefs.Ordine.ASC())
err := stmt.Query(postgres, &images)
if err != nil {
return nil, fmt.Errorf("failed to get images: %w", err)
}
return images, nil
}
func GetCodiceImagesFromDB(codice string) ([]models.ImagesRefs, error) {
var images []models.ImagesRefs
stmt := SELECT(ImagesRefs.AllColumns).FROM(ImagesRefs).WHERE(ImagesRefs.Codice.EQ(Text(codice))).ORDER_BY(ImagesRefs.Ordine.ASC())
err := stmt.Query(postgres, &images)
if err != nil {
return nil, fmt.Errorf("failed to get images: %w", err)
}
return images, nil
}
func ClearImagesRefsByCodice(codice string) error {
stmt := ImagesRefs.DELETE().WHERE(ImagesRefs.Codice.EQ(Text(codice)))
_, err := stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to clear images refs by codice: %w", err)
}
return nil
}

51
apps/backend/queries/media.go Normal file
View file

@ -0,0 +1,51 @@
package queries
import (
models "backend/gen/postgres/postgres/public/model"
. "backend/gen/postgres/postgres/public/table"
"fmt"
. "github.com/go-jet/jet/v2/postgres"
)
func SetMediaToDB(to_insert models.MediaRefs) error {
stmt := MediaRefs.INSERT(MediaRefs.AllColumns).MODEL(to_insert).ON_CONFLICT(MediaRefs.Codice, MediaRefs.Ordine, MediaRefs.IsThumbnail, MediaRefs.MediaType).DO_UPDATE(SET(MediaRefs.StorageID.SET(MediaRefs.EXCLUDED.StorageID), MediaRefs.OgURL.SET(MediaRefs.EXCLUDED.OgURL)))
_, err := stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to insert/update media_refs: %w", err)
}
return nil
}
func GetMediaFromDB() ([]models.MediaRefs, error) {
var media []models.MediaRefs
stmt := SELECT(MediaRefs.AllColumns).FROM(MediaRefs).ORDER_BY(MediaRefs.Ordine.ASC())
err := stmt.Query(postgres, &media)
if err != nil {
return nil, fmt.Errorf("failed to get media: %w", err)
}
return media, nil
}
// func GetCodiceMediaFromDB(codice string) ([]models.MediaRefs, error) {
// var media []models.MediaRefs
// stmt := SELECT(MediaRefs.AllColumns).FROM(MediaRefs).WHERE(MediaRefs.Codice.EQ(Text(codice))).ORDER_BY(MediaRefs.Ordine.ASC())
// err := stmt.Query(postgres, &media)
// if err != nil {
// return nil, fmt.Errorf("failed to get media: %w", err)
// }
// return media, nil
// }
// func ClearMediaRefsByCodice(codice string) error {
// stmt := MediaRefs.DELETE().WHERE(MediaRefs.Codice.EQ(Text(codice)))
// _, err := stmt.Exec(postgres)
// if err != nil {
// return fmt.Errorf("failed to clear media_refs by codice: %w", err)
// }
// return nil
// }

3
apps/backend/queries/queries.go
View file

@ -3,6 +3,7 @@ package queries
import ( import (
"database/sql" "database/sql"
"fmt" "fmt"
"log"
"os" "os"
"github.com/joho/godotenv" "github.com/joho/godotenv"
@ -19,7 +20,7 @@ func init() {
connStr := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", os.Getenv("POSTGRES_USER"), os.Getenv("POSTGRES_PASSWORD"), os.Getenv("PGHOST"), os.Getenv("PGPORT"), os.Getenv("POSTGRES_DB")) connStr := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", os.Getenv("POSTGRES_USER"), os.Getenv("POSTGRES_PASSWORD"), os.Getenv("PGHOST"), os.Getenv("PGPORT"), os.Getenv("POSTGRES_DB"))
postgres, err = sql.Open("postgres", connStr) postgres, err = sql.Open("postgres", connStr)
if err != nil { if err != nil {
panic(err) log.Fatalf("Error connecting to database: %v", err)
} }
} }

50
apps/backend/queries/storage.go
View file

@ -8,7 +8,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
. "github.com/go-jet/jet/v2/postgres" //. "github.com/go-jet/jet/v2/postgres"
"github.com/google/uuid" "github.com/google/uuid"
) )
@ -21,32 +21,32 @@ import (
// return nil // return nil
// } // }
func DeleteMultFile(ids uuid.UUIDs) error { // func DeleteMultFile(ids uuid.UUIDs) error {
if len(ids) == 0 { // if len(ids) == 0 {
return nil // return nil
} // }
var sqlIDs []Expression // var sqlIDs []Expression
for _, id := range ids { // for _, id := range ids {
sqlIDs = append(sqlIDs, UUID(id)) // sqlIDs = append(sqlIDs, UUID(id))
} // }
stmt := Storage.DELETE().WHERE(Storage.ID.IN(sqlIDs...)) // stmt := Storage.DELETE().WHERE(Storage.ID.IN(sqlIDs...))
_, err := stmt.Exec(postgres) // _, err := stmt.Exec(postgres)
if err != nil { // if err != nil {
return fmt.Errorf("failed to delete files: %w", err) // return fmt.Errorf("failed to delete files: %w", err)
} // }
return nil // return nil
} // }
func GetStorageByTag(tag string) ([]models.Storage, error) { // func GetStorageByTag(tag string) ([]models.Storage, error) {
var rows []models.Storage // var rows []models.Storage
stmt := SELECT(Storage.AllColumns).FROM(Storage).WHERE(Storage.Tag.EQ(Text(tag))) // stmt := SELECT(Storage.AllColumns).FROM(Storage).WHERE(Storage.Tag.EQ(Text(tag)))
err := stmt.Query(postgres, &rows) // err := stmt.Query(postgres, &rows)
if err != nil { // if err != nil {
return nil, fmt.Errorf("failed to get storage by tag: %w", err) // return nil, fmt.Errorf("failed to get storage by tag: %w", err)
} // }
return rows, nil // return rows, nil
} // }
func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) { func UploadFile(path string, mimeType string, tag string) (uuid.UUID, error) {

65
apps/backend/queries/videos.go
View file

@ -1,65 +0,0 @@
package queries
import (
models "backend/gen/postgres/postgres/public/model"
. "backend/gen/postgres/postgres/public/table"
"backend/typesdefs"
"fmt"
"log"
. "github.com/go-jet/jet/v2/postgres"
)
func SetVideosToDB(pool *[]typesdefs.UploadedMedia) error {
if len(*pool) == 0 {
log.Println("No new videos to save to DB.")
return nil
}
var to_insert []models.VideosRefs
for _, data := range *pool {
to_insert = append(to_insert, models.VideosRefs{
Codice: data.Codice,
Ordine: int32(data.Order),
Video: data.MediaId,
Thumb: data.ThumbId,
OgURL: data.OGUrl,
})
}
stmt := VideosRefs.INSERT(VideosRefs.AllColumns).MODELS(to_insert).ON_CONFLICT(VideosRefs.Codice, VideosRefs.Ordine).DO_UPDATE(SET(VideosRefs.Video.SET(VideosRefs.EXCLUDED.Video), VideosRefs.Thumb.SET(VideosRefs.EXCLUDED.Thumb)))
_, err := stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to insert video refs: %w", err)
}
return nil
}
func GetVideosFromDB() ([]models.VideosRefs, error) {
var videos []models.VideosRefs
stmt := VideosRefs.SELECT(VideosRefs.AllColumns).FROM(VideosRefs).ORDER_BY(VideosRefs.Ordine.ASC())
err := stmt.Query(postgres, &videos)
if err != nil {
return nil, fmt.Errorf("failed to get videos: %w", err)
}
return videos, nil
}
func GetCodiceVideosFromDB(codice string) ([]models.VideosRefs, error) {
var videos []models.VideosRefs
stmt := VideosRefs.SELECT(VideosRefs.AllColumns).FROM(VideosRefs).WHERE(VideosRefs.Codice.EQ(Text(codice))).ORDER_BY(VideosRefs.Ordine.ASC())
err := stmt.Query(postgres, &videos)
if err != nil {
return nil, fmt.Errorf("failed to get videos: %w", err)
}
return videos, nil
}
func ClearVideosRefsByCodice(codice string) error {
stmt := VideosRefs.DELETE().WHERE(VideosRefs.Codice.EQ(Text(codice)))
_, err := stmt.Exec(postgres)
if err != nil {
return fmt.Errorf("failed to clear videos refs by codice: %w", err)
}
return nil
}

137
apps/backend/server.go
View file

@ -3,34 +3,25 @@ package main
import ( import (
"backend/config" "backend/config"
"backend/queries" "backend/queries"
"backend/typesdefs"
"fmt" "fmt"
"io" "io"
"log" "log"
"math/rand"
"net/http" "net/http"
"os" "os"
"strconv" "strconv"
"strings" "strings"
"text/template" "text/template"
"time"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
) )
type Server struct { type Server struct {
miogest *MiogestHandler config *config.Config
config *config.Config
} }
func NewServer(cfg *config.Config) (*Server, error) { func NewServer(cfg *config.Config) (*Server, error) {
miogest := NewMiogestHandler()
if miogest == nil {
return nil, fmt.Errorf("error connecting to miogest")
}
return &Server{ return &Server{
miogest: miogest, config: cfg,
config: cfg,
}, nil }, nil
} }
@ -78,126 +69,18 @@ func (s *Server) SetupRoutes() *echo.Echo {
return c.JSONPretty(http.StatusOK, fmt.Sprintf("DB Connection: %v", h), " ") return c.JSONPretty(http.StatusOK, fmt.Sprintf("DB Connection: %v", h), " ")
}) })
// MIOGEST ROUTES
e.GET("/cats", func(c echo.Context) error {
var data = s.miogest.GetCaratteristiche()
return c.JSONPretty(http.StatusOK, data, " ")
})
e.GET("/categ", func(c echo.Context) error {
var data = s.miogest.GetCategorie()
return c.JSONPretty(http.StatusOK, data, " ")
})
e.GET("/miogest", func(c echo.Context) error {
data, err := s.miogest.GetAnnunci()
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
return c.JSONPretty(http.StatusOK, data, " ")
})
e.GET("/miogest-sample", func(c echo.Context) error {
xml, err := s.miogest.GetAnnunci()
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
rand := rand.Intn(len(xml.Annuncio))
data := xml.Annuncio[rand]
return c.JSONPretty(http.StatusOK, data, " ")
})
e.GET("/parse", func(c echo.Context) error {
start := time.Now()
var data, err = s.miogest.GetAnnunciParsed(nil, nil, false)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
end := time.Now()
fmt.Println("time taken:", end.Sub(start).String())
return c.JSONPretty(http.StatusOK, data, " ")
})
e.GET("/update", func(c echo.Context) error { e.GET("/update", func(c echo.Context) error {
forceMediaStale := false //da inizializzare on server startup non qui
forceMediaStr := c.QueryParam("forcemedia") u := NewUpdater(UpdaterConfig{
if forceMediaStr == "true" { isDryRun: c.QueryParam("dryrun") == "true",
forceMediaStale = true codFilter: c.QueryParam("cod"),
} forceMediaRefresh: c.QueryParam("forcemedia") == "true",
imgPool := []typesdefs.MediaToProcess{} })
videoPool := []typesdefs.MediaToProcess{} err := u.Update()
data, err := s.miogest.GetAnnunciParsed(&imgPool, &videoPool, forceMediaStale)
if err != nil { if err != nil {
return c.String(http.StatusInternalServerError, err.Error()) return c.String(http.StatusInternalServerError, err.Error())
} }
err = queries.AnnunciInsert(data, true) return c.String(http.StatusOK, "update")
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
imgRefs, err := ProcessImagePool(&imgPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetImagesToDB(&imgRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetVideosToDB(&videoRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
/*
err = invalidateAllAnnunciFB()
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
*/
log.Println("Update completed")
return c.String(http.StatusOK, "Updated")
})
e.GET("/update-cod/:cod", func(c echo.Context) error {
cod := c.Param("cod")
imgPool := []typesdefs.MediaToProcess{}
videoPool := []typesdefs.MediaToProcess{}
data, err := s.miogest.GetAnnuncioParsed(cod, &imgPool, &videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.AnnunciInsert(data, false)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
imgRefs, err := ProcessImagePool(&imgPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetImagesToDB(&imgRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
videoRefs, err := ProcessVideoPool(&videoPool)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
err = queries.SetVideosToDB(&videoRefs)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
/*
url := fmt.Sprintf("https://www.infoalloggi.it/annuncio/%s", cod)
err = invalidateFacebookCache(url)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
*/
log.Printf("Update %s completed", cod)
return c.String(http.StatusOK, "Updated")
}) })
//IMAGE ROUTES //IMAGE ROUTES

22
apps/backend/typesdefs/models.go
View file

@ -2,8 +2,6 @@ package typesdefs
import ( import (
"time" "time"
"github.com/google/uuid"
) )
type AnnunciXML struct { type AnnunciXML struct {
@ -128,7 +126,9 @@ type AnnuncioXML struct {
Url *string `xml:"Url"` Url *string `xml:"Url"`
} `xml:"AMVideo"` } `xml:"AMVideo"`
} `xml:"AMMedias"` } `xml:"AMMedias"`
Dettaglio *string `xml:"Dettaglio"` Dettaglio *string `xml:"Dettaglio"`
ProcessingImages bool
ProcessingVideos bool
} }
type AnnunciParsed struct { type AnnunciParsed struct {
@ -218,19 +218,3 @@ type MediaToProcess struct {
Url string Url string
Order int Order int
} }
type ProcessedMedia struct {
Codice string
Order int
Path string
ThumnailPath string
OGUrl string
}
type UploadedMedia struct {
Codice string
Order int
MediaId uuid.UUID
ThumbId uuid.UUID
OGUrl string
}

359
apps/backend/update.go Normal file
View file

@ -0,0 +1,359 @@
package main
import (
"backend/config"
models "backend/gen/postgres/postgres/public/model"
"backend/parser"
"backend/queries"
"backend/typesdefs"
"backend/utils"
"fmt"
"log"
"slices"
"strings"
"time"
)
type UpdaterConfig struct {
// Dry run flag to simulate the update process without making changes.
isDryRun bool
// Filter by codice filter for specific updates. If empty, all records are updated.
codFilter string
// Force media refresh flag to re-download and upload media
forceMediaRefresh bool
}
type Updater struct {
isDryRun bool
codFilter string
forceMediaRefresh bool
caratterisiche typesdefs.ListaCaratteristiche
categorie []typesdefs.Categoria
imgPool []typesdefs.MediaToProcess
videoPool []typesdefs.MediaToProcess
annunci typesdefs.AnnunciParsed
}
func NewUpdater(props UpdaterConfig) *Updater {
u := Updater{
isDryRun: props.isDryRun,
codFilter: props.codFilter,
forceMediaRefresh: props.forceMediaRefresh,
}
fmt.Printf("updater cfg: %+v\n", props)
u.init_Updater()
return &u
}
func (u *Updater) init_Updater() {
var err error
u.caratterisiche, err = genCaratteristiche()
if err != nil {
log.Fatalf("Error generating carateristiche: %v", err)
}
u.categorie, err = genCategorie()
if err != nil {
log.Fatalf("Error generating categorie: %v", err)
}
}
func genCaratteristiche() (typesdefs.ListaCaratteristiche, error) {
var values typesdefs.ListaCaratteristiche
//https://www.miogest.com/apps/revo.aspx?tipo=schede
var result, err = utils.GetXMLFromFile[typesdefs.DefaultCaratteristiche]("./caratteristiche.xml")
if err != nil {
return values, err
}
for _, tag := range result.Scheda {
var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml}
values.Caratteristiche = append(values.Caratteristiche, c)
}
return values, nil
}
func genCategorie() ([]typesdefs.Categoria, error) {
var values []typesdefs.Categoria
//https://www.miogest.com/apps/revo.aspx?tipo=categorie
var result, err = utils.GetXMLFromFile[typesdefs.DefaultCategories]("./categorie.xml")
if err != nil {
return values, err
}
for _, elem := range result.Cat {
var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome}
values = append(values, c)
}
return values, nil
}
func (u *Updater) Update() error {
var err error
err = u.parseAnnunci()
if err != nil {
return err
}
err = queries.AnnunciInsert(u.annunci, u.codFilter == "", u.isDryRun)
if err != nil {
return err
}
err = ProcessImagePool(&u.imgPool, u.isDryRun)
if err != nil {
return err
}
err = ProcessVideoPool(&u.videoPool, u.isDryRun)
if err != nil {
return err
}
log.Println("Update completed")
return nil
}
func (u *Updater) parseAnnunci() error {
var err error
xml, err := getAnnunciMiogest()
if err != nil {
return err
}
// filter by codFilter if provided
if u.codFilter != "" {
idx := slices.IndexFunc(xml.Annuncio, func(a typesdefs.AnnuncioXML) bool {
return *a.Codice == u.codFilter
})
if idx == -1 {
return fmt.Errorf("Codice Filtering: Annuncio con codice %s not found", u.codFilter)
}
xml.Annuncio = []typesdefs.AnnuncioXML{xml.Annuncio[idx]}
}
u.setupMediaWorkload(&xml)
for i := range xml.Annuncio {
u.annunci.Annuncio = append(u.annunci.Annuncio, u.extractAnnuncioData(&xml.Annuncio[i]))
}
return err
}
func getAnnunciMiogest() (typesdefs.AnnunciXML, error) {
var result typesdefs.AnnunciXML
var err error
result, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml")
if err != nil {
return result, fmt.Errorf("Error fetching data from Miogest: %w", err)
}
if len(result.Annuncio) == 0 {
return result, fmt.Errorf("No annunci found in the XML data")
}
for i, d := range result.Annuncio {
sanified := strings.ReplaceAll(*d.Codice, " ", "")
result.Annuncio[i].Codice = &sanified
}
return result, nil
}
func (u *Updater) setupMediaWorkload(xml *typesdefs.AnnunciXML) {
if u.forceMediaRefresh && (config.Cfg.Images.Enabled || config.Cfg.Videos.Enabled) {
for i := range xml.Annuncio {
xml.Annuncio[i].ProcessingImages = config.Cfg.Images.Enabled
xml.Annuncio[i].ProcessingVideos = config.Cfg.Videos.Enabled
}
return
}
db_media, err := queries.GetMediaFromDB()
if err != nil {
log.Fatalf("failed to get media from db: %v", err)
return
}
for i, a := range xml.Annuncio {
if a.Codice == nil {
continue
}
var in_db []models.MediaRefs
for _, v := range db_media {
if v.Codice == *a.Codice {
in_db = append(in_db, v)
}
}
fmt.Printf("codice: %s, in_bd: %d\n", *a.Codice, len(in_db))
if len(in_db) == 0 {
// not found in db
if config.Cfg.Images.Enabled {
xml.Annuncio[i].ProcessingImages = true
}
if config.Cfg.Videos.Enabled {
xml.Annuncio[i].ProcessingVideos = true
}
continue
}
if config.Cfg.Images.Enabled {
img_process := false
for _, fotoUrl := range *a.Foto {
found_img := false
found_thumb := false
for _, ref := range in_db {
if found_img && found_thumb {
break
}
fmt.Printf("%s -> %s\n", fotoUrl, ref.OgURL)
if fotoUrl == ref.OgURL {
fmt.Println("match")
if ref.IsThumbnail {
found_thumb = true
continue
}
found_img = true
}
}
if !found_img || !found_thumb {
fmt.Printf("not found img %s\n", fotoUrl)
img_process = true
break
}
}
xml.Annuncio[i].ProcessingImages = img_process
}
if config.Cfg.Videos.Enabled {
vid_process := false
for _, media := range a.AMMedias.AMVideo {
if media.Url == nil {
continue
}
if !strings.Contains(*media.Url, "video.miogest") {
continue
}
found_vid := false
found_thumb := false
for _, ref := range in_db {
if found_vid && found_thumb {
break
}
if *media.Url == ref.OgURL {
if ref.IsThumbnail {
found_thumb = true
continue
}
found_vid = true
}
}
if !found_vid || !found_thumb {
vid_process = true
break
}
}
xml.Annuncio[i].ProcessingImages = vid_process
}
}
if u.codFilter != "" {
for i := range xml.Annuncio {
if config.Cfg.Images.Enabled {
xml.Annuncio[i].ProcessingImages = true
}
if config.Cfg.Videos.Enabled {
xml.Annuncio[i].ProcessingImages = true
}
}
}
}
func (u *Updater) extractAnnuncioData(annuncio *typesdefs.AnnuncioXML) typesdefs.AnnuncioParsed {
tmp := typesdefs.AnnuncioParsed{}
tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice))
processLocatore(&tmp, annuncio)
processIndirizzo(&tmp, annuncio)
tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2)
tmp.Consegna = parser.ParseConsegna(annuncio.Consegna)
processCategorie(&tmp, annuncio, &u.categorie)
processDatiImmobile(&tmp, annuncio)
tmp.Stato = annuncio.Stato
tmp.Homepage = boolUpd(false)
if annuncio.HomePage != nil {
if *annuncio.HomePage == "si" {
tmp.Homepage = boolUpd(true)
}
}
layout := "20060102150405"
if annuncio.Creato != nil {
t, err := time.Parse(layout, *annuncio.Creato)
if err != nil {
tmp.Creato_il = nil
} else {
tmp.Creato_il = &t
}
}
if annuncio.Modifica != nil {
t, err := time.Parse(layout, *annuncio.Modifica)
if err != nil {
tmp.Modificato_il = nil
} else {
tmp.Modificato_il = &t
}
}
tmp.Titolo_it = annuncio.Titolo
tmp.Titolo_en = annuncio.TitoloEn
tmp.Desc_it = annuncio.Descrizione
tmp.Desc_en = annuncio.DescrizioneEn
if annuncio.Dettaglio != nil {
disp, parm, pers, err := parser.ParseDettaglio(*annuncio.Dettaglio, tmp.Consegna)
if err == nil {
tmp.Disponibile_da = disp
tmp.Permanenza = parm
tmp.Persone = pers
}
}
tmp.Web = boolUpd(true)
if annuncio.ProcessingVideos {
AddToVideoProcessing(&tmp, annuncio, &u.videoPool)
}
if annuncio.Video != nil {
ext_videos := []string{}
for _, v := range *annuncio.Video {
if strings.Contains(v, "youtu") {
ext_videos = append(ext_videos, v)
}
tmp.External_videos = &ext_videos
}
} else {
tmp.External_videos = &[]string{}
}
if annuncio.Accessori != nil {
tmp.Accessori = annuncio.Accessori
}
tmpcaratt := parseCaratteristiche(annuncio, &u.caratterisiche)
tmp.Caratteristiche = &tmpcaratt
if annuncio.ProcessingImages {
AddToImageProcessing(&tmp, annuncio, &u.imgPool)
}
return tmp
}

257
apps/backend/video.go
View file

@ -2,6 +2,7 @@ package main
import ( import (
"backend/config" "backend/config"
models "backend/gen/postgres/postgres/public/model"
"backend/queries" "backend/queries"
"backend/typesdefs" "backend/typesdefs"
"backend/utils" "backend/utils"
@ -17,7 +18,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/google/uuid"
"github.com/nfnt/resize" "github.com/nfnt/resize"
"github.com/nickalie/go-webpbin" "github.com/nickalie/go-webpbin"
) )
@ -45,25 +45,25 @@ func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.Ann
} }
} }
func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.UploadedMedia, error) { func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess, isDryRun bool) error {
if len(*toProcess) == 0 { if len(*toProcess) == 0 {
log.Println("No new videos to process.") log.Println("No new videos to process.")
return []typesdefs.UploadedMedia{}, nil return nil
} }
var ( var wg sync.WaitGroup
ProcessedVideos []typesdefs.ProcessedMedia
UploadedVideos []typesdefs.UploadedMedia
mu sync.Mutex
wg sync.WaitGroup
)
totalToProcess := len(*toProcess) totalToProcess := len(*toProcess)
log.Printf("Processing %d videos with concurrency limit %d", totalToProcess, config.Cfg.Videos.ConcurrentLimit) log.Printf("Processing %d videos with concurrency limit %d", totalToProcess, config.Cfg.Videos.ConcurrentLimit)
sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit) sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
// clear videos folder // clear videos folder
err := clearVideoFolder() if isDryRun {
if err != nil { log.Println("Dry run. Clearing videos folder.")
log.Fatalf("Failed to initialize image processing: %v", err) } else {
err := clearVideoFolder()
if err != nil {
log.Fatalf("Failed to initialize image processing: %v", err)
}
} }
// process videos // process videos
@ -73,71 +73,73 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
go func(video typesdefs.MediaToProcess) { go func(video typesdefs.MediaToProcess) {
defer wg.Done() defer wg.Done()
defer func() { <-sem }() // release semaphore defer func() { <-sem }() // release semaphore
if isDryRun {
log.Printf("Dry run. video processing: %s_%d", video.Codice, video.Order)
return
}
log.Printf("%s_%d: elaborating", video.Codice, video.Order)
videoPath := filepath.Join(config.Cfg.Videos.Path, fmt.Sprintf("%s_%d%s", video.Codice, video.Order, filepath.Ext(video.Url))) videoPath := filepath.Join(config.Cfg.Videos.Path, fmt.Sprintf("%s_%d%s", video.Codice, video.Order, filepath.Ext(video.Url)))
log.Printf("%s_%d: download", video.Codice, video.Order)
err := downloadVideo(video.Url, videoPath) err := downloadVideo(video.Url, videoPath)
if err != nil { if err != nil {
log.Printf("Failed to download video %s: %v", video.Url, err) log.Printf("Failed to download video %s: %v", video.Url, err)
return // Exit goroutine on download failure return // Exit goroutine on download failure
} }
// Transcode and generate thumbnail
log.Printf("%s_%d: conversion", video.Codice, video.Order)
mp4Path, thumbnailPath, err := VideoConversion(videoPath) mp4Path, thumbnailPath, err := VideoConversion(videoPath)
if err != nil { if err != nil {
log.Printf("Failed to convert video %s: %v", videoPath, err) log.Printf("Failed to convert video %s: %v", videoPath, err)
return return
} }
mu.Lock()
ProcessedVideos = append(ProcessedVideos, typesdefs.ProcessedMedia{
Codice: video.Codice,
Order: video.Order,
Path: mp4Path,
ThumnailPath: thumbnailPath,
OGUrl: video.Url,
})
mu.Unlock()
}(video)
}
wg.Wait()
log.Printf("Video processing completed for %d videos. Starting upload...", len(ProcessedVideos))
fmt.Println()
totalToUpload := len(ProcessedVideos) log.Printf("%s_%d: upload", video.Codice, video.Order)
log.Printf("Starting upload of %d videos with concurrency limit %d...", totalToUpload, config.Cfg.Videos.ConcurrentLimit) videoId, err := queries.UploadFile(mp4Path, "video/mp4", "videos")
uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
var uploadWg sync.WaitGroup
for _, video := range ProcessedVideos {
uploadSem <- struct{}{} // acquire semaphore
uploadWg.Add(1)
go func(video typesdefs.ProcessedMedia) {
defer uploadWg.Done()
defer func() { <-uploadSem }() // release semaphore
videoId, err := queries.UploadFile(video.Path, "video/mp4", "videos")
if err != nil { if err != nil {
log.Printf("failed to upload video: %v", err) log.Printf("failed to upload video: %v", err)
return return
} }
thumbId, err := queries.UploadFile(video.ThumnailPath, "image/webp", "videos") thumbId, err := queries.UploadFile(thumbnailPath, "image/webp", "videos")
if err != nil { if err != nil {
log.Printf("failed to upload thumbnail: %v", err) log.Printf("failed to upload thumbnail: %v", err)
return return
} }
mu.Lock()
UploadedVideos = append(UploadedVideos, typesdefs.UploadedMedia{
Codice: video.Codice,
Order: video.Order,
MediaId: videoId,
ThumbId: thumbId,
OGUrl: video.OGUrl,
})
mu.Unlock() log.Printf("%s_%d: videos_refs upd", video.Codice, video.Order)
err = queries.SetMediaToDB(models.MediaRefs{
Codice: video.Codice,
Ordine: int32(video.Order),
OgURL: video.Url,
MediaType: "video",
StorageID: videoId,
IsThumbnail: false,
})
if err != nil {
log.Printf("failed to set media to db: %v", err)
return
}
err = queries.SetMediaToDB(models.MediaRefs{
Codice: video.Codice,
Ordine: int32(video.Order),
OgURL: video.Url,
MediaType: "video",
StorageID: thumbId,
IsThumbnail: true,
})
if err != nil {
log.Printf("failed to set media to db: %v", err)
return
}
log.Printf("%s_%d: done", video.Codice, video.Order)
}(video) }(video)
} }
uploadWg.Wait() wg.Wait()
log.Printf("Upload completed for %d videos.", len(UploadedVideos)) log.Println("Uploaded all videos.")
fmt.Println() return nil
return UploadedVideos, nil
} }
func clearVideoFolder() error { func clearVideoFolder() error {
@ -323,150 +325,3 @@ func extractThumbnail(input, output string) error {
return nil return nil
} }
type videRef struct {
Og_url string
Video uuid.UUID
Thumb uuid.UUID
}
func CodiciToProcessVideos(annunci *typesdefs.AnnunciXML) ([]string, error) {
currentBucketState, err := queries.GetStorageByTag("videos")
if err != nil {
return nil, fmt.Errorf("failed to list bucket: %w", err)
}
//make a list of bucket ids
bucketIdsMap := make(map[uuid.UUID]bool)
for _, item := range currentBucketState {
bucketIdsMap[item.ID] = true
}
db_videos, err := queries.GetVideosFromDB()
if err != nil {
return nil, err
}
dbVideosMap := make(map[string][]videRef) // Convert slice to map for faster lookup
for _, ref := range db_videos {
dbVideosMap[ref.Codice] = append(dbVideosMap[ref.Codice], videRef{
Og_url: ref.OgURL,
Video: ref.Video,
Thumb: ref.Thumb,
})
}
var codiciToProcess []string
for _, annuncio := range annunci.Annuncio {
if annuncio.Codice != nil && annuncio.AMMedias != nil {
mediaUrls := []string{}
for _, media := range annuncio.AMMedias.AMVideo {
if media.Url != nil {
if strings.Contains(*media.Url, "video.miogest") {
mediaUrls = append(mediaUrls, *media.Url)
}
}
}
if len(mediaUrls) == 0 {
continue
}
refVideos, existsInDb := dbVideosMap[*annuncio.Codice]
if !existsInDb {
codiciToProcess = append(codiciToProcess, *annuncio.Codice)
continue
}
needProcess := false
for _, media := range annuncio.AMMedias.AMVideo {
if media.Url == nil {
continue
}
if !strings.Contains(*media.Url, "video.miogest") {
continue
}
found := false
for _, ref := range refVideos {
if *media.Url == ref.Og_url {
if bucketIdsMap[ref.Video] && bucketIdsMap[ref.Thumb] {
found = true
break
}
}
}
if !found {
needProcess = true
break
}
}
if needProcess {
codiciToProcess = append(codiciToProcess, *annuncio.Codice)
}
}
}
return codiciToProcess, nil
}
func ForceClearVideos(codici []string) error {
var idsToDelete uuid.UUIDs
for _, codice := range codici {
refs, err := queries.GetCodiceVideosFromDB(codice)
if err != nil {
return fmt.Errorf("failed to get videos by codice %s: %w", codice, err)
}
for _, ref := range refs {
idsToDelete = append(idsToDelete, ref.Video)
idsToDelete = append(idsToDelete, ref.Thumb)
}
}
err := queries.DeleteMultFile(idsToDelete)
if err != nil {
return fmt.Errorf("failed to delete videos from storage: %w", err)
}
for _, codice := range codici {
err := queries.ClearVideosRefsByCodice(codice)
if err != nil {
return fmt.Errorf("failed to clear videos by codice %s: %w", codice, err)
}
}
return nil
}
// ReconcileVideos removes stranded videos (in storage but not in DB)
func ReconcileVideos() error {
currentBucketState, err := queries.GetStorageByTag("videos")
if err != nil {
return fmt.Errorf("failed to list bucket: %w", err)
}
//make a list of bucket ids
bucketIdsMap := make(map[uuid.UUID]bool)
for _, item := range currentBucketState {
bucketIdsMap[item.ID] = true
}
db_videos, err := queries.GetVideosFromDB()
if err != nil {
return fmt.Errorf("failed to get videos from DB: %w", err)
}
var strandedIds uuid.UUIDs
for _, ref := range db_videos {
if !bucketIdsMap[ref.Video] {
strandedIds = append(strandedIds, ref.Video)
}
if !bucketIdsMap[ref.Thumb] {
strandedIds = append(strandedIds, ref.Thumb)
}
}
if len(strandedIds) > 0 {
err := queries.DeleteMultFile(strandedIds)
if err != nil {
return fmt.Errorf("failed to delete stranded videos from storage: %w", err)
}
}
return nil
}

68
apps/db/migrations/57_mediatable.up.sql Normal file
View file

@ -0,0 +1,68 @@
-- Media Refs
CREATE TABLE IF NOT EXISTS public.media_refs (
codice TEXT NOT NULL,
ordine INTEGER NOT NULL,
storage_id UUID NOT NULL,
og_url TEXT NOT NULL,
media_type TEXT NOT NULL CHECK (media_type IN ('image', 'video')),
is_thumbnail BOOLEAN NOT NULL DEFAULT FALSE
);
-- Indexes
CREATE INDEX IF NOT EXISTS idx_media_refs_codice ON public.media_refs (codice);
CREATE INDEX IF NOT EXISTS idx_media_refs_storage_id ON public.media_refs (storage_id);
-- Unique Codice and Ordine
DO $$ BEGIN IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'media_refs_unique'
) THEN
ALTER TABLE public.media_refs
ADD CONSTRAINT media_refs_unique UNIQUE (codice, ordine, is_thumbnail, media_type);
END IF;
END $$;
-- Foreign Key to Annunci
DO $$ BEGIN IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'media_refs_annuncio'
) THEN
ALTER TABLE public.media_refs
ADD CONSTRAINT media_refs_annuncio FOREIGN KEY (codice) REFERENCES public.annunci (codice) ON UPDATE CASCADE ON DELETE CASCADE;
END IF;
END $$;
--Foreign Key to Storage
DO $$ BEGIN IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'media_refs_storage'
) THEN
ALTER TABLE public.media_refs
ADD CONSTRAINT media_refs_storage FOREIGN KEY (storage_id) REFERENCES public.storage (id) ON UPDATE CASCADE ON DELETE CASCADE;
END IF;
END $$;
DO $$
BEGIN
IF EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = 'media_refs'
) THEN
EXECUTE '
DELETE FROM public.storage s
WHERE s.tag IS NOT NULL
AND NOT EXISTS (
SELECT 1
FROM public.media_refs m
WHERE s.id = m.storage_id
);
';
END IF;
END $$;
-- Drop Tables if they exist
DROP TABLE IF EXISTS public.images_refs;
DROP TABLE IF EXISTS public.videos_refs;

102
apps/infoalloggi/src/components/annuncio_card.tsx
View file

@ -30,9 +30,11 @@ import { camereTesti, handleConsegna } from "~/lib/annuncio_details";
import { getStorageUrl } from "~/lib/storage_utils"; import { getStorageUrl } from "~/lib/storage_utils";
import { cn, formatCurrency } from "~/lib/utils"; import { cn, formatCurrency } from "~/lib/utils";
import { useTranslation } from "~/providers/I18nProvider"; import { useTranslation } from "~/providers/I18nProvider";
import type { ImagesRefs } from "~/schemas/public/ImagesRefs"; import type {
import type { VideosRefs } from "~/schemas/public/VideosRefs"; AnnuncioRicerca,
import type { AnnuncioRicerca } from "~/server/controllers/annunci.controller"; ImgMedia,
VideoMedia,
} from "~/server/controllers/annunci.controller";
import { Badge } from "./ui/badge"; import { Badge } from "./ui/badge";
import { Skeleton } from "./ui/skeleton"; import { Skeleton } from "./ui/skeleton";
import { VideoPlayer } from "./videoPlayer"; import { VideoPlayer } from "./videoPlayer";
@ -217,13 +219,17 @@ export const CardAnnuncio = ({
<CarouselItem key={`${img.img}key`}> <CarouselItem key={`${img.img}key`}>
<Image <Image
alt={t.card.alt_immagine} alt={t.card.alt_immagine}
blurDataURL={getStorageUrl({ blurDataURL={
id: img.thumb, img.thumb
params: { ? getStorageUrl({
cacheKey: media_updated_at?.toISOString(), id: img.thumb,
media: "image", params: {
}, cacheKey: media_updated_at?.toISOString(),
})} media: "image",
},
})
: undefined
}
className={cn( className={cn(
"h-64 w-full bg-[#e6e9ec] object-cover", "h-64 w-full bg-[#e6e9ec] object-cover",
(stato === "Trattativa" || homepage) && "rounded-[7px]", (stato === "Trattativa" || homepage) && "rounded-[7px]",
@ -253,13 +259,17 @@ export const CardAnnuncio = ({
> >
<VideoPlayer <VideoPlayer
className="h-64 object-cover" className="h-64 object-cover"
coverImage={getStorageUrl({ coverImage={
id: video.thumb, video.thumb
params: { ? getStorageUrl({
cacheKey: media_updated_at?.toISOString(), id: video.thumb,
media: "image", params: {
}, cacheKey: media_updated_at?.toISOString(),
})} media: "image",
},
})
: "/fallback-video.png"
}
key={`videoplayer-${video}`} key={`videoplayer-${video}`}
videoSrc={getStorageUrl({ videoSrc={getStorageUrl({
id: video.video, id: video.video,
@ -299,13 +309,17 @@ export const CardAnnuncio = ({
{images && images.length > 0 && images[0] ? ( {images && images.length > 0 && images[0] ? (
<Image <Image
alt={t.card.alt_immagine} alt={t.card.alt_immagine}
blurDataURL={getStorageUrl({ blurDataURL={
id: images[0].thumb, images[0].thumb
params: { ? getStorageUrl({
cacheKey: media_updated_at?.toISOString(), id: images[0].thumb,
media: "image", params: {
}, cacheKey: media_updated_at?.toISOString(),
})} media: "image",
},
})
: undefined
}
className={"h-64 w-full bg-[#e6e9ec] object-cover"} className={"h-64 w-full bg-[#e6e9ec] object-cover"}
height={1080} height={1080}
priority={true} priority={true}
@ -421,8 +435,8 @@ export const CarouselAnnuncio = ({
updated_at, updated_at,
}: { }: {
single?: boolean; single?: boolean;
immagini: Pick<ImagesRefs, "img" | "thumb">[]; immagini: ImgMedia[];
videos: Pick<VideosRefs, "video" | "thumb">[]; videos: VideoMedia[];
updated_at: Date | null; updated_at: Date | null;
}) => { }) => {
const [openModal, setOpenModal] = useState(false); const [openModal, setOpenModal] = useState(false);
@ -484,13 +498,17 @@ export const CarouselAnnuncio = ({
> >
<VideoPlayer <VideoPlayer
className="aspect-square max-h-92 object-cover sm:max-h-80 xl:max-h-100" className="aspect-square max-h-92 object-cover sm:max-h-80 xl:max-h-100"
coverImage={getStorageUrl({ coverImage={
id: video.thumb, video.thumb
params: { ? getStorageUrl({
cacheKey: updated_at?.toISOString(), id: video.thumb,
media: "image", params: {
}, cacheKey: updated_at?.toISOString(),
})} media: "image",
},
})
: "/fallback-video.png"
}
key={`videoplayer-${ key={`videoplayer-${
// biome-ignore lint/suspicious/noArrayIndexKey: <ok> // biome-ignore lint/suspicious/noArrayIndexKey: <ok>
idx idx
@ -561,13 +579,17 @@ export const CarouselAnnuncio = ({
> >
<VideoPlayer <VideoPlayer
className="absolute mx-auto h-[90vh] w-full max-w-fit rounded-md object-contain" className="absolute mx-auto h-[90vh] w-full max-w-fit rounded-md object-contain"
coverImage={getStorageUrl({ coverImage={
id: video.thumb, video.thumb
params: { ? getStorageUrl({
cacheKey: updated_at?.toISOString(), id: video.thumb,
media: "image", params: {
}, cacheKey: updated_at?.toISOString(),
})} media: "image",
},
})
: "/fallback-video.png"
}
videoSrc={getStorageUrl({ videoSrc={getStorageUrl({
id: video.video, id: video.video,
params: { params: {

27
apps/infoalloggi/src/forms/FormEditAnnuncio.tsx
View file

@ -1885,9 +1885,7 @@ export const AnnuncioEditForm = ({ data }: { data: AnnunciWithMedia }) => {
/> />
<Confirm <Confirm
description="Sei sicuro di voler rimuovere questo media? Questa azione è irreversibile fino al prossimo aggiornamento." description="Sei sicuro di voler rimuovere questo media? Questa azione è irreversibile fino al prossimo aggiornamento."
onConfirm={async () => onConfirm={async () => removeMedia(img.img)}
removeMedia({ type: "img", id: img.img })
}
title="Rimuovi media" title="Rimuovi media"
> >
<Button <Button
@ -1905,13 +1903,18 @@ export const AnnuncioEditForm = ({ data }: { data: AnnunciWithMedia }) => {
<div className="relative" key={video.video}> <div className="relative" key={video.video}>
<VideoPlayer <VideoPlayer
className="aspect-square size-52 object-cover" className="aspect-square size-52 object-cover"
coverImage={getStorageUrl({ coverImage={
id: video.thumb, video.thumb
params: { ? getStorageUrl({
cacheKey: data.media_updated_at?.toISOString(), id: video.thumb,
media: "image", params: {
}, cacheKey:
})} data.media_updated_at?.toISOString(),
media: "image",
},
})
: "/fallback-video.png"
}
key={`videoplayer-${video.video}`} key={`videoplayer-${video.video}`}
videoSrc={getStorageUrl({ videoSrc={getStorageUrl({
id: video.video, id: video.video,
@ -1923,9 +1926,7 @@ export const AnnuncioEditForm = ({ data }: { data: AnnunciWithMedia }) => {
/> />
<Confirm <Confirm
description="Sei sicuro di voler rimuovere questo media? Questa azione è irreversibile fino al prossimo aggiornamento." description="Sei sicuro di voler rimuovere questo media? Questa azione è irreversibile fino al prossimo aggiornamento."
onConfirm={async () => onConfirm={async () => removeMedia(video.video)}
removeMedia({ type: "video", id: video.video })
}
title="Rimuovi media" title="Rimuovi media"
> >
<Button <Button

18
apps/infoalloggi/src/pages/annuncio/[cod].tsx
View file

@ -658,13 +658,17 @@ const AnnuncioFooter = () => {
return ( return (
<VideoPlayer <VideoPlayer
className="h-96 max-w-96" className="h-96 max-w-96"
coverImage={getStorageUrl({ coverImage={
id: video.thumb, video.thumb
params: { ? getStorageUrl({
cacheKey: media_updated_at || undefined, id: video.thumb,
media: "image", params: {
}, cacheKey: media_updated_at || undefined,
})} media: "image",
},
})
: "/fallback-video.png"
}
key={`videoplayer-${video.video}`} // Cache busting key={`videoplayer-${video.video}`} // Cache busting
videoSrc={getStorageUrl({ videoSrc={getStorageUrl({
id: video.video, id: video.video,

46
apps/infoalloggi/src/schemas/public/ImagesRefs.ts
View file

@ -1,46 +0,0 @@
import { storageId, type StorageId } from './Storage';
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
import { z } from 'zod';
/** Represents the table public.images_refs */
export default interface ImagesRefsTable {
codice: ColumnType<string, string, string>;
ordine: ColumnType<number, number, number>;
img: ColumnType<StorageId, StorageId, StorageId>;
thumb: ColumnType<StorageId, StorageId, StorageId>;
og_url: ColumnType<string, string, string>;
}
export type ImagesRefs = Selectable<ImagesRefsTable>;
export type NewImagesRefs = Insertable<ImagesRefsTable>;
export type ImagesRefsUpdate = Updateable<ImagesRefsTable>;
export const ImagesRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
img: storageId,
thumb: storageId,
og_url: z.string(),
});
export const NewImagesRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
img: storageId,
thumb: storageId,
og_url: z.string(),
});
export const ImagesRefsUpdateSchema = z.object({
codice: z.string().optional(),
ordine: z.number().optional(),
img: storageId.optional(),
thumb: storageId.optional(),
og_url: z.string().optional(),
});

51
apps/infoalloggi/src/schemas/public/MediaRefs.ts Normal file
View file

@ -0,0 +1,51 @@
import { storageId, type StorageId } from './Storage';
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
import { z } from 'zod';
/** Represents the table public.media_refs */
export default interface MediaRefsTable {
codice: ColumnType<string, string, string>;
ordine: ColumnType<number, number, number>;
storage_id: ColumnType<StorageId, StorageId, StorageId>;
og_url: ColumnType<string, string, string>;
media_type: ColumnType<string, string, string>;
is_thumbnail: ColumnType<boolean, boolean | undefined, boolean>;
}
export type MediaRefs = Selectable<MediaRefsTable>;
export type NewMediaRefs = Insertable<MediaRefsTable>;
export type MediaRefsUpdate = Updateable<MediaRefsTable>;
export const MediaRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
storage_id: storageId,
og_url: z.string(),
media_type: z.string(),
is_thumbnail: z.boolean(),
});
export const NewMediaRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
storage_id: storageId,
og_url: z.string(),
media_type: z.string(),
is_thumbnail: z.boolean().optional(),
});
export const MediaRefsUpdateSchema = z.object({
codice: z.string().optional(),
ordine: z.number().optional(),
storage_id: storageId.optional(),
og_url: z.string().optional(),
media_type: z.string().optional(),
is_thumbnail: z.boolean().optional(),
});

9
apps/infoalloggi/src/schemas/public/PublicSchema.ts
View file

@ -23,14 +23,13 @@ import type { default as EtichetteTable } from './Etichette';
import type { default as MessagesTable } from './Messages'; import type { default as MessagesTable } from './Messages';
import type { default as UsersAnagraficaTable } from './UsersAnagrafica'; import type { default as UsersAnagraficaTable } from './UsersAnagrafica';
import type { default as ProvincieTable } from './Provincie'; import type { default as ProvincieTable } from './Provincie';
import type { default as VideosRefsTable } from './VideosRefs';
import type { default as BanlistTable } from './Banlist'; import type { default as BanlistTable } from './Banlist';
import type { default as NazioniTable } from './Nazioni'; import type { default as NazioniTable } from './Nazioni';
import type { default as StorageTable } from './Storage'; import type { default as StorageTable } from './Storage';
import type { default as AppuntiGroupsTable } from './AppuntiGroups'; import type { default as AppuntiGroupsTable } from './AppuntiGroups';
import type { default as ImagesRefsTable } from './ImagesRefs';
import type { default as BannersTable } from './Banners'; import type { default as BannersTable } from './Banners';
import type { default as ServizioTable } from './Servizio'; import type { default as ServizioTable } from './Servizio';
import type { default as MediaRefsTable } from './MediaRefs';
import type { default as MessagesAttachmentsTable } from './MessagesAttachments'; import type { default as MessagesAttachmentsTable } from './MessagesAttachments';
export default interface PublicSchema { export default interface PublicSchema {
@ -84,8 +83,6 @@ export default interface PublicSchema {
provincie: ProvincieTable; provincie: ProvincieTable;
videos_refs: VideosRefsTable;
banlist: BanlistTable; banlist: BanlistTable;
nazioni: NazioniTable; nazioni: NazioniTable;
@ -94,11 +91,11 @@ export default interface PublicSchema {
appunti_groups: AppuntiGroupsTable; appunti_groups: AppuntiGroupsTable;
images_refs: ImagesRefsTable;
banners: BannersTable; banners: BannersTable;
servizio: ServizioTable; servizio: ServizioTable;
media_refs: MediaRefsTable;
messages_attachments: MessagesAttachmentsTable; messages_attachments: MessagesAttachmentsTable;
} }

46
apps/infoalloggi/src/schemas/public/VideosRefs.ts
View file

@ -1,46 +0,0 @@
import { storageId, type StorageId } from './Storage';
import type { ColumnType, Selectable, Insertable, Updateable } from 'kysely';
import { z } from 'zod';
/** Represents the table public.videos_refs */
export default interface VideosRefsTable {
codice: ColumnType<string, string, string>;
ordine: ColumnType<number, number, number>;
video: ColumnType<StorageId, StorageId, StorageId>;
thumb: ColumnType<StorageId, StorageId, StorageId>;
og_url: ColumnType<string, string, string>;
}
export type VideosRefs = Selectable<VideosRefsTable>;
export type NewVideosRefs = Insertable<VideosRefsTable>;
export type VideosRefsUpdate = Updateable<VideosRefsTable>;
export const VideosRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
video: storageId,
thumb: storageId,
og_url: z.string(),
});
export const NewVideosRefsSchema = z.object({
codice: z.string(),
ordine: z.number(),
video: storageId,
thumb: storageId,
og_url: z.string(),
});
export const VideosRefsUpdateSchema = z.object({
codice: z.string().optional(),
ordine: z.number().optional(),
video: storageId.optional(),
thumb: storageId.optional(),
og_url: z.string().optional(),
});

9
apps/infoalloggi/src/server/api/routers/annunci.ts
View file

@ -1,6 +1,7 @@
import { z } from "zod/v4"; import { z } from "zod/v4";
import { AnnunciUpdateSchema, annunciId } from "~/schemas/public/Annunci"; import { AnnunciUpdateSchema, annunciId } from "~/schemas/public/Annunci";
import { servizioServizioId } from "~/schemas/public/Servizio"; import { servizioServizioId } from "~/schemas/public/Servizio";
import { storageId } from "~/schemas/public/Storage";
import { import {
adminProcedure, adminProcedure,
apiProcedure, apiProcedure,
@ -19,9 +20,7 @@ import {
getCodici_AnnunciHandler, getCodici_AnnunciHandler,
getOptions_AnnunciHandler, getOptions_AnnunciHandler,
getProprietarioDataHandler, getProprietarioDataHandler,
type ImgToRemove,
removeAnnuncioMedia, removeAnnuncioMedia,
type VideoToRemove,
} from "~/server/controllers/annunci.controller"; } from "~/server/controllers/annunci.controller";
import { getIncrociAnnuncio } from "~/server/controllers/servizio.controller"; import { getIncrociAnnuncio } from "~/server/controllers/servizio.controller";
import { db } from "~/server/db"; import { db } from "~/server/db";
@ -175,11 +174,9 @@ export const annunciRouter = createTRPCRouter({
return { status: "ok", timestamp: Date.now() }; return { status: "ok", timestamp: Date.now() };
}), }),
removeAnnuncioMedia: adminProcedure removeAnnuncioMedia: adminProcedure
.input(z.custom<ImgToRemove | VideoToRemove>()) .input(storageId)
.mutation(async ({ input }) => { .mutation(async ({ input }) => {
return await removeAnnuncioMedia({ return await removeAnnuncioMedia(input);
...input,
});
}), }),
getIncrociAnnuncio: protectedProcedure getIncrociAnnuncio: protectedProcedure
.input( .input(

2
apps/infoalloggi/src/server/api/routers/revalidation.ts
View file

@ -105,7 +105,7 @@ export const revalidationRouter = createTRPCRouter({
.mutation(async ({ input }) => { .mutation(async ({ input }) => {
try { try {
const response = await fetch( const response = await fetch(
`${env.BACKENDSERVER_URL}/update-cod/${input.cod}`, `${env.BACKENDSERVER_URL}/update?cod=${input.cod}`,
); );
if (!response.ok) { if (!response.ok) {

65
apps/infoalloggi/src/server/controllers/annunci.controller.ts
View file

@ -7,19 +7,28 @@ import type {
AnnunciId, AnnunciId,
AnnunciUpdate, AnnunciUpdate,
} from "~/schemas/public/Annunci"; } from "~/schemas/public/Annunci";
import type { ImagesRefs } from "~/schemas/public/ImagesRefs";
import type { ServizioServizioId } from "~/schemas/public/Servizio"; import type { ServizioServizioId } from "~/schemas/public/Servizio";
import type { VideosRefs } from "~/schemas/public/VideosRefs"; import type { StorageId } from "~/schemas/public/Storage";
import { db } from "~/server/db"; import { db } from "~/server/db";
import type { AnnuncioTemplateData } from "~/server/services/typst.service"; import type { AnnuncioTemplateData } from "~/server/services/typst.service";
import { withImages, withVideos } from "~/utils/kysely-helper"; import { withImages, withVideos } from "~/utils/kysely-helper";
import type { Result } from "~/utils/result"; import type { Result } from "~/utils/result";
import { revalidate } from "../utils/revalidationHelper"; import { revalidate } from "../utils/revalidationHelper";
export type ImgMedia = {
ordine: number;
img: StorageId;
thumb: StorageId | null;
};
export type VideoMedia = {
ordine: number;
video: StorageId;
thumb: StorageId | null;
};
export type AnnunciWithMedia = Annunci & { export type AnnunciWithMedia = Annunci & {
tipo: string; tipo: string;
images: Pick<ImagesRefs, "img" | "thumb">[]; images: ImgMedia[];
videos: Pick<VideosRefs, "video" | "thumb">[]; videos: VideoMedia[];
}; };
export const getAnnunciListHandler = async (): Promise< export const getAnnunciListHandler = async (): Promise<
@ -104,8 +113,8 @@ export type AnnuncioData = Pick<
> & { > & {
tipo: string; tipo: string;
media_updated_at: string | null; media_updated_at: string | null;
images: Pick<ImagesRefs, "img" | "thumb">[]; images: ImgMedia[];
videos: Pick<VideosRefs, "video" | "thumb">[]; videos: VideoMedia[];
ogUrl: string; ogUrl: string;
ogImage: string; ogImage: string;
}; };
@ -121,7 +130,7 @@ export const getAnnuncioData = async ({
}): Promise<Result<AnnuncioData, getAnnuncioDataError>> => { }): Promise<Result<AnnuncioData, getAnnuncioDataError>> => {
try { try {
const annuncio = await db const annuncio = await db
.$pickTables<"annunci" | "images_refs" | "videos_refs">() .$pickTables<"annunci" | "media_refs">()
.selectFrom("annunci") .selectFrom("annunci")
.select([ .select([
"anno", "anno",
@ -199,7 +208,7 @@ export const getAnnunciById_rawImgUrls = async ({
}): Promise<AnnunciWithMedia | null> => { }): Promise<AnnunciWithMedia | null> => {
try { try {
const annuncio = await db const annuncio = await db
.$pickTables<"annunci" | "images_refs" | "videos_refs">() .$pickTables<"annunci" | "media_refs">()
.selectFrom("annunci") .selectFrom("annunci")
.selectAll() .selectAll()
.select((_eb) => [withImages(), withVideos()]) .select((_eb) => [withImages(), withVideos()])
@ -242,7 +251,7 @@ export const get_AnnunciPositionsHandler = async ({
}): Promise<AnnuncioRicercaWPosition[]> => { }): Promise<AnnuncioRicercaWPosition[]> => {
try { try {
let query = db let query = db
.$pickTables<"annunci" | "images_refs" | "videos_refs">() .$pickTables<"annunci" | "media_refs">()
.selectFrom("annunci") .selectFrom("annunci")
.select((_eb) => [ .select((_eb) => [
"id", "id",
@ -326,8 +335,8 @@ export type AnnuncioRicerca = Pick<
| "homepage" | "homepage"
| "web" | "web"
> & { > & {
images: Pick<ImagesRefs, "img" | "thumb">[]; images: ImgMedia[];
videos: Pick<VideosRefs, "video" | "thumb">[]; videos: VideoMedia[];
}; };
export const getAnnunciRicerca = async ({ export const getAnnunciRicerca = async ({
@ -347,7 +356,7 @@ export const getAnnunciRicerca = async ({
}): Promise<AnnuncioRicerca[]> => { }): Promise<AnnuncioRicerca[]> => {
try { try {
let query = db let query = db
.$pickTables<"annunci" | "images_refs" | "videos_refs">() .$pickTables<"annunci" | "media_refs">()
.selectFrom("annunci") .selectFrom("annunci")
.select((_eb) => [ .select((_eb) => [
"id", "id",
@ -537,33 +546,13 @@ export const getProprietarioDataHandler = async ({
} }
}; };
export type ImgToRemove = { export const removeAnnuncioMedia = async (id: StorageId) => {
type: "img";
id: ImagesRefs["img"];
};
export type VideoToRemove = {
type: "video";
id: VideosRefs["video"];
};
export const removeAnnuncioMedia = async ({
id,
type,
}: ImgToRemove | VideoToRemove) => {
try { try {
if (type === "img") { await db
await db .$pickTables<"media_refs">()
.$pickTables<"images_refs">() .deleteFrom("media_refs")
.deleteFrom("images_refs") .where("storage_id", "=", id)
.where("img", "=", id) .execute();
.execute();
}
if (type === "video") {
await db
.$pickTables<"videos_refs">()
.deleteFrom("videos_refs")
.where("video", "=", id)
.execute();
}
} catch (e) { } catch (e) {
throw new TRPCError({ throw new TRPCError({
code: "INTERNAL_SERVER_ERROR", code: "INTERNAL_SERVER_ERROR",

13
apps/infoalloggi/src/server/controllers/servizio.controller.ts
View file

@ -5,7 +5,6 @@ import { z } from "zod/v4";
import { env } from "~/env"; import { env } from "~/env";
import { withParsedDates } from "~/lib/dateParserJsonKysely"; import { withParsedDates } from "~/lib/dateParserJsonKysely";
import type { Annunci, AnnunciId } from "~/schemas/public/Annunci"; import type { Annunci, AnnunciId } from "~/schemas/public/Annunci";
import type { ImagesRefs } from "~/schemas/public/ImagesRefs";
import type { Ordini, OrdiniOrdineId } from "~/schemas/public/Ordini"; import type { Ordini, OrdiniOrdineId } from "~/schemas/public/Ordini";
import type { Prezziario } from "~/schemas/public/Prezziario"; import type { Prezziario } from "~/schemas/public/Prezziario";
import type { import type {
@ -20,7 +19,7 @@ import type TipologiaPosizioneEnum from "~/schemas/public/TipologiaPosizioneEnum
import type { Users, UsersId } from "~/schemas/public/Users"; import type { Users, UsersId } from "~/schemas/public/Users";
import type { UsersAnagraficaUpdate } from "~/schemas/public/UsersAnagrafica"; import type { UsersAnagraficaUpdate } from "~/schemas/public/UsersAnagrafica";
import type { UsersStorageUserStorageId } from "~/schemas/public/UsersStorage"; import type { UsersStorageUserStorageId } from "~/schemas/public/UsersStorage";
import type { VideosRefs } from "~/schemas/public/VideosRefs";
import { import {
addEventToQueue, addEventToQueue,
canSendNotification, canSendNotification,
@ -37,7 +36,11 @@ import { getUser } from "~/server/services/user.service";
import { withImages, withVideos } from "~/utils/kysely-helper"; import { withImages, withVideos } from "~/utils/kysely-helper";
import { GetUserInterests } from "../services/interests.service"; import { GetUserInterests } from "../services/interests.service";
import { getServizioById } from "../services/servizio.service"; import { getServizioById } from "../services/servizio.service";
import type { AnnuncioRicerca } from "./annunci.controller"; import type {
AnnuncioRicerca,
ImgMedia,
VideoMedia,
} from "./annunci.controller";
export const getDataPerAcquisto = async (servizioId: ServizioServizioId) => { export const getDataPerAcquisto = async (servizioId: ServizioServizioId) => {
try { try {
@ -358,8 +361,8 @@ export type ServizioAnnuncioData = ServizioAnnunci &
| "persone" | "persone"
| "permanenza" | "permanenza"
> & { > & {
images: Pick<ImagesRefs, "img" | "thumb">[]; images: ImgMedia[];
videos: Pick<VideosRefs, "video" | "thumb">[]; videos: VideoMedia[];
}; };
export type ServizioData = Servizio & { export type ServizioData = Servizio & {
doc_personale_fronte: DocRef | null; doc_personale_fronte: DocRef | null;

2
apps/infoalloggi/src/server/services/interests.service.ts
View file

@ -103,7 +103,7 @@ export const GetUserInterestsAnnunci = async (userId: UsersId) => {
return []; return [];
} }
const annunci = await db const annunci = await db
.$pickTables<"annunci" | "images_refs" | "videos_refs">() .$pickTables<"annunci" | "media_refs">()
.selectFrom("annunci") .selectFrom("annunci")
.select((_eb) => [ .select((_eb) => [
"annunci.id", "annunci.id",

46
apps/infoalloggi/src/utils/kysely-helper.ts
View file

@ -16,10 +16,25 @@ export function withImages() {
const eb = expressionBuilder<Database, "annunci">(); const eb = expressionBuilder<Database, "annunci">();
return jsonArrayFrom( return jsonArrayFrom(
eb eb
.selectFrom("images_refs") .selectFrom("media_refs as image")
.select(["img", "thumb"]) .select([
.whereRef("images_refs.codice", "=", "annunci.codice") "image.storage_id as img",
.orderBy("images_refs.ordine", "asc"), "image.ordine",
(eb2) =>
eb2
.selectFrom("media_refs as thumb_ref")
.select("thumb_ref.storage_id")
.where("thumb_ref.is_thumbnail", "=", true)
.where("thumb_ref.media_type", "=", "image")
.whereRef("thumb_ref.ordine", "=", "image.ordine")
.whereRef("thumb_ref.codice", "=", "image.codice")
.limit(1)
.as("thumb"),
])
.whereRef("image.codice", "=", "annunci.codice")
.where("image.media_type", "=", "image")
.where("image.is_thumbnail", "=", false)
.orderBy("image.ordine", "asc"),
).as("images"); ).as("images");
} }
@ -27,10 +42,25 @@ export function withVideos() {
const eb = expressionBuilder<Database, "annunci">(); const eb = expressionBuilder<Database, "annunci">();
return jsonArrayFrom( return jsonArrayFrom(
eb eb
.selectFrom("videos_refs") .selectFrom("media_refs as video")
.select(["video", "thumb"]) .select([
.whereRef("videos_refs.codice", "=", "annunci.codice") "video.storage_id as video",
.orderBy("videos_refs.ordine", "asc"), "video.ordine",
(eb2) =>
eb2
.selectFrom("media_refs as thumb_ref")
.select("thumb_ref.storage_id")
.where("thumb_ref.is_thumbnail", "=", true)
.where("thumb_ref.media_type", "=", "video")
.whereRef("thumb_ref.ordine", "=", "video.ordine")
.whereRef("thumb_ref.codice", "=", "video.codice")
.limit(1)
.as("thumb"),
])
.whereRef("video.codice", "=", "annunci.codice")
.where("video.media_type", "=", "video")
.where("video.is_thumbnail", "=", false)
.orderBy("video.ordine", "asc"),
).as("videos"); ).as("videos");
} }