infoalloggi-monorepo/apps/backend/update.go

package main

import (
	"backend/config"
	models "backend/gen/postgres/postgres/public/model"
	"backend/parser"
	"backend/queries"
	"backend/typesdefs"
	"backend/utils"
	"context"
	"fmt"
	"log"
	"slices"
	"strings"
	"time"
)

type UpdateParams struct {
	// Dry run flag to simulate the update process without making changes.
	isDryRun bool
	// Filter by codice filter for specific updates. If empty, all records are updated.
	codFilter string
	// Force media refresh flag to re-download and upload media
	forceMediaRefresh bool
}

type Updater struct {
	isDryRun          bool
	codFilter         string
	forceMediaRefresh bool

	caratterisiche typesdefs.ListaCaratteristiche
	categorie      []typesdefs.Categoria

	imgPool   []typesdefs.MediaToProcess
	videoPool []typesdefs.MediaToProcess
	annunci   typesdefs.AnnunciParsed
}

func NewUpdater() *Updater {
	u := Updater{}
	u.init_Updater()
	return &u
}

func (u *Updater) init_Updater() {
	var err error
	u.caratterisiche, err = genCaratteristiche()
	if err != nil {
		log.Fatalf("Error generating carateristiche: %v", err)
	}

	u.categorie, err = genCategorie()
	if err != nil {
		log.Fatalf("Error generating categorie: %v", err)
	}
}

func genCaratteristiche() (typesdefs.ListaCaratteristiche, error) {
	var values typesdefs.ListaCaratteristiche
	//https://www.miogest.com/apps/revo.aspx?tipo=schede
	var result, err = utils.GetXMLFromFile[typesdefs.DefaultCaratteristiche]("./caratteristiche.xml")
	if err != nil {
		return values, err

	}
	for _, tag := range result.Scheda {
		var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml}
		values.Caratteristiche = append(values.Caratteristiche, c)
	}
	return values, nil
}

func genCategorie() ([]typesdefs.Categoria, error) {
	var values []typesdefs.Categoria
	//https://www.miogest.com/apps/revo.aspx?tipo=categorie
	var result, err = utils.GetXMLFromFile[typesdefs.DefaultCategories]("./categorie.xml")
	if err != nil {
		return values, err
	}
	for _, elem := range result.Cat {
		var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome}
		values = append(values, c)
	}
	return values, nil
}

func (u *Updater) Update(ctx context.Context, params UpdateParams) error {
	u.isDryRun = params.isDryRun
	u.forceMediaRefresh = params.forceMediaRefresh
	u.codFilter = params.codFilter
	u.annunci = typesdefs.AnnunciParsed{}
	u.imgPool = []typesdefs.MediaToProcess{}
	u.videoPool = []typesdefs.MediaToProcess{}

	var err error
	err = u.parseAnnunci()
	if err != nil {
		return err
	}
	err = queries.AnnunciInsert(u.annunci, u.codFilter == "", u.isDryRun)
	if err != nil {
		return err
	}
	err = ProcessImagePool(&u.imgPool, u.isDryRun)
	if err != nil {
		return err
	}

	err = ProcessVideoPool(&u.videoPool, u.isDryRun)
	if err != nil {
		return err
	}
	log.Println("Update completed")

	return nil
}

func (u *Updater) parseAnnunci() error {
	var err error

	xml, err := getAnnunciMiogest()
	if err != nil {
		return err
	}

	// filter by codFilter if provided
	if u.codFilter != "" {
		idx := slices.IndexFunc(xml.Annuncio, func(a typesdefs.AnnuncioXML) bool {
			return *a.Codice == u.codFilter
		})
		if idx == -1 {
			return fmt.Errorf("Codice Filtering: Annuncio con codice %s not found", u.codFilter)
		}
		xml.Annuncio = []typesdefs.AnnuncioXML{xml.Annuncio[idx]}
	}
	u.setupMediaWorkload(&xml)

	for i := range xml.Annuncio {
		u.annunci.Annuncio = append(u.annunci.Annuncio, u.extractAnnuncioData(&xml.Annuncio[i]))
	}

	return nil
}

func getAnnunciMiogest() (typesdefs.AnnunciXML, error) {
	var result typesdefs.AnnunciXML
	var err error
	result, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml")
	if err != nil {
		return result, fmt.Errorf("Error fetching data from Miogest: %w", err)
	}
	if len(result.Annuncio) == 0 {
		return result, fmt.Errorf("No annunci found in the XML data")
	}
	for i, d := range result.Annuncio {
		sanified := strings.ReplaceAll(*d.Codice, " ", "")
		result.Annuncio[i].Codice = &sanified
	}

	return result, nil

}

func (u *Updater) setupMediaWorkload(xml *typesdefs.AnnunciXML) {

	if u.forceMediaRefresh && (config.Cfg.Images.Enabled || config.Cfg.Videos.Enabled) {
		for i := range xml.Annuncio {
			xml.Annuncio[i].ProcessingImages = config.Cfg.Images.Enabled
			xml.Annuncio[i].ProcessingVideos = config.Cfg.Videos.Enabled
		}
		return
	}

	db_media, err := queries.GetMediaFromDB()
	if err != nil {
		log.Fatalf("failed to get media from db: %v", err)
		return
	}
	for i, a := range xml.Annuncio {
		if a.Codice == nil {
			continue
		}
		var in_db []models.MediaRefs
		for _, v := range db_media {
			if v.Codice == *a.Codice {
				in_db = append(in_db, v)
			}
		}

		if len(in_db) == 0 {
			// not found in db
			if config.Cfg.Images.Enabled {
				xml.Annuncio[i].ProcessingImages = true
			}
			if config.Cfg.Videos.Enabled {
				xml.Annuncio[i].ProcessingVideos = true
			}
			continue
		}
		if config.Cfg.Images.Enabled {

			img_process := false
			for _, fotoUrl := range *a.Foto {
				found_img := false
				found_thumb := false

				for _, ref := range in_db {
					if found_img && found_thumb {
						break
					}
					if fotoUrl == ref.OgURL {
						if ref.IsThumbnail {
							found_thumb = true
							continue
						}
						found_img = true
					}
				}
				if !found_img || !found_thumb {
					img_process = true
					break
				}
			}

			xml.Annuncio[i].ProcessingImages = img_process
		}
		if config.Cfg.Videos.Enabled {
			vid_process := false
			for _, media := range a.AMMedias.AMVideo {

				if media.Url == nil {
					continue
				}
				if !strings.Contains(*media.Url, "video.miogest") {
					continue
				}

				found_vid := false
				found_thumb := false
				for _, ref := range in_db {
					if found_vid && found_thumb {
						break
					}
					if *media.Url == ref.OgURL {
						if ref.IsThumbnail {
							found_thumb = true
							continue
						}
						found_vid = true
					}
				}
				if !found_vid || !found_thumb {

					vid_process = true
					break
				}

			}
			xml.Annuncio[i].ProcessingImages = vid_process
		}
	}

	if u.codFilter != "" {
		for i := range xml.Annuncio {

			if config.Cfg.Images.Enabled {
				xml.Annuncio[i].ProcessingImages = true
			}
			if config.Cfg.Videos.Enabled {
				xml.Annuncio[i].ProcessingImages = true
			}
		}
	}

}

func (u *Updater) extractAnnuncioData(annuncio *typesdefs.AnnuncioXML) typesdefs.AnnuncioParsed {
	tmp := typesdefs.AnnuncioParsed{}

	tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice))
	processLocatore(&tmp, annuncio)
	processIndirizzo(&tmp, annuncio)

	tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2)
	tmp.Consegna = parser.ParseConsegna(annuncio.Consegna)
	processCategorie(&tmp, annuncio, &u.categorie)
	processDatiImmobile(&tmp, annuncio)
	tmp.Stato = annuncio.Stato

	tmp.Homepage = boolUpd(false)
	if annuncio.HomePage != nil {
		if *annuncio.HomePage == "si" {

			tmp.Homepage = boolUpd(true)
		}
	}
	layout := "20060102150405"
	if annuncio.Creato != nil {
		t, err := time.Parse(layout, *annuncio.Creato)
		if err != nil {
			tmp.Creato_il = nil
		} else {
			tmp.Creato_il = &t
		}
	}
	if annuncio.Modifica != nil {
		t, err := time.Parse(layout, *annuncio.Modifica)
		if err != nil {
			tmp.Modificato_il = nil
		} else {
			tmp.Modificato_il = &t
		}
	}
	tmp.Titolo_it = annuncio.Titolo
	tmp.Titolo_en = annuncio.TitoloEn
	tmp.Desc_it = annuncio.Descrizione
	tmp.Desc_en = annuncio.DescrizioneEn

	if annuncio.Dettaglio != nil {
		disp, parm, pers, err := parser.ParseDettaglio(*annuncio.Dettaglio, tmp.Consegna)
		if err == nil {
			tmp.Disponibile_da = disp
			tmp.Permanenza = parm
			tmp.Persone = pers

		}
	}

	tmp.Web = boolUpd(true)

	if annuncio.ProcessingVideos {
		AddToVideoProcessing(&tmp, annuncio, &u.videoPool)
	}

	if annuncio.Video != nil {
		ext_videos := []string{}
		for _, v := range *annuncio.Video {
			if strings.Contains(v, "youtu") {
				ext_videos = append(ext_videos, v)
			}
			tmp.External_videos = &ext_videos
		}
	} else {
		tmp.External_videos = &[]string{}
	}

	if annuncio.Accessori != nil {
		tmp.Accessori = annuncio.Accessori
	}
	tmpcaratt := parseCaratteristiche(annuncio, &u.caratterisiche)
	tmp.Caratteristiche = &tmpcaratt

	if annuncio.ProcessingImages {
		AddToImageProcessing(&tmp, annuncio, &u.imgPool)
	}
	return tmp
}