package main import ( "backend/config" models "backend/gen/postgres/postgres/public/model" "backend/parser" "backend/queries" "backend/typesdefs" "backend/utils" "context" "fmt" "log" "slices" "strings" "time" ) type UpdateParams struct { // Dry run flag to simulate the update process without making changes. isDryRun bool // Filter by codice filter for specific updates. If empty, all records are updated. codFilter string // Force media refresh flag to re-download and upload media forceMediaRefresh bool } type Updater struct { isDryRun bool codFilter string forceMediaRefresh bool caratterisiche typesdefs.ListaCaratteristiche categorie []typesdefs.Categoria imgPool []typesdefs.MediaToProcess videoPool []typesdefs.MediaToProcess annunci typesdefs.AnnunciParsed } func NewUpdater() *Updater { u := Updater{} u.init_Updater() return &u } func (u *Updater) init_Updater() { var err error u.caratterisiche, err = genCaratteristiche() if err != nil { log.Fatalf("Error generating carateristiche: %v", err) } u.categorie, err = genCategorie() if err != nil { log.Fatalf("Error generating categorie: %v", err) } } func genCaratteristiche() (typesdefs.ListaCaratteristiche, error) { var values typesdefs.ListaCaratteristiche //https://www.miogest.com/apps/revo.aspx?tipo=schede var result, err = utils.GetXMLFromFile[typesdefs.DefaultCaratteristiche]("./caratteristiche.xml") if err != nil { return values, err } for _, tag := range result.Scheda { var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml} values.Caratteristiche = append(values.Caratteristiche, c) } return values, nil } func genCategorie() ([]typesdefs.Categoria, error) { var values []typesdefs.Categoria //https://www.miogest.com/apps/revo.aspx?tipo=categorie var result, err = utils.GetXMLFromFile[typesdefs.DefaultCategories]("./categorie.xml") if err != nil { return values, err } for _, elem := range result.Cat { var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome} values = append(values, c) } return values, nil } func (u *Updater) Update(ctx context.Context, params UpdateParams) error { u.isDryRun = params.isDryRun u.forceMediaRefresh = params.forceMediaRefresh u.codFilter = params.codFilter u.annunci = typesdefs.AnnunciParsed{} u.imgPool = []typesdefs.MediaToProcess{} u.videoPool = []typesdefs.MediaToProcess{} var err error err = u.parseAnnunci() if err != nil { return err } err = queries.AnnunciInsert(u.annunci, u.codFilter == "", u.isDryRun) if err != nil { return err } err = ProcessImagePool(&u.imgPool, u.isDryRun) if err != nil { return err } err = ProcessVideoPool(&u.videoPool, u.isDryRun) if err != nil { return err } log.Println("Update completed") return nil } func (u *Updater) parseAnnunci() error { var err error xml, err := getAnnunciMiogest() if err != nil { return err } // filter by codFilter if provided if u.codFilter != "" { idx := slices.IndexFunc(xml.Annuncio, func(a typesdefs.AnnuncioXML) bool { return *a.Codice == u.codFilter }) if idx == -1 { return fmt.Errorf("Codice Filtering: Annuncio con codice %s not found", u.codFilter) } xml.Annuncio = []typesdefs.AnnuncioXML{xml.Annuncio[idx]} } u.setupMediaWorkload(&xml) for i := range xml.Annuncio { u.annunci.Annuncio = append(u.annunci.Annuncio, u.extractAnnuncioData(&xml.Annuncio[i])) } return nil } func getAnnunciMiogest() (typesdefs.AnnunciXML, error) { var result typesdefs.AnnunciXML var err error result, err = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml") if err != nil { return result, fmt.Errorf("Error fetching data from Miogest: %w", err) } if len(result.Annuncio) == 0 { return result, fmt.Errorf("No annunci found in the XML data") } for i, d := range result.Annuncio { sanified := strings.ReplaceAll(*d.Codice, " ", "") result.Annuncio[i].Codice = &sanified } return result, nil } func (u *Updater) setupMediaWorkload(xml *typesdefs.AnnunciXML) { if u.forceMediaRefresh && (config.Cfg.Images.Enabled || config.Cfg.Videos.Enabled) { for i := range xml.Annuncio { xml.Annuncio[i].ProcessingImages = config.Cfg.Images.Enabled xml.Annuncio[i].ProcessingVideos = config.Cfg.Videos.Enabled } return } db_media, err := queries.GetMediaFromDB() if err != nil { log.Fatalf("failed to get media from db: %v", err) return } for i, a := range xml.Annuncio { if a.Codice == nil { continue } var in_db []models.MediaRefs for _, v := range db_media { if v.Codice == *a.Codice { in_db = append(in_db, v) } } if len(in_db) == 0 { // not found in db if config.Cfg.Images.Enabled { xml.Annuncio[i].ProcessingImages = true } if config.Cfg.Videos.Enabled { xml.Annuncio[i].ProcessingVideos = true } continue } if config.Cfg.Images.Enabled { img_process := false for _, fotoUrl := range *a.Foto { found_img := false found_thumb := false for _, ref := range in_db { if found_img && found_thumb { break } if fotoUrl == ref.OgURL { if ref.IsThumbnail { found_thumb = true continue } found_img = true } } if !found_img || !found_thumb { img_process = true break } } xml.Annuncio[i].ProcessingImages = img_process } if config.Cfg.Videos.Enabled { vid_process := false for _, media := range a.AMMedias.AMVideo { if media.Url == nil { continue } if !strings.Contains(*media.Url, "video.miogest") { continue } found_vid := false found_thumb := false for _, ref := range in_db { if found_vid && found_thumb { break } if *media.Url == ref.OgURL { if ref.IsThumbnail { found_thumb = true continue } found_vid = true } } if !found_vid || !found_thumb { vid_process = true break } } xml.Annuncio[i].ProcessingImages = vid_process } } if u.codFilter != "" { for i := range xml.Annuncio { if config.Cfg.Images.Enabled { xml.Annuncio[i].ProcessingImages = true } if config.Cfg.Videos.Enabled { xml.Annuncio[i].ProcessingImages = true } } } } func (u *Updater) extractAnnuncioData(annuncio *typesdefs.AnnuncioXML) typesdefs.AnnuncioParsed { tmp := typesdefs.AnnuncioParsed{} tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice)) processLocatore(&tmp, annuncio) processIndirizzo(&tmp, annuncio) tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2) tmp.Consegna = parser.ParseConsegna(annuncio.Consegna) processCategorie(&tmp, annuncio, &u.categorie) processDatiImmobile(&tmp, annuncio) tmp.Stato = annuncio.Stato tmp.Homepage = boolUpd(false) if annuncio.HomePage != nil { if *annuncio.HomePage == "si" { tmp.Homepage = boolUpd(true) } } layout := "20060102150405" if annuncio.Creato != nil { t, err := time.Parse(layout, *annuncio.Creato) if err != nil { tmp.Creato_il = nil } else { tmp.Creato_il = &t } } if annuncio.Modifica != nil { t, err := time.Parse(layout, *annuncio.Modifica) if err != nil { tmp.Modificato_il = nil } else { tmp.Modificato_il = &t } } tmp.Titolo_it = annuncio.Titolo tmp.Titolo_en = annuncio.TitoloEn tmp.Desc_it = annuncio.Descrizione tmp.Desc_en = annuncio.DescrizioneEn if annuncio.Dettaglio != nil { disp, parm, pers, err := parser.ParseDettaglio(*annuncio.Dettaglio, tmp.Consegna) if err == nil { tmp.Disponibile_da = disp tmp.Permanenza = parm tmp.Persone = pers } } tmp.Web = boolUpd(true) if annuncio.ProcessingVideos { AddToVideoProcessing(&tmp, annuncio, &u.videoPool) } if annuncio.Video != nil { ext_videos := []string{} for _, v := range *annuncio.Video { if strings.Contains(v, "youtu") { ext_videos = append(ext_videos, v) } tmp.External_videos = &ext_videos } } else { tmp.External_videos = &[]string{} } if annuncio.Accessori != nil { tmp.Accessori = annuncio.Accessori } tmpcaratt := parseCaratteristiche(annuncio, &u.caratterisiche) tmp.Caratteristiche = &tmpcaratt if annuncio.ProcessingImages { AddToImageProcessing(&tmp, annuncio, &u.imgPool) } return tmp }