From 1ac2d4b3abe04fe77b93cb7577640768ac8c4840 Mon Sep 17 00:00:00 2001 From: Marco Pedone Date: Fri, 22 Aug 2025 12:57:33 +0200 Subject: [PATCH] Refactor storage handlers to use centralized configuration - Updated `fs.go` and `minio.go` to replace hardcoded storage paths with values from the new config package. - Introduced a new `config` package to manage application configuration, including database and storage settings. - Removed the `models.go` file from `typedefs` as it was no longer needed. - Added a new `config.go` file to handle loading environment variables and application settings. - Created a new `types.go` file in `typedefs` to define storage strategies. - Updated `storage_handler.go` to utilize the new configuration structure. --- apps/backend/config/config.go | 137 ++++++ apps/backend/db/handler.go | 2 +- apps/backend/from_bkp.go | 20 +- apps/backend/image_handler.go | 17 +- apps/backend/images.go | 91 ---- apps/backend/miogest_handler.go | 103 ++--- apps/backend/queries/annunci.go | 6 +- apps/backend/queries/images.go | 8 +- apps/backend/queries/storage.go | 6 +- apps/backend/server.go | 409 +++++++++++------- apps/backend/storage.go | 107 +---- apps/backend/storage_handlers/fs.go | 11 +- apps/backend/storage_handlers/minio.go | 11 +- .../storage_handlers/storage_handler.go | 22 +- .../backend/{typedefs => typesdefs}/models.go | 2 +- apps/backend/typesdefs/types.go | 8 + 16 files changed, 515 insertions(+), 445 deletions(-) create mode 100644 apps/backend/config/config.go rename apps/backend/{typedefs => typesdefs}/models.go (99%) create mode 100644 apps/backend/typesdefs/types.go diff --git a/apps/backend/config/config.go b/apps/backend/config/config.go new file mode 100644 index 0000000..b013ce5 --- /dev/null +++ b/apps/backend/config/config.go @@ -0,0 +1,137 @@ +package config + +import ( + "backend/typesdefs" + "fmt" + "os" + "path/filepath" + "strconv" + "sync" + + "github.com/joho/godotenv" +) + +// Config holds all application configuration +type Config struct { + Database struct { + Host string + Port int + User string + Password string + DBName string + SSLMode string + } + + Images struct { + Enabled bool + ConcurrentLimit int + Path string + } + Storage struct { + Strategy typesdefs.StorageStrategy + Path string + TempLimit int + Minio struct { + URL string + RootUser string + Password string + } + } + + Server struct { + Port int + } +} + +var ( + Cfg Config + once sync.Once +) + +// Load initializes configuration once +func Load() *Config { + + once.Do(func() { + // Try loading from .env file first + _ = godotenv.Load() + + pwd, err := os.Getwd() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // Database config + Cfg.Database.Host = getEnv("PGHOST", "localhost") + Cfg.Database.Port = getEnvAsInt("PGPORT", 5432) + Cfg.Database.User = getEnv("POSTGRES_USER", "postgres") + Cfg.Database.Password = getEnv("POSTGRES_PASSWORD", "") + Cfg.Database.DBName = getEnv("POSTGRES_DB", "postgres") + Cfg.Database.SSLMode = getEnv("PGSSLMODE", "true") + + // Image processing config + Cfg.Images.Enabled = getEnvAsBool("IMAGEOPTION", true) + Cfg.Images.ConcurrentLimit = getEnvAsInt("CONCURRENT_IMAGES", 5) + Cfg.Images.Path = filepath.Join(pwd, "images") + + // Storage config + Cfg.Storage.Path = filepath.Join(pwd, "storage") + Cfg.Storage.TempLimit = getEnvAsInt("TEMP_STORAGE_LIMIT", 5) + strategy := getEnv("STORAGE_STRATEGY", "fs") + + if strategy == "s3" || strategy == "minio" { + Cfg.Storage.Minio.URL = getEnv("MINIO_URL", "") + Cfg.Storage.Minio.RootUser = getEnv("MINIO_ROOT_USER", "") + Cfg.Storage.Minio.Password = getEnv("MINIO_ROOT_PASSWORD", "") + + if Cfg.Storage.Minio.URL == "" || Cfg.Storage.Minio.RootUser == "" || Cfg.Storage.Minio.Password == "" { + fmt.Println("Error: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD and MINIO_URL must be set") + Cfg.Storage.Strategy = typesdefs.FS + } else { + Cfg.Storage.Strategy = typesdefs.S3 + } + } else { + Cfg.Storage.Strategy = typesdefs.FS + } + + // Server config + Cfg.Server.Port = getEnvAsInt("PORT", 1323) + + if os.Getenv("ASPNETCORE_PORT") != "" { + Cfg.Server.Port = getEnvAsInt("ASPNETCORE_PORT", 1323) + } + + // Validate required config + if Cfg.Database.Password == "" { + fmt.Printf("required environment variable POSTGRES_PASSWORD not set\n") + } + }) + + return &Cfg +} + +// Helper functions +func getEnv(key, fallback string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + return fallback +} + +func getEnvAsInt(key string, fallback int) int { + if strValue := getEnv(key, ""); strValue != "" { + if value, err := strconv.Atoi(strValue); err == nil { + return value + } + } + return fallback +} + +func getEnvAsBool(key string, fallback bool) bool { + if strValue := getEnv(key, ""); strValue != "" { + if value, err := strconv.ParseBool(strValue); err == nil { + return value + } + } + return fallback +} diff --git a/apps/backend/db/handler.go b/apps/backend/db/handler.go index 7a3c25f..43ce2fa 100644 --- a/apps/backend/db/handler.go +++ b/apps/backend/db/handler.go @@ -33,7 +33,7 @@ func NewDbHandler(configs Configs) *DbHandler { } connStr := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s?sslmode=disable", configs.User, configs.Password, configs.Host, configs.Port, configs.Dbname) - fmt.Printf("Connecting to db with connection string: %s", connStr) + fmt.Printf("Db connection string: %s\n", connStr) config, err := pgxpool.ParseConfig(connStr) if err != nil { panic(err) diff --git a/apps/backend/from_bkp.go b/apps/backend/from_bkp.go index ac90747..601aecc 100644 --- a/apps/backend/from_bkp.go +++ b/apps/backend/from_bkp.go @@ -2,7 +2,7 @@ package main import ( "backend/parser" - "backend/typedefs" + "backend/typesdefs" "backend/utils" "encoding/xml" "fmt" @@ -13,7 +13,7 @@ import ( "time" ) -func From_Backup() typedefs.XmlBkp { +func From_Backup() typesdefs.XmlBkp { //open the bkp.xml file xmlFile, err := os.Open("bkp.xml") if err != nil { @@ -25,20 +25,20 @@ func From_Backup() typedefs.XmlBkp { if err != nil { fmt.Println(err) } - var miogest typedefs.XmlBkp + var miogest typesdefs.XmlBkp xml.Unmarshal(byteValue, &miogest) return miogest } -func ParseUpdateBkp() typedefs.AnnunciParsed { +func ParseUpdateBkp(m *MiogestHandler) typesdefs.AnnunciParsed { miogest := From_Backup() - var AnnunciArray typedefs.AnnunciParsed - extractedData := make(chan typedefs.AnnuncioParsed) - defaultCaratteristiche := Miogest.GetCaratteristiche() - defaultCategorie := Miogest.GetCategorie() + var AnnunciArray typesdefs.AnnunciParsed + extractedData := make(chan typesdefs.AnnuncioParsed) + defaultCaratteristiche := m.GetCaratteristiche() + defaultCategorie := m.GetCategorie() var wg sync.WaitGroup wg.Add(len(miogest.Annunci.Record)) @@ -62,9 +62,9 @@ func ParseUpdateBkp() typedefs.AnnunciParsed { return AnnunciArray } -func extractData_bkp(annuncio typedefs.AnnuncioBKP, extractedData chan typedefs.AnnuncioParsed, defaultCaratteristiche typedefs.ListaCaratteristiche, defaultCategorie []typedefs.Categoria) { +func extractData_bkp(annuncio typesdefs.AnnuncioBKP, extractedData chan typesdefs.AnnuncioParsed, defaultCaratteristiche typesdefs.ListaCaratteristiche, defaultCategorie []typesdefs.Categoria) { - var tmp typedefs.AnnuncioParsed = typedefs.AnnuncioParsed{} + var tmp typesdefs.AnnuncioParsed = typesdefs.AnnuncioParsed{} codice := utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice)) tmp.Codice = codice diff --git a/apps/backend/image_handler.go b/apps/backend/image_handler.go index bf08e1e..c5960ce 100644 --- a/apps/backend/image_handler.go +++ b/apps/backend/image_handler.go @@ -1,8 +1,9 @@ package main import ( + "backend/config" "backend/queries" - "backend/typedefs" + "backend/typesdefs" "fmt" "log" "os" @@ -16,16 +17,12 @@ func NewImageRefManager() *ImageRefManager { return &ImageRefManager{} } -// func (i *ImageRefManager) GetImagesRef() ([]typedefs.MiogestImagesRef_Table, error) { -// return queries.ImagesGetAll() -// } - -func (i *ImageRefManager) InsertImagesRef(annunci *typedefs.AnnunciXML) error { +func (i *ImageRefManager) InsertImagesRef(annunci *typesdefs.AnnunciXML) error { return queries.ImagesBatchInsert(annunci) } -func (i *ImageRefManager) FindIsUpdate(annunci *typedefs.AnnunciXML) ([]string, error) { +func (i *ImageRefManager) FindIsUpdate(annunci *typesdefs.AnnunciXML) ([]string, error) { r, err := queries.ImagesGetAll() if err != nil { return nil, err @@ -69,7 +66,7 @@ func (i *ImageRefManager) ClearImages(to_clear []string) error { for _, codice := range to_clear { - folder := filepath.Join(imgbasepath, codice) + folder := filepath.Join(config.Cfg.Images.Path, codice) err := os.RemoveAll(folder) if err != nil { return err @@ -90,11 +87,11 @@ func (i *ImageRefManager) Reset() error { return err } - err = os.RemoveAll(imgbasepath) + err = os.RemoveAll(config.Cfg.Images.Path) if err != nil { log.Fatal(err) } - err = os.Mkdir(imgbasepath, 0755) + err = os.Mkdir(config.Cfg.Images.Path, 0755) if err != nil { log.Fatal(err) } diff --git a/apps/backend/images.go b/apps/backend/images.go index 7c1f82a..a4f8ea1 100644 --- a/apps/backend/images.go +++ b/apps/backend/images.go @@ -1,8 +1,6 @@ package main import ( - "backend/queries" - "bytes" "fmt" "image" "image/jpeg" @@ -11,7 +9,6 @@ import ( "net/http" "os" "path/filepath" - "strconv" "github.com/labstack/echo/v4" "github.com/nfnt/resize" @@ -112,94 +109,6 @@ func thumbnail_creation(img image.Image, output string, outputdir string) error return nil } -func ImageRoutes(e *echo.Echo) { - - e.GET("/initcwebp", func(c echo.Context) error { - Conversion("test.jpg") - return c.String(http.StatusOK, "Cwebp initialized") - }) - - e.GET("/images/list", func(c echo.Context) error { - annunci, err := queries.AnnunciGetActive() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - return c.JSON(http.StatusOK, annunci) - }) - - e.GET("/images/get/:cod/:imgNum", func(c echo.Context) error { - cod := c.Param("cod") - imgNum := c.Param("imgNum") - thumb := c.QueryParam("thumbnail") == "true" - widthStr := c.QueryParam("w") - //quality := c.QueryParam("q") - fmt.Printf("request for %s/%s, thumbnail=%t, w=%s\n", cod, imgNum, thumb, widthStr) - prefix := "" - if thumb { - prefix = "thumbnail-" - } - - Path := filepath.Join(imgbasepath, cod, imgNum, fmt.Sprintf("%s%s_%s.webp", prefix, cod, imgNum)) - fmt.Println(Path) - if _, err := os.Stat(Path); os.IsNotExist(err) { - - fmt.Println("Image does not exist:", Path) - // Serve a fallback image if the requested image is not found - - return FallbackImage(c) - } - if widthStr != "" { - width, err := strconv.Atoi(widthStr) - if err != nil { - fmt.Println("Invalid width parameter:", widthStr) - - return FallbackImage(c) - } - f, err := os.Open(Path) - if err != nil { - return err - } - img, err := webpbin.Decode(f) - if err != nil { - fmt.Println("Failed to decode image:", err) - return FallbackImage(c) - } - resizedImg := resize.Resize(uint(width), 0, img, resize.Lanczos3) - buffer := new(bytes.Buffer) - err = webpbin.Encode(buffer, resizedImg) - if err != nil { - fmt.Println("Failed to encode image:", err) - return FallbackImage(c) - } - return c.Blob(http.StatusOK, "image/webp", buffer.Bytes()) - } - - return c.File(Path) - - }) - - e.GET("/images/resetimages", func(c echo.Context) error { - - err := ImageManager.Reset() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - - return c.String(http.StatusOK, "OK") - }) - - e.POST("/image-option-toggle", func(c echo.Context) error { - // Toggle the imageOption value - imageOption = !imageOption - - // Convert the boolean value to a string and set the environment variable - os.Setenv("IMAGEOPTION", strconv.FormatBool(imageOption)) - - return c.String(http.StatusOK, "OK") - }) - -} - func FallbackImage(c echo.Context) error { // Serve a fallback image if the requested image is not found fallbackPath := "fallback.webp" // Path to your fallback image diff --git a/apps/backend/miogest_handler.go b/apps/backend/miogest_handler.go index 92e0931..d0bf152 100644 --- a/apps/backend/miogest_handler.go +++ b/apps/backend/miogest_handler.go @@ -1,8 +1,9 @@ package main import ( + "backend/config" "backend/parser" - "backend/typedefs" + "backend/typesdefs" "backend/utils" "fmt" "io" @@ -19,27 +20,27 @@ import ( ) type MiogestHandler struct { - Caratterisiche typedefs.ListaCaratteristiche - Categorie []typedefs.Categoria - AnnunciXML typedefs.AnnunciXML - AnnunciParsed typedefs.AnnunciParsed + Caratterisiche typesdefs.ListaCaratteristiche + Categorie []typesdefs.Categoria + AnnunciXML typesdefs.AnnunciXML + AnnunciParsed typesdefs.AnnunciParsed annunci_cache_expires time.Time vars_cache_expires time.Time } func NewMiogestHandler() *MiogestHandler { return &MiogestHandler{ - Caratterisiche: typedefs.ListaCaratteristiche{}, + Caratterisiche: typesdefs.ListaCaratteristiche{}, Categorie: nil, - AnnunciXML: typedefs.AnnunciXML{}, - AnnunciParsed: typedefs.AnnunciParsed{}, + AnnunciXML: typesdefs.AnnunciXML{}, + AnnunciParsed: typesdefs.AnnunciParsed{}, annunci_cache_expires: time.Now(), vars_cache_expires: time.Now(), } } // ANNUNCI FROM MIOGEST -func (m *MiogestHandler) GetAnnunci() typedefs.AnnunciXML { +func (m *MiogestHandler) GetAnnunci() typesdefs.AnnunciXML { if len(m.AnnunciXML.Annuncio) == 0 || time.Now().After(m.annunci_cache_expires) { m.GetAnnunciFromMiogest() } @@ -47,7 +48,7 @@ func (m *MiogestHandler) GetAnnunci() typedefs.AnnunciXML { } func (m *MiogestHandler) GetAnnunciFromMiogest() { - m.AnnunciXML = utils.GetDataXML[typedefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml") + m.AnnunciXML = utils.GetDataXML[typesdefs.AnnunciXML]("http://partner.miogest.com/agenzie/infoalloggi.xml") m.annunci_cache_expires = time.Now().Add(1 * time.Hour) } @@ -57,14 +58,14 @@ func (m *MiogestHandler) InitDefaults() { m.GenerateCategorie() } -func (m *MiogestHandler) GetCaratteristiche() typedefs.ListaCaratteristiche { +func (m *MiogestHandler) GetCaratteristiche() typesdefs.ListaCaratteristiche { if len(m.Caratterisiche.Caratteristiche) == 0 || time.Now().After(m.vars_cache_expires) { m.GenerateCaratteristiche() } return m.Caratterisiche } -func (m *MiogestHandler) GetCategorie() []typedefs.Categoria { +func (m *MiogestHandler) GetCategorie() []typesdefs.Categoria { if len(m.Categorie) == 0 || time.Now().After(m.vars_cache_expires) { m.GenerateCategorie() } @@ -72,10 +73,10 @@ func (m *MiogestHandler) GetCategorie() []typedefs.Categoria { } func (m *MiogestHandler) GenerateCaratteristiche() { - var result = utils.GetDataXML[typedefs.DefaultCaratteristiche]("https://www.miogest.com/apps/revo.aspx?tipo=schede") - var array typedefs.ListaCaratteristiche + var result = utils.GetDataXML[typesdefs.DefaultCaratteristiche]("https://www.miogest.com/apps/revo.aspx?tipo=schede") + var array typesdefs.ListaCaratteristiche for _, tag := range result.Scheda { - var c = typedefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml} + var c = typesdefs.Caratteristica{Id: tag.Id, Tagxml: tag.Tagxml} array.Caratteristiche = append(array.Caratteristiche, c) } m.Caratterisiche = array @@ -83,10 +84,10 @@ func (m *MiogestHandler) GenerateCaratteristiche() { } func (m *MiogestHandler) GenerateCategorie() { - var result = utils.GetDataXML[typedefs.DefaultCategories]("https://www.miogest.com/apps/revo.aspx?tipo=categorie") - var listaCategorie []typedefs.Categoria + var result = utils.GetDataXML[typesdefs.DefaultCategories]("https://www.miogest.com/apps/revo.aspx?tipo=categorie") + var listaCategorie []typesdefs.Categoria for _, elem := range result.Cat { - var c = typedefs.Categoria{Id: elem.ID, Nome: elem.Nome} + var c = typesdefs.Categoria{Id: elem.ID, Nome: elem.Nome} listaCategorie = append(listaCategorie, c) } m.Categorie = listaCategorie @@ -94,32 +95,32 @@ func (m *MiogestHandler) GenerateCategorie() { } // ANNUNCI PARSED -func (m *MiogestHandler) GetAnnunciParsed() typedefs.AnnunciParsed { - m.ParseAnnunci() +func (m *MiogestHandler) GetAnnunciParsed(i *ImageRefManager) typesdefs.AnnunciParsed { + m.ParseAnnunci(i) return m.AnnunciParsed } -func (m *MiogestHandler) ParseAnnunci() { +func (m *MiogestHandler) ParseAnnunci(i *ImageRefManager) { annunci := m.GetAnnunci() cod_image_to_update := []string{} - if imageOption { + if config.Cfg.Images.Enabled { var err error - cod_image_to_update, err = ImageManager.FindIsUpdate(&annunci) + cod_image_to_update, err = i.FindIsUpdate(&annunci) if err != nil { log.Fatalf("Failed to find images to update: %v", err.Error()) } - if err := ImageManager.ClearImages(cod_image_to_update); err != nil { + if err := i.ClearImages(cod_image_to_update); err != nil { log.Fatalf("Failed to clear images: %v", err.Error()) } - ImageManager.InsertImagesRef(&annunci) + i.InsertImagesRef(&annunci) } - var AnnunciArray typedefs.AnnunciParsed - AnnunciArray.Annuncio = make([]typedefs.AnnuncioParsed, 0, len(annunci.Annuncio)) - Miogest.InitDefaults() + var AnnunciArray typesdefs.AnnunciParsed + AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio)) + m.InitDefaults() var wg sync.WaitGroup wg.Add(len(annunci.Annuncio)) - sem := make(chan struct{}, concurrentImages) //Limit concurrent goroutines + sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) //Limit concurrent goroutines lista_codici := []string{} for i := range annunci.Annuncio { @@ -128,12 +129,12 @@ func (m *MiogestHandler) ParseAnnunci() { go func(i int) { defer wg.Done() defer func() { <-sem }() - result := extractData_update(&annunci.Annuncio[i], utils.Contains(cod_image_to_update, *annunci.Annuncio[i].Codice)) + result := extractData_update(&annunci.Annuncio[i], utils.Contains(cod_image_to_update, *annunci.Annuncio[i].Codice), m) AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result) }(i) } wg.Wait() - if imageOption { + if config.Cfg.Images.Enabled { for _, codice := range lista_codici { err := folderCleanup(codice) if err != nil { @@ -154,15 +155,15 @@ func strUpd(s string) *string { return &s } -func extractData_update(annuncio *typedefs.AnnuncioXML, updateImages bool) typedefs.AnnuncioParsed { - var tmp typedefs.AnnuncioParsed = typedefs.AnnuncioParsed{} +func extractData_update(annuncio *typesdefs.AnnuncioXML, updateImages bool, m *MiogestHandler) typesdefs.AnnuncioParsed { + var tmp typesdefs.AnnuncioParsed = typesdefs.AnnuncioParsed{} tmp.Codice = utils.RemoveWhitespace(utils.GetStringValue(annuncio.Codice)) processLocatore(&tmp, annuncio) processIndirizzo(&tmp, annuncio) tmp.Tipo = parser.ParseTipologia(annuncio.Tipologia, annuncio.Tipologia2) tmp.Consegna = parser.ParseConsegna(annuncio.Consegna) - processCategorie(&tmp, annuncio) + processCategorie(&tmp, annuncio, m) processDatiImmobile(&tmp, annuncio) tmp.Stato = annuncio.Stato @@ -215,7 +216,7 @@ func extractData_update(annuncio *typedefs.AnnuncioXML, updateImages bool) typed if annuncio.Accessori != nil { tmp.Accessori = annuncio.Accessori } - tmpcaratt := parseCaratteristiche(annuncio) + tmpcaratt := parseCaratteristiche(annuncio, m) tmp.Caratteristiche = &tmpcaratt tmp.Og_url = nil if len(*annuncio.Foto) != 0 { @@ -226,9 +227,9 @@ func extractData_update(annuncio *typedefs.AnnuncioXML, updateImages bool) typed if updateImages { processImages(&tmp, annuncio) } - folders := utils.GetPathDirsArray(filepath.Join(imgbasepath, tmp.Codice)) + folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, tmp.Codice)) for _, folder := range folders { - folderpath := filepath.Join(filepath.Join(imgbasepath, tmp.Codice), folder) + folderpath := filepath.Join(filepath.Join(config.Cfg.Images.Path, tmp.Codice), folder) files, err := os.ReadDir(folderpath) if err != nil { panic(err) @@ -247,7 +248,7 @@ func extractData_update(annuncio *typedefs.AnnuncioXML, updateImages bool) typed return tmp } -func processLocatore(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXML) { +func processLocatore(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { if annuncio.Clienti == nil || annuncio.Clienti.Cliente == nil { return } @@ -270,7 +271,7 @@ func processLocatore(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXM tmp.Idlocatore = annuncio.Clienti.Cliente.Id } -func processIndirizzo(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXML) { +func processIndirizzo(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { tmp.Indirizzo = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Indirizzo))) tmp.Civico = annuncio.Civico tmp.Comune = strUpd(utils.MakeUppercase(utils.GetStringValue(annuncio.Comune))) @@ -285,13 +286,13 @@ func processIndirizzo(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioX tmp.Lon_secondario = annuncio.LongitudineSecondario } -func processCategorie(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXML) { +func processCategorie(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, m *MiogestHandler) { if annuncio.Categoria == nil { return } resultArray := make([]string, 0) cats := *annuncio.Categoria - defaultCategorie := Miogest.GetCategorie() + defaultCategorie := m.GetCategorie() for _, cat := range cats { for _, catdef := range defaultCategorie { if cat == catdef.Id { @@ -305,7 +306,7 @@ func processCategorie(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioX *tmp.Categorie = resultArray } -func processDatiImmobile(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXML) { +func processDatiImmobile(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { tmpanno := utils.GetStringValue(annuncio.Anno) if tmpanno != "0" && tmpanno != "" { tmp.Anno = &tmpanno @@ -329,15 +330,15 @@ func processDatiImmobile(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.Annunc tmp.Numero_postiauto = annuncio.PostiAuto } -func processImages(tmp *typedefs.AnnuncioParsed, annuncio *typedefs.AnnuncioXML) { - path := filepath.Join(imgbasepath, tmp.Codice) +func processImages(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML) { + path := filepath.Join(config.Cfg.Images.Path, tmp.Codice) err := os.MkdirAll(path, 0755) if err != nil { - log.Printf("Failed to create directory: %v", err) + fmt.Printf("Failed to create directory: %v\n", err) panic(err) } var wg sync.WaitGroup - sem := make(chan struct{}, concurrentImages) // Limit concurrent goroutines + sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) // Limit concurrent goroutines nfoto := 0 var fotos []string if annuncio.Foto != nil { @@ -388,7 +389,7 @@ func processWebpImages(path string) { log.Fatal(err) } var wg sync.WaitGroup - sem := make(chan struct{}, concurrentImages) // Limit concurrent goroutines + sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit) // Limit concurrent goroutines for _, img := range array { wg.Add(1) sem <- struct{}{} @@ -405,10 +406,10 @@ func processWebpImages(path string) { } func folderCleanup(codice string) error { - folders := utils.GetPathDirsArray(filepath.Join(imgbasepath, codice)) + folders := utils.GetPathDirsArray(filepath.Join(config.Cfg.Images.Path, codice)) non_webp := []string{} for _, folder := range folders { - folderpath := filepath.Join(imgbasepath, codice, folder) + folderpath := filepath.Join(config.Cfg.Images.Path, codice, folder) files, err := os.ReadDir(folderpath) if err != nil { return err @@ -428,7 +429,7 @@ func folderCleanup(codice string) error { return nil } -func parseCaratteristiche(wrkrecord *typedefs.AnnuncioXML) map[string]interface{} { +func parseCaratteristiche(wrkrecord *typesdefs.AnnuncioXML, m *MiogestHandler) map[string]interface{} { if reflect.TypeOf(wrkrecord).Kind() != reflect.Ptr || reflect.ValueOf(wrkrecord).Elem().Kind() != reflect.Struct { return nil } @@ -446,7 +447,7 @@ func parseCaratteristiche(wrkrecord *typedefs.AnnuncioXML) map[string]interface{ } } } - for _, scheda := range Miogest.GetCaratteristiche().Caratteristiche { + for _, scheda := range m.GetCaratteristiche().Caratteristiche { if _, ok := tmpCaratteristiche[scheda.Tagxml]; !ok { tmpCaratteristiche[scheda.Tagxml] = nil } diff --git a/apps/backend/queries/annunci.go b/apps/backend/queries/annunci.go index 36495fa..767c358 100644 --- a/apps/backend/queries/annunci.go +++ b/apps/backend/queries/annunci.go @@ -1,7 +1,7 @@ package queries import ( - "backend/typedefs" + "backend/typesdefs" "backend/utils" "fmt" @@ -18,7 +18,7 @@ func AnnunciGetActive() ([]string, error) { return annunci, nil } -func AnnunciInsert(annunci typedefs.AnnunciParsed) error { +func AnnunciInsert(annunci typesdefs.AnnunciParsed) error { //SET ALL WEB TO FALSE _, err := Db.Dbpool.Exec(Db.Ctx, "UPDATE public.annunci SET web = false") @@ -184,7 +184,7 @@ func clearRecords() { } } -func SetFromBkp(annunci typedefs.AnnunciParsed) error { +func SetFromBkp(annunci typesdefs.AnnunciParsed) error { clearRecords() batch := &pgx.Batch{} diff --git a/apps/backend/queries/images.go b/apps/backend/queries/images.go index 115807e..3c152d8 100644 --- a/apps/backend/queries/images.go +++ b/apps/backend/queries/images.go @@ -1,15 +1,15 @@ package queries import ( - "backend/typedefs" + "backend/typesdefs" "fmt" "github.com/georgysavva/scany/v2/pgxscan" "github.com/jackc/pgx/v5" ) -func ImagesGetAll() ([]typedefs.MiogestImagesRef_Table, error) { - var images []typedefs.MiogestImagesRef_Table +func ImagesGetAll() ([]typesdefs.MiogestImagesRef_Table, error) { + var images []typesdefs.MiogestImagesRef_Table err := pgxscan.Select(Db.Ctx, Db.Dbpool, &images, "SELECT * FROM public.miogest_images_ref") if err != nil && err != pgx.ErrNoRows { return nil, fmt.Errorf("failed to get all images: %w", err) @@ -17,7 +17,7 @@ func ImagesGetAll() ([]typedefs.MiogestImagesRef_Table, error) { return images, nil } -func ImagesBatchInsert(annunci *typedefs.AnnunciXML) error { +func ImagesBatchInsert(annunci *typesdefs.AnnunciXML) error { batch := &pgx.Batch{} for _, annuncio := range annunci.Annuncio { diff --git a/apps/backend/queries/storage.go b/apps/backend/queries/storage.go index d4b8571..ed743e5 100644 --- a/apps/backend/queries/storage.go +++ b/apps/backend/queries/storage.go @@ -1,7 +1,7 @@ package queries import ( - "backend/typedefs" + "backend/typesdefs" "fmt" "time" @@ -84,9 +84,9 @@ func StorageCleanExpiredTokens(token string) error { return nil } -func StorageGetExpiredFiles() ([]typedefs.Storageindex, error) { +func StorageGetExpiredFiles() ([]typesdefs.Storageindex, error) { - var items_db []typedefs.Storageindex + var items_db []typesdefs.Storageindex err := pgxscan.Select(Db.Ctx, Db.Dbpool, &items_db, "SELECT * FROM public.storageindex WHERE expires_at is not null and expires_at < NOW()") if err != nil { return nil, fmt.Errorf("failed to get expired items: %w", err) diff --git a/apps/backend/server.go b/apps/backend/server.go index 8bafb3a..44d48a3 100644 --- a/apps/backend/server.go +++ b/apps/backend/server.go @@ -1,8 +1,10 @@ package main import ( + "backend/config" "backend/queries" storagehandlers "backend/storage_handlers" + "bytes" "fmt" "io" "log" @@ -15,118 +17,42 @@ import ( "text/template" "time" - "github.com/joho/godotenv" "github.com/labstack/echo/v4" + "github.com/nfnt/resize" + "github.com/nickalie/go-webpbin" ) -var ( - imgbasepath string - imageOption bool - concurrentImages int = 1 - storagepath string - Miogest *MiogestHandler - ImageManager *ImageRefManager - Storage storagehandlers.StorageHandler - tempStorageLimit int = 5 -) - -func main() { - // SETUP WORK DIRECTORY - pwd, err := os.Getwd() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - // join paths indipedently of the OS - imgbasepath = filepath.Join(pwd, "images") - fmt.Println("Image base path: ", imgbasepath) - storagepath = filepath.Join(pwd, "storage") - - // SETUP ENV VARIABLES - err = godotenv.Load(".env") - if err != nil { - fmt.Println("Error loading .env file") - } - if os.Getenv("IMAGEOPTION") != "" { - if os.Getenv("IMAGEOPTION") == "true" { - imageOption = true - - } else { - imageOption = false - } - } - if os.Getenv("CONCURRENT_IMAGES") != "" { - concurrentImages, err = strconv.Atoi(os.Getenv("CONCURRENT_IMAGES")) - if err != nil { - fmt.Println("Error parsing CONCURRENT_IMAGES:", err) - } - } - - // SETUP MIOGEST HANDLER - Miogest = NewMiogestHandler() - if Miogest == nil { - log.Fatal("Error connecting to miogest") - } - - // SETUP IMAGE MANAGER - ImageManager = NewImageRefManager() - if ImageManager == nil { - log.Fatal("Error creating image manager") - } - - // SETUP STORAGE HANDLER - storagehandlers.Storagepath = storagepath - var strategy storagehandlers.StorageStrategy = storagehandlers.FS - STORAGE_STRATEGY := os.Getenv("STORAGE_STRATEGY") - - if STORAGE_STRATEGY == "s3" || STORAGE_STRATEGY == "minio" { - MINIO_URL := os.Getenv("MINIO_URL") - MINIO_ROOT_USER := os.Getenv("MINIO_ROOT_USER") - MINIO_ROOT_PASSWORD := os.Getenv("MINIO_ROOT_PASSWORD") - - if MINIO_URL == "" || MINIO_ROOT_USER == "" || MINIO_ROOT_PASSWORD == "" { - fmt.Println("Error: MINIO_ROOT_USER, MINIO_ROOT_PASSWORD and MINIO_URL must be set") - strategy = storagehandlers.FS - } else { - strategy = storagehandlers.S3 - } - } - - h, err := storagehandlers.NewStorageHandler(strategy) - if err != nil { - log.Fatal("Error creating storage handler: " + err.Error()) - } - Storage = h - fmt.Printf("Storage Strategy: %s\n", strategy) - - // SETUP ECHO - e := SetupRoutes() - - // SETUP PORT - port := "1323" - if os.Getenv("ASPNETCORE_PORT") != "" { - port = os.Getenv("ASPNETCORE_PORT") - } - - fmt.Println("Image Option: ", imageOption) - fmt.Println("Concurrent Images: ", concurrentImages) - fmt.Println("Server url: http://localhost:" + port) - - Conversion("test.jpg") - - // START ECHO - e.Logger.Info(e.Start(":" + port)) +type Server struct { + miogest *MiogestHandler + imageManager *ImageRefManager + storage storagehandlers.StorageHandler + config *config.Config } -type Template struct { - templates *template.Template -} +func NewServer(cfg *config.Config) (*Server, error) { + miogest := NewMiogestHandler() + if miogest == nil { + return nil, fmt.Errorf("error connecting to miogest") + } -func (t *Template) Render(w io.Writer, name string, data any, c echo.Context) error { - return t.templates.ExecuteTemplate(w, name, data) -} + imageManager := NewImageRefManager() + if imageManager == nil { + return nil, fmt.Errorf("error creating image manager") + } -func SetupRoutes() *echo.Echo { + storage, err := storagehandlers.NewStorageHandler(cfg.Storage.Strategy) + if err != nil { + return nil, fmt.Errorf("error creating storage handler: %w", err) + } + + return &Server{ + miogest: miogest, + imageManager: imageManager, + storage: storage, + config: cfg, + }, nil +} +func (s *Server) SetupRoutes() *echo.Echo { funcs := map[string]any{ "contains": strings.Contains, "hasPrefix": strings.HasPrefix, @@ -148,8 +74,8 @@ func SetupRoutes() *echo.Echo { e.Static("/static", "public/static") e.GET("/", func(c echo.Context) error { return c.Render(http.StatusOK, "main", map[string]any{ - "imageOption": imageOption, - "concurrentImages": concurrentImages, + "imageOption": s.config.Images.Enabled, + "concurrentImages": s.config.Images.ConcurrentLimit, }) }) e.GET("/test", func(c echo.Context) error { @@ -166,15 +92,43 @@ func SetupRoutes() *echo.Echo { return c.String(http.StatusOK, fmt.Sprintf("DB Connection: %v", h)) }) - MiogestRoutes(e) - BkpRoutes(e) - StorageRoutes(e) - ImageRoutes(e) + // MIOGEST ROUTES + e.GET("/cats", func(c echo.Context) error { + var data = s.miogest.GetCaratteristiche() + return c.JSONPretty(http.StatusOK, data, " ") + }) + e.GET("/categ", func(c echo.Context) error { + var data = s.miogest.GetCategorie() + return c.JSONPretty(http.StatusOK, data, " ") + }) - return e -} + e.GET("/miogest", func(c echo.Context) error { + data := s.miogest.GetAnnunci() + return c.JSONPretty(http.StatusOK, data, " ") + }) + e.GET("/miogest-sample", func(c echo.Context) error { + xml := s.miogest.GetAnnunci() + rand := rand.Intn(len(xml.Annuncio)) + data := xml.Annuncio[rand] + return c.JSONPretty(http.StatusOK, data, " ") + }) + e.GET("/parse", func(c echo.Context) error { + start := time.Now() + var data = s.miogest.GetAnnunciParsed(s.imageManager) + end := time.Now() + fmt.Println("time taken:", end.Sub(start).String()) + return c.JSONPretty(http.StatusOK, data, " ") + }) + e.GET("/update", func(c echo.Context) error { + var data = s.miogest.GetAnnunciParsed(s.imageManager) + err := queries.AnnunciInsert(data) + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + return c.String(http.StatusOK, "Updated") + }) -func BkpRoutes(e *echo.Echo) { + // BACKUP ROUTES e.GET("/upload-backup", func(c echo.Context) error { return c.Render(http.StatusOK, "backup-upload", nil) }) @@ -203,54 +157,209 @@ func BkpRoutes(e *echo.Echo) { return c.JSONPretty(http.StatusOK, data, " ") }) e.GET("/parsebkp", func(c echo.Context) error { - var data = ParseUpdateBkp() + var data = ParseUpdateBkp(s.miogest) return c.JSONPretty(http.StatusOK, data, " ") }) e.GET("/setbkp", func(c echo.Context) error { - var data = ParseUpdateBkp() + var data = ParseUpdateBkp(s.miogest) err := queries.SetFromBkp(data) if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } return c.String(http.StatusOK, "Setted") }) -} -func MiogestRoutes(e *echo.Echo) { - // MIOGEST DEFAULTS - e.GET("/cats", func(c echo.Context) error { - var data = Miogest.GetCaratteristiche() - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/categ", func(c echo.Context) error { - var data = Miogest.GetCategorie() - return c.JSONPretty(http.StatusOK, data, " ") - }) - - // MIOGEST UPDATE - e.GET("/miogest", func(c echo.Context) error { - data := Miogest.GetAnnunci() - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/miogest-sample", func(c echo.Context) error { - xml := Miogest.GetAnnunci() - rand := rand.Intn(len(xml.Annuncio)) - data := xml.Annuncio[rand] - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/parse", func(c echo.Context) error { - start := time.Now() - var data = Miogest.GetAnnunciParsed() - end := time.Now() - fmt.Println("time taken:", end.Sub(start).String()) - return c.JSONPretty(http.StatusOK, data, " ") - }) - e.GET("/update", func(c echo.Context) error { - var data = Miogest.GetAnnunciParsed() - err := queries.AnnunciInsert(data) + //STORAGE ROUTES + e.GET("/storage/list", func(c echo.Context) error { + filenames, err := s.storage.List() if err != nil { return c.String(http.StatusInternalServerError, err.Error()) } - return c.String(http.StatusOK, "Updated") + return c.JSON(http.StatusOK, filenames) }) + + e.GET("/storage/get/:filename", func(c echo.Context) error { + err := TempFolderCleaner(s.config.Storage.TempLimit) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to clean temp folder: "+err.Error()) + } + err = StorageAuth(c) + if err != nil { + return c.JSON(http.StatusUnauthorized, "Unauthorized: "+err.Error()) + } + + filename := c.Param("filename") + path, err := s.storage.Get(filename) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to get file: "+err.Error()) + } + + return c.File(path) + }) + e.POST("/storage/upload", func(c echo.Context) error { + fmt.Println("Uploading file") + + err := StorageAuth(c) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) + } + + from_admin := c.QueryParam("from_admin") == "true" + file, err := c.FormFile("file") + if err != nil { + return err + } + + fileId, err := Upload(file, from_admin, s.storage) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to upload file: "+err.Error()) + } + + return c.String(http.StatusOK, fileId) + }) + e.DELETE("/storage/delete/:filename", func(c echo.Context) error { + err := StorageAuth(c) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) + } + filename := c.Param("filename") + + err = s.storage.Delete(filename) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to remove file / file not found: "+err.Error()) + } + return c.String(http.StatusOK, "File removed") + + }) + e.GET("/storage/remove-expired", func(c echo.Context) error { + + err := RemoveExpired(s.storage) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to remove files / files not found: "+err.Error()) + } + return c.String(http.StatusOK, "File removed") + + }) + e.GET("/storage/test", func(c echo.Context) error { + err := CheckConcruency(s.storage) + if err != nil { + return c.String(http.StatusInternalServerError, "Failed to check concruency: "+err.Error()) + } + return c.String(http.StatusOK, "OK") + }) + + //IMAGE ROUTES + e.GET("/initcwebp", func(c echo.Context) error { + Conversion("test.jpg") + return c.String(http.StatusOK, "Cwebp initialized") + }) + + e.GET("/images/list", func(c echo.Context) error { + annunci, err := queries.AnnunciGetActive() + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + return c.JSON(http.StatusOK, annunci) + }) + + e.GET("/images/get/:cod/:imgNum", func(c echo.Context) error { + cod := c.Param("cod") + imgNum := c.Param("imgNum") + thumb := c.QueryParam("thumbnail") == "true" + widthStr := c.QueryParam("w") + //quality := c.QueryParam("q") + fmt.Printf("request for %s/%s, thumbnail=%t, w=%s\n", cod, imgNum, thumb, widthStr) + prefix := "" + if thumb { + prefix = "thumbnail-" + } + + Path := filepath.Join(config.Cfg.Images.Path, cod, imgNum, fmt.Sprintf("%s%s_%s.webp", prefix, cod, imgNum)) + fmt.Println(Path) + if _, err := os.Stat(Path); os.IsNotExist(err) { + + fmt.Println("Image does not exist:", Path) + // Serve a fallback image if the requested image is not found + + return FallbackImage(c) + } + if widthStr != "" { + width, err := strconv.Atoi(widthStr) + if err != nil { + fmt.Println("Invalid width parameter:", widthStr) + + return FallbackImage(c) + } + f, err := os.Open(Path) + if err != nil { + return err + } + img, err := webpbin.Decode(f) + if err != nil { + fmt.Println("Failed to decode image:", err) + return FallbackImage(c) + } + resizedImg := resize.Resize(uint(width), 0, img, resize.Lanczos3) + buffer := new(bytes.Buffer) + err = webpbin.Encode(buffer, resizedImg) + if err != nil { + fmt.Println("Failed to encode image:", err) + return FallbackImage(c) + } + return c.Blob(http.StatusOK, "image/webp", buffer.Bytes()) + } + + return c.File(Path) + + }) + + e.GET("/images/resetimages", func(c echo.Context) error { + + err := s.imageManager.Reset() + if err != nil { + return c.String(http.StatusInternalServerError, err.Error()) + } + + return c.String(http.StatusOK, "OK") + }) + + e.POST("/image-option-toggle", func(c echo.Context) error { + // Toggle the imageOption value + s.config.Images.Enabled = !s.config.Images.Enabled + + // Convert the boolean value to a string and set the environment variable + os.Setenv("IMAGEOPTION", strconv.FormatBool(s.config.Images.Enabled)) + + return c.String(http.StatusOK, "OK") + }) + + return e +} + +func (s *Server) Start() { + e := s.SetupRoutes() + + Conversion("test.jpg") + fmt.Println("Image Option: ", s.config.Images.Enabled) + fmt.Println("Concurrent Images: ", s.config.Images.ConcurrentLimit) + fmt.Println("Server url: http://localhost:" + strconv.Itoa(s.config.Server.Port)) + + e.Logger.Info(e.Start(":" + strconv.Itoa(s.config.Server.Port))) +} + +func main() { + cfg := config.Load() + server, err := NewServer(cfg) + if err != nil { + log.Fatalf("Server initialization error: %v", err) + } + server.Start() +} + +type Template struct { + templates *template.Template +} + +func (t *Template) Render(w io.Writer, name string, data any, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) } diff --git a/apps/backend/storage.go b/apps/backend/storage.go index c88c0a3..117993e 100644 --- a/apps/backend/storage.go +++ b/apps/backend/storage.go @@ -1,9 +1,10 @@ package main import ( + "backend/config" "backend/queries" + storagehandlers "backend/storage_handlers" "fmt" - "log" "mime/multipart" "net/http" "os" @@ -15,15 +16,15 @@ import ( ) // Upload saves a file to the storage directory. -func Upload(file *multipart.FileHeader, from_admin bool) (string, error) { - log.Printf("Uploading file: %s", file.Filename) +func Upload(file *multipart.FileHeader, from_admin bool, s storagehandlers.StorageHandler) (string, error) { + fmt.Printf("Uploading file: %s\n", file.Filename) newId, err := queries.StorageNewId() if err != nil { return "", err } - err = Storage.Add(file, newId) + err = s.Add(file, newId) if err != nil { return "", err } @@ -71,7 +72,7 @@ func RemoveFromDb(ref string) error { return queries.StorageDelete(ref) } -func RemoveExpired() error { +func RemoveExpired(s storagehandlers.StorageHandler) error { //get all expired files items_db, err := queries.StorageGetExpiredFiles() if err != nil { @@ -80,7 +81,7 @@ func RemoveExpired() error { for _, item := range items_db { formattedName := item.Id + item.Ext - err = Storage.Delete(formattedName) + err = s.Delete(formattedName) if err != nil { return fmt.Errorf("failed to delete file %s: %w", formattedName, err) } @@ -93,13 +94,13 @@ func RemoveExpired() error { return nil } -func CheckConcruency() error { +func CheckConcruency(s storagehandlers.StorageHandler) error { items_db, err := queries.StorageGet_min() if err != nil { return err } - files, err := Storage.List() + files, err := s.List() if err != nil { return err } @@ -129,13 +130,13 @@ func CheckConcruency() error { return nil } -func TempFolderCleaner() error { - files, err := os.ReadDir(filepath.Join(storagepath, "/tmp/")) +func TempFolderCleaner(limit int) error { + files, err := os.ReadDir(filepath.Join(config.Cfg.Storage.Path, "/tmp/")) if err != nil { return err } fmt.Println("Temp files: ", len(files)) - if len(files) >= tempStorageLimit { + if len(files) >= limit { var oldest_Time time.Time var oldest_File string @@ -149,7 +150,7 @@ func TempFolderCleaner() error { oldest_File = file.Name() } } - err = os.Remove(filepath.Join(storagepath, "/tmp/", oldest_File)) + err = os.Remove(filepath.Join(config.Cfg.Storage.Path, "/tmp/", oldest_File)) if err != nil { return err } @@ -158,85 +159,3 @@ func TempFolderCleaner() error { return nil } - -// ROUTES -func StorageRoutes(e *echo.Echo) { - - e.GET("/storage/list", func(c echo.Context) error { - filenames, err := Storage.List() - if err != nil { - return c.String(http.StatusInternalServerError, err.Error()) - } - return c.JSON(http.StatusOK, filenames) - }) - - e.GET("/storage/get/:filename", func(c echo.Context) error { - err := TempFolderCleaner() - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to clean temp folder: "+err.Error()) - } - err = StorageAuth(c) - if err != nil { - return c.JSON(http.StatusUnauthorized, "Unauthorized: "+err.Error()) - } - - filename := c.Param("filename") - path, err := Storage.Get(filename) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to get file: "+err.Error()) - } - - return c.File(path) - }) - e.POST("/storage/upload", func(c echo.Context) error { - fmt.Println("Uploading file") - - err := StorageAuth(c) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) - } - - from_admin := c.QueryParam("from_admin") == "true" - file, err := c.FormFile("file") - if err != nil { - return err - } - - fileId, err := Upload(file, from_admin) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to upload file: "+err.Error()) - } - - return c.String(http.StatusOK, fileId) - }) - e.DELETE("/storage/delete/:filename", func(c echo.Context) error { - err := StorageAuth(c) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to verify Auth: "+err.Error()) - } - filename := c.Param("filename") - - err = Storage.Delete(filename) - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to remove file / file not found: "+err.Error()) - } - return c.String(http.StatusOK, "File removed") - - }) - e.GET("/storage/remove-expired", func(c echo.Context) error { - - err := RemoveExpired() - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to remove files / files not found: "+err.Error()) - } - return c.String(http.StatusOK, "File removed") - - }) - e.GET("/storage/test", func(c echo.Context) error { - err := CheckConcruency() - if err != nil { - return c.String(http.StatusInternalServerError, "Failed to check concruency: "+err.Error()) - } - return c.String(http.StatusOK, "OK") - }) -} diff --git a/apps/backend/storage_handlers/fs.go b/apps/backend/storage_handlers/fs.go index 046e8eb..7bc4aeb 100644 --- a/apps/backend/storage_handlers/fs.go +++ b/apps/backend/storage_handlers/fs.go @@ -1,6 +1,7 @@ package storagehandlers import ( + "backend/config" "io" "log" "mime/multipart" @@ -15,7 +16,7 @@ func NewFsHandler() *Fs_Handler { return &Fs_Handler{} } func (f *Fs_Handler) Init() error { - if _, err := os.Stat(Storagepath); os.IsNotExist(err) { + if _, err := os.Stat(config.Cfg.Storage.Path); os.IsNotExist(err) { return err } return nil @@ -30,7 +31,7 @@ func (f *Fs_Handler) Add(file *multipart.FileHeader, fileId string) error { defer src.Close() extension := filepath.Ext(file.Filename) - destPath := filepath.Join(Storagepath, fileId+extension) + destPath := filepath.Join(config.Cfg.Storage.Path, fileId+extension) dst, err := os.Create(destPath) if err != nil { log.Println("Error creating file") @@ -45,7 +46,7 @@ func (f *Fs_Handler) Add(file *multipart.FileHeader, fileId string) error { } func (f *Fs_Handler) Delete(filename string) error { - filePath := filepath.Join(Storagepath, filename) + filePath := filepath.Join(config.Cfg.Storage.Path, filename) if _, err := os.Stat(filePath); os.IsNotExist(err) { return err } @@ -53,7 +54,7 @@ func (f *Fs_Handler) Delete(filename string) error { } func (f *Fs_Handler) Get(filename string) (string, error) { - path := filepath.Join(Storagepath, filename) + path := filepath.Join(config.Cfg.Storage.Path, filename) if _, err := os.Stat(path); os.IsNotExist(err) { return "", err @@ -62,7 +63,7 @@ func (f *Fs_Handler) Get(filename string) (string, error) { } func (f *Fs_Handler) List() ([]string, error) { - files, err := os.ReadDir(Storagepath) + files, err := os.ReadDir(config.Cfg.Storage.Path) if err != nil { return nil, err } diff --git a/apps/backend/storage_handlers/minio.go b/apps/backend/storage_handlers/minio.go index a1e14b7..93efc4f 100644 --- a/apps/backend/storage_handlers/minio.go +++ b/apps/backend/storage_handlers/minio.go @@ -1,11 +1,11 @@ package storagehandlers import ( + "backend/config" "context" "fmt" "log" "mime/multipart" - "os" "path/filepath" "github.com/minio/minio-go/v7" @@ -42,12 +42,9 @@ func (s *S3_Handler) Init() error { } func NewS3Handler() (*S3_Handler, error) { - MINIO_URL := os.Getenv("MINIO_URL") - MINIO_ROOT_USER := os.Getenv("MINIO_ROOT_USER") - MINIO_ROOT_PASSWORD := os.Getenv("MINIO_ROOT_PASSWORD") - minioClient, err := minio.New(MINIO_URL, &minio.Options{ - Creds: credentials.NewStaticV4(MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, ""), + minioClient, err := minio.New(config.Cfg.Storage.Minio.URL, &minio.Options{ + Creds: credentials.NewStaticV4(config.Cfg.Storage.Minio.RootUser, config.Cfg.Storage.Minio.Password, ""), Secure: useSSL, }) if err != nil { @@ -104,7 +101,7 @@ func (s *S3_Handler) Delete(filename string) error { // } func (s *S3_Handler) Get(filename string) (string, error) { - localPath := filepath.Join(Storagepath, "/tmp/", filename) + localPath := filepath.Join(config.Cfg.Storage.Path, "/tmp/", filename) err := s.minioClient.FGetObject(context.Background(), bucketName, filename, localPath, minio.GetObjectOptions{}) if err != nil { diff --git a/apps/backend/storage_handlers/storage_handler.go b/apps/backend/storage_handlers/storage_handler.go index 8b436b6..bd41a09 100644 --- a/apps/backend/storage_handlers/storage_handler.go +++ b/apps/backend/storage_handlers/storage_handler.go @@ -1,14 +1,13 @@ package storagehandlers import ( + "backend/config" + "backend/typesdefs" + "errors" "mime/multipart" ) -var ( - Storagepath string -) - type StorageHandler interface { Init() error Add(file *multipart.FileHeader, fileId string) error @@ -17,19 +16,12 @@ type StorageHandler interface { List() ([]string, error) } -type StorageStrategy string - -const ( - FS StorageStrategy = "fs" - S3 StorageStrategy = "s3" -) - -func NewStorageHandler(strategy StorageStrategy) (StorageHandler, error) { - if Storagepath == "" { +func NewStorageHandler(strategy typesdefs.StorageStrategy) (StorageHandler, error) { + if config.Cfg.Storage.Path == "" { return nil, errors.New("storage path not set") } switch strategy { - case FS: + case typesdefs.FS: h := NewFsHandler() err := h.Init() if err != nil { @@ -37,7 +29,7 @@ func NewStorageHandler(strategy StorageStrategy) (StorageHandler, error) { } return h, nil - case S3: + case typesdefs.S3: h, err := NewS3Handler() if err != nil { return nil, err diff --git a/apps/backend/typedefs/models.go b/apps/backend/typesdefs/models.go similarity index 99% rename from apps/backend/typedefs/models.go rename to apps/backend/typesdefs/models.go index afdda25..d3c7389 100644 --- a/apps/backend/typedefs/models.go +++ b/apps/backend/typesdefs/models.go @@ -1,4 +1,4 @@ -package typedefs +package typesdefs import ( "os" diff --git a/apps/backend/typesdefs/types.go b/apps/backend/typesdefs/types.go new file mode 100644 index 0000000..d0e8c11 --- /dev/null +++ b/apps/backend/typesdefs/types.go @@ -0,0 +1,8 @@ +package typesdefs + +type StorageStrategy string + +const ( + FS StorageStrategy = "fs" + S3 StorageStrategy = "s3" +)