refactor: remove progressbar dependency and replace with logging in image and video processing
This commit is contained in:
parent
1e2eb19d80
commit
14eb5bf113
5 changed files with 13 additions and 35 deletions
|
|
@ -7,7 +7,6 @@ require (
|
|||
github.com/jackc/pgx/v5 v5.9.1
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/labstack/echo/v4 v4.15.1
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
|
|
@ -16,11 +15,8 @@ require (
|
|||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/lib/pq v1.12.3 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/nickalie/go-binwrapper v0.0.0-20260217210625-2cc2a03abe15 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/term v0.42.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.0 h1:/5znzg5n373N/3ESjHF5SMLxiW4RKB05Ql//KWfeTFs=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.0/go.mod h1:u3MiKYGupPPjkn3ozknpMUpxPaNLTFWAya419/zv6eI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -33,10 +31,6 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP
|
|||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
|
||||
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/nickalie/go-binwrapper v0.0.0-20260217210625-2cc2a03abe15 h1:PRBiX3Nf8l++Sp0YrgtvbL1voYxcIb/Mzt9I5Qa5r6I=
|
||||
|
|
@ -47,10 +41,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
|
|
@ -72,8 +62,6 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
|||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import (
|
|||
"github.com/labstack/echo/v4"
|
||||
"github.com/nfnt/resize"
|
||||
"github.com/nickalie/go-webpbin"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
)
|
||||
|
||||
func AddToImageProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
|
||||
|
|
@ -52,7 +51,7 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
)
|
||||
sem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
|
||||
totalImages := len(*toProcess)
|
||||
bar := progressbar.Default(int64(totalImages), "Processing images")
|
||||
log.Printf("Starting processing of %d images with concurrency limit %d...", totalImages, config.Cfg.Images.ConcurrentLimit)
|
||||
// clear images folder
|
||||
err := clearImageFolder()
|
||||
if err != nil {
|
||||
|
|
@ -85,15 +84,14 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
OGUrl: img.Url,
|
||||
})
|
||||
mu.Unlock()
|
||||
bar.Add(1)
|
||||
}(img)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.Finish()
|
||||
log.Printf("Processed all images")
|
||||
fmt.Println()
|
||||
|
||||
totalToUpload := len(ProcessedImages)
|
||||
uploadBar := progressbar.Default(int64(totalToUpload), "Uploading images")
|
||||
log.Printf("Starting upload of %d images with concurrency limit %d...", totalToUpload, config.Cfg.Images.ConcurrentLimit)
|
||||
uploadSem := make(chan struct{}, config.Cfg.Images.ConcurrentLimit)
|
||||
var uploadWg sync.WaitGroup
|
||||
for _, img := range ProcessedImages {
|
||||
|
|
@ -121,12 +119,12 @@ func ProcessImagePool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
ThumbId: thumbId,
|
||||
OGUrl: img.OGUrl,
|
||||
})
|
||||
uploadBar.Add(1)
|
||||
|
||||
mu.Unlock()
|
||||
}(img)
|
||||
}
|
||||
uploadWg.Wait()
|
||||
uploadBar.Finish()
|
||||
log.Printf("Uploaded all images")
|
||||
fmt.Println()
|
||||
|
||||
return UploadedImages, nil
|
||||
|
|
|
|||
|
|
@ -13,8 +13,6 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/schollz/progressbar/v3"
|
||||
)
|
||||
|
||||
type MiogestHandler struct {
|
||||
|
|
@ -247,7 +245,7 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
|
|||
}
|
||||
|
||||
totalAnnunci := len(annunci.Annuncio)
|
||||
bar := progressbar.Default(int64(totalAnnunci), "Parsing annunci")
|
||||
log.Printf("Processing %d annunci (images to update: %d, videos to update: %d)", totalAnnunci, len(image_codes_to_process), len(video_codes_to_process))
|
||||
var AnnunciArray typesdefs.AnnunciParsed
|
||||
AnnunciArray.Annuncio = make([]typesdefs.AnnuncioParsed, 0, len(annunci.Annuncio))
|
||||
m.InitDefaults()
|
||||
|
|
@ -257,9 +255,9 @@ func (m *MiogestHandler) ParseAnnunci(codFilter string, imgPool *[]typesdefs.Med
|
|||
updateVideos := slices.Contains(video_codes_to_process, *annunci.Annuncio[i].Codice)
|
||||
result := extractData_update(&annunci.Annuncio[i], imgToUpdate, updateVideos, m, imgPool, videoPool)
|
||||
AnnunciArray.Annuncio = append(AnnunciArray.Annuncio, result)
|
||||
bar.Add(1)
|
||||
|
||||
}
|
||||
bar.Finish()
|
||||
log.Printf("Processed %d annunci", len(AnnunciArray.Annuncio))
|
||||
fmt.Println()
|
||||
return AnnunciArray, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import (
|
|||
"github.com/labstack/echo/v4"
|
||||
"github.com/nfnt/resize"
|
||||
"github.com/nickalie/go-webpbin"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
)
|
||||
|
||||
func AddToVideoProcessing(tmp *typesdefs.AnnuncioParsed, annuncio *typesdefs.AnnuncioXML, p *[]typesdefs.MediaToProcess) {
|
||||
|
|
@ -58,7 +57,7 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
wg sync.WaitGroup
|
||||
)
|
||||
totalToProcess := len(*toProcess)
|
||||
bar := progressbar.Default(int64(totalToProcess), "Processing videos")
|
||||
log.Printf("Processing %d videos with concurrency limit %d", totalToProcess, config.Cfg.Videos.ConcurrentLimit)
|
||||
sem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
|
||||
|
||||
// clear videos folder
|
||||
|
|
@ -94,16 +93,15 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
ThumnailPath: thumbnailPath,
|
||||
OGUrl: video.Url,
|
||||
})
|
||||
bar.Add(1)
|
||||
mu.Unlock()
|
||||
}(video)
|
||||
}
|
||||
wg.Wait()
|
||||
bar.Finish()
|
||||
log.Printf("Video processing completed for %d videos. Starting upload...", len(ProcessedVideos))
|
||||
fmt.Println()
|
||||
|
||||
totalToUpload := len(ProcessedVideos)
|
||||
uploadBar := progressbar.Default(int64(totalToUpload), "Uploading videos")
|
||||
log.Printf("Starting upload of %d videos with concurrency limit %d...", totalToUpload, config.Cfg.Videos.ConcurrentLimit)
|
||||
uploadSem := make(chan struct{}, config.Cfg.Videos.ConcurrentLimit)
|
||||
var uploadWg sync.WaitGroup
|
||||
for _, video := range ProcessedVideos {
|
||||
|
|
@ -131,12 +129,12 @@ func ProcessVideoPool(toProcess *[]typesdefs.MediaToProcess) ([]typesdefs.Upload
|
|||
ThumbId: thumbId,
|
||||
OGUrl: video.OGUrl,
|
||||
})
|
||||
uploadBar.Add(1)
|
||||
|
||||
mu.Unlock()
|
||||
}(video)
|
||||
}
|
||||
uploadWg.Wait()
|
||||
uploadBar.Finish()
|
||||
log.Printf("Upload completed for %d videos.", len(UploadedVideos))
|
||||
fmt.Println()
|
||||
|
||||
return UploadedVideos, nil
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue