From cb07f1dca8e9d5177fd9307240f6f699fb82a086 Mon Sep 17 00:00:00 2001 From: Marco Pedone Date: Tue, 13 Jan 2026 10:53:27 +0100 Subject: [PATCH] dbg --- apps/backend/Dockerfile | 1 + apps/backend/utils/fails.go | 78 +++++++++++++++++++++++++++++++++++++ apps/backend/utils/utils.go | 38 +++++++++++------- 3 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 apps/backend/utils/fails.go diff --git a/apps/backend/Dockerfile b/apps/backend/Dockerfile index d94e3fc..8c0d0c7 100644 --- a/apps/backend/Dockerfile +++ b/apps/backend/Dockerfile @@ -57,6 +57,7 @@ RUN chmod +x /app/scripts/*.sh # create folders RUN mkdir images RUN mkdir videos +RUN mkdir failed_requests RUN apk add curl diff --git a/apps/backend/utils/fails.go b/apps/backend/utils/fails.go new file mode 100644 index 0000000..c936e44 --- /dev/null +++ b/apps/backend/utils/fails.go @@ -0,0 +1,78 @@ +package utils + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "time" +) + +const ( + debugDir = "failed_requests" + maxSaveBody = 5 * 1024 * 1024 // 5 MiB, configurable + filePerm = 0640 + dirPerm = 0750 + timestampFormat = "20060102T150405Z" // safe for filenames +) + +// helper: generate a small random hex id +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// save a failing request atomically with metadata +func saveFailedRequest(prefix, url string, reqHeaders http.Header, respHeaders http.Header, status int, body []byte, errMsg string) (string, error) { + if err := os.MkdirAll(debugDir, dirPerm); err != nil { + return "", fmt.Errorf("mkdir debug dir: %w", err) + } + + ts := time.Now().UTC().Format(timestampFormat) + id := randomHex(6) + base := fmt.Sprintf("%s_%s_%s", prefix, ts, id) + + // Truncate body if too large + truncated := false + if len(body) > maxSaveBody { + body = body[:maxSaveBody] + truncated = true + } + + meta := map[string]interface{}{ + "timestamp_utc": ts, + "url": url, + "status": status, + "error": errMsg, + "truncated": truncated, + "req_headers": reqHeaders, + "resp_headers": respHeaders, + } + + metaBytes, _ := json.MarshalIndent(meta, "", " ") + + metaTmp := filepath.Join(debugDir, base+".meta.json.tmp") + bodyTmp := filepath.Join(debugDir, base+".body.tmp") + metaPath := filepath.Join(debugDir, base+".meta.json") + bodyPath := filepath.Join(debugDir, base+".body") + + // write temp + rename (atomic on POSIX filesystems) + if err := os.WriteFile(metaTmp, metaBytes, filePerm); err != nil { + return "", fmt.Errorf("write meta tmp: %w", err) + } + if err := os.Rename(metaTmp, metaPath); err != nil { + return "", fmt.Errorf("rename meta: %w", err) + } + if err := os.WriteFile(bodyTmp, body, filePerm); err != nil { + return "", fmt.Errorf("write body tmp: %w", err) + } + if err := os.Rename(bodyTmp, bodyPath); err != nil { + return "", fmt.Errorf("rename body: %w", err) + } + + return base, nil +} diff --git a/apps/backend/utils/utils.go b/apps/backend/utils/utils.go index b6c217f..b863397 100644 --- a/apps/backend/utils/utils.go +++ b/apps/backend/utils/utils.go @@ -1,10 +1,10 @@ package utils import ( + "compress/gzip" "encoding/xml" "fmt" "io" - "log" "net/http" "os" "reflect" @@ -71,7 +71,7 @@ func GetDataXML[T any](url string) (T, error) { } // Add browser-like headers to avoid 403 - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9,it;q=0.8") req.Header.Set("Accept-Encoding", "gzip, deflate") @@ -79,28 +79,40 @@ func GetDataXML[T any](url string) (T, error) { req.Header.Set("Referer", "https://www.miogest.com/") resp, err := client.Do(req) - log.Printf("Response Status: %s", resp.Status) - // if jsonBytes, err := json.MarshalIndent(resp.Header, "", " "); err == nil { - // log.Printf("Headers:\n%s", string(jsonBytes)) - // } + if err != nil { return result, fmt.Errorf("Failed to fetch url %s: %w", url, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - log.Printf("Fetch XML error: status error: %d", resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + _, _ = saveFailedRequest("non_200", url, req.Header, resp.Header, resp.StatusCode, body, "status error") return result, fmt.Errorf("Failed to fetch url %s: status error: %d", url, resp.StatusCode) } - - body, err := io.ReadAll(resp.Body) + // handle gzip decompression if server used it + var raw []byte + if strings.Contains(resp.Header.Get("Content-Encoding"), "gzip") { + gz, gerr := gzip.NewReader(resp.Body) + if gerr != nil { + body, _ := io.ReadAll(resp.Body) // best-effort + _, _ = saveFailedRequest("gzip_error", url, req.Header, resp.Header, resp.StatusCode, body, gerr.Error()) + return result, fmt.Errorf("gzip new reader: %w", gerr) + } + defer gz.Close() + raw, err = io.ReadAll(gz) + } else { + raw, err = io.ReadAll(resp.Body) + } if err != nil { - return result, fmt.Errorf("failed to read response: %w, body: %s", err, string(body)) + _, _ = saveFailedRequest("read_error", url, req.Header, resp.Header, resp.StatusCode, raw, err.Error()) + return result, fmt.Errorf("failed to read response: %w", err) } - err = xml.Unmarshal(body, &result) - if err != nil { - return result, fmt.Errorf("failed to unmarshal XML: %w, body: %s", err, string(body)) + // attempt to unmarshal + if err := xml.Unmarshal(raw, &result); err != nil { + _, _ = saveFailedRequest("unmarshal_error", url, req.Header, resp.Header, resp.StatusCode, raw, err.Error()) + return result, fmt.Errorf("failed to unmarshal XML: %w", err) } return result, nil }