dbg

apps/backend/Dockerfile

@@ -57,6 +57,7 @@ RUN chmod +x /app/scripts/*.sh
 # create folders 
 RUN mkdir images
 RUN mkdir videos
+RUN mkdir failed_requests
 
 RUN apk add curl
 

apps/backend/utils/fails.go

@@ -0,0 +1,78 @@
+package utils
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const (
+	debugDir        = "failed_requests"
+	maxSaveBody     = 5 * 1024 * 1024 // 5 MiB, configurable
+	filePerm        = 0640
+	dirPerm         = 0750
+	timestampFormat = "20060102T150405Z" // safe for filenames
+)
+
+// helper: generate a small random hex id
+func randomHex(n int) string {
+	b := make([]byte, n)
+	_, _ = rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
+// save a failing request atomically with metadata
+func saveFailedRequest(prefix, url string, reqHeaders http.Header, respHeaders http.Header, status int, body []byte, errMsg string) (string, error) {
+	if err := os.MkdirAll(debugDir, dirPerm); err != nil {
+		return "", fmt.Errorf("mkdir debug dir: %w", err)
+	}
+
+	ts := time.Now().UTC().Format(timestampFormat)
+	id := randomHex(6)
+	base := fmt.Sprintf("%s_%s_%s", prefix, ts, id)
+
+	// Truncate body if too large
+	truncated := false
+	if len(body) > maxSaveBody {
+		body = body[:maxSaveBody]
+		truncated = true
+	}
+
+	meta := map[string]interface{}{
+		"timestamp_utc": ts,
+		"url":           url,
+		"status":        status,
+		"error":         errMsg,
+		"truncated":     truncated,
+		"req_headers":   reqHeaders,
+		"resp_headers":  respHeaders,
+	}
+
+	metaBytes, _ := json.MarshalIndent(meta, "", "  ")
+
+	metaTmp := filepath.Join(debugDir, base+".meta.json.tmp")
+	bodyTmp := filepath.Join(debugDir, base+".body.tmp")
+	metaPath := filepath.Join(debugDir, base+".meta.json")
+	bodyPath := filepath.Join(debugDir, base+".body")
+
+	// write temp + rename (atomic on POSIX filesystems)
+	if err := os.WriteFile(metaTmp, metaBytes, filePerm); err != nil {
+		return "", fmt.Errorf("write meta tmp: %w", err)
+	}
+	if err := os.Rename(metaTmp, metaPath); err != nil {
+		return "", fmt.Errorf("rename meta: %w", err)
+	}
+	if err := os.WriteFile(bodyTmp, body, filePerm); err != nil {
+		return "", fmt.Errorf("write body tmp: %w", err)
+	}
+	if err := os.Rename(bodyTmp, bodyPath); err != nil {
+		return "", fmt.Errorf("rename body: %w", err)
+	}
+
+	return base, nil
+}

apps/backend/utils/utils.go

@@ -1,10 +1,10 @@
 package utils
 
 import (
+	"compress/gzip"
 	"encoding/xml"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"reflect"
@@ -71,7 +71,7 @@ func GetDataXML[T any](url string) (T, error) {
 	}
 
 	// Add browser-like headers to avoid 403
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9,it;q=0.8")
 	req.Header.Set("Accept-Encoding", "gzip, deflate")
@@ -79,28 +79,40 @@ func GetDataXML[T any](url string) (T, error) {
 	req.Header.Set("Referer", "https://www.miogest.com/")
 
 	resp, err := client.Do(req)
-	log.Printf("Response Status: %s", resp.Status)
+
-	// if jsonBytes, err := json.MarshalIndent(resp.Header, "", "  "); err == nil {
-	// 	log.Printf("Headers:\n%s", string(jsonBytes))
-	// }
 	if err != nil {
 		return result, fmt.Errorf("Failed to fetch url %s: %w", url, err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		log.Printf("Fetch XML error: status error: %d", resp.StatusCode)
+		body, _ := io.ReadAll(resp.Body)
+		_, _ = saveFailedRequest("non_200", url, req.Header, resp.Header, resp.StatusCode, body, "status error")
 		return result, fmt.Errorf("Failed to fetch url %s: status error: %d", url, resp.StatusCode)
 	}
-
+	// handle gzip decompression if server used it
-	body, err := io.ReadAll(resp.Body)
+	var raw []byte
+	if strings.Contains(resp.Header.Get("Content-Encoding"), "gzip") {
+		gz, gerr := gzip.NewReader(resp.Body)
+		if gerr != nil {
+			body, _ := io.ReadAll(resp.Body) // best-effort
+			_, _ = saveFailedRequest("gzip_error", url, req.Header, resp.Header, resp.StatusCode, body, gerr.Error())
+			return result, fmt.Errorf("gzip new reader: %w", gerr)
+		}
+		defer gz.Close()
+		raw, err = io.ReadAll(gz)
+	} else {
+		raw, err = io.ReadAll(resp.Body)
+	}
 	if err != nil {
-		return result, fmt.Errorf("failed to read response: %w, body: %s", err, string(body))
+		_, _ = saveFailedRequest("read_error", url, req.Header, resp.Header, resp.StatusCode, raw, err.Error())
+		return result, fmt.Errorf("failed to read response: %w", err)
 	}
 
-	err = xml.Unmarshal(body, &result)
+	// attempt to unmarshal
-	if err != nil {
+	if err := xml.Unmarshal(raw, &result); err != nil {
-		return result, fmt.Errorf("failed to unmarshal XML: %w, body: %s", err, string(body))
+		_, _ = saveFailedRequest("unmarshal_error", url, req.Header, resp.Header, resp.StatusCode, raw, err.Error())
+		return result, fmt.Errorf("failed to unmarshal XML: %w", err)
 	}
 	return result, nil
 }