ext/har: add HAR logger extension (#610)

* ext/har: add HAR logger extension

Port HAR logging from abourget/goproxy to ext package.
Closes #609

* reverted to go.mod and go.sum files

* fixed code issues from pull request 610 other then testing and SaveToFile

* added tesify testing and added fruther logs to ParseRequest

* added testify dependancy

* modernised fillIpaddress func to work for ipv6 addresses

* refactored HAR exporting componant

* Removed SaveToFile, SaveToFile Tests, AppendEntry, AppendPage, Clear, and GetEntries methods

* removed currentCOunt and lastExport functions to and refactored associated functions

* refactored exportLoop to use a timer and interval, refactored logger to account for edge cases of timer set with no interval

* fixed export system to work better and be more readble, switched to fixed channel size for the go routine

* fixed a compilation issue in types and fixed the logger_tests to use the callback function and to no longer by flakey

* fixed issues from the pull request including readability issues in the exportLoop, test viability,and package fixing

* updated Interval test to use a larger export Interval

* added client.Do in testing

* removed uneeded spaces and fixed issues in testing from batching and mutex issues

* removed the entry copies on the ExportFunc and refactored threshold logger test to measure based on freq not a sorterd set

* ctx is always not nil

* Avoid to expose request and response parser functions

* Update logger.go

---------

Co-authored-by: Erik Pellizzon <[email protected]>

ext/go.mod

@@ -4,6 +4,13 @@ go 1.20
 
 require (
 	github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c
+	github.com/stretchr/testify v1.10.0
 	golang.org/x/net v0.34.0
 	golang.org/x/text v0.21.0
 )
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

ext/go.sum

@@ -1,6 +1,16 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c h1:yWAGp1CjD1mQGLUsADqPn5s1n2AkGAX33XLDUgoXzyo=
 github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c/go.mod h1:P73liMk9TZCyF9fXG/RyMeSizmATvpvy3ZS61/1eXn4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
 golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

ext/har/logger.go

@@ -0,0 +1,124 @@
+package har
+
+import (
+    "net/http"
+    "time"
+
+    "github.com/elazarl/goproxy"
+)
+
+// ExportFunc is a function type that users can implement to handle exported entries
+type ExportFunc func([]Entry)
+
+// Logger implements a HAR logging extension for goproxy
+type Logger struct {
+    exportFunc      ExportFunc
+    exportInterval  time.Duration
+    exportThreshold int
+    dataCh          chan Entry
+}
+
+// LoggerOption is a function type for configuring the Logger
+type LoggerOption func(*Logger)
+
+// WithExportInterval sets the interval for automatic exports
+func WithExportInterval(d time.Duration) LoggerOption {
+    return func(l *Logger) {
+        l.exportInterval = d
+    }
+}
+
+// WithExportCount sets the number of requests after which to export entries
+func WithExportThreshold(threshold int) LoggerOption {
+    return func(l *Logger) {
+        l.exportThreshold = threshold
+    }
+}
+
+// NewLogger creates a new HAR logger instance
+func NewLogger(exportFunc ExportFunc, opts ...LoggerOption) *Logger {
+    l := &Logger{
+        exportFunc:     exportFunc,
+        exportThreshold: 100,    // Default threshold
+        exportInterval: 0,       // Default no interval
+        dataCh:         make(chan Entry), 
+    }
+
+    // Apply options
+    for _, opt := range opts {
+        opt(l)
+    }
+
+    go l.exportLoop()
+    return l
+}
+// OnRequest handles incoming HTTP requests
+func (l *Logger) OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
+    ctx.UserData = time.Now()
+    return req, nil
+}
+
+// OnResponse handles HTTP responses
+func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
+    if resp == nil || ctx.Req == nil || ctx.UserData == nil {
+        return resp
+    }
+    startTime, ok := ctx.UserData.(time.Time)
+    if !ok {
+        return resp
+    }
+
+    entry := Entry{
+        StartedDateTime: startTime,
+        Time:           time.Since(startTime).Milliseconds(),
+        Request:        parseRequest(ctx),
+        Response:       parseResponse(ctx),
+        Timings: Timings{
+            Send:    0,
+            Wait:    time.Since(startTime).Milliseconds(),
+            Receive: 0,
+        },
+    }
+    entry.fillIPAddress(ctx.Req)
+
+    l.dataCh <- entry 
+    return resp
+}
+
+func (l *Logger) exportLoop() {
+   var entries []Entry 
+
+   exportIfNeeded := func() {
+        if len(entries) > 0 {
+            go l.exportFunc(entries)
+            entries = nil 
+        } 
+    } 
+
+    var tickerC <-chan time.Time
+    if l.exportInterval > 0 {
+        ticker := time.NewTicker(l.exportInterval)
+        defer ticker.Stop()
+        tickerC = ticker.C
+    }
+
+    for {
+        select {
+        case entry, ok := <-l.dataCh:
+            if !ok {
+                exportIfNeeded()
+                return
+            }
+            entries = append(entries, entry)
+            if l.exportThreshold > 0 && len(entries) >= l.exportThreshold {
+                exportIfNeeded()
+            } 
+        case <-tickerC:
+            exportIfNeeded()
+        }
+    }
+}
+
+func (l *Logger) Stop() {
+    close(l.dataCh)
+}

ext/har/logger_test.go

@@ -0,0 +1,217 @@
+package har
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+	"github.com/elazarl/goproxy"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// ConstantHandler is a simple HTTP handler that returns a constant response
+type ConstantHandler string
+
+func (h ConstantHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+    w.Header().Set("Content-Type", "text/plain")
+    io.WriteString(w, string(h))
+}
+
+// createTestProxy sets up a test proxy with a HAR logger
+func createTestProxy(logger *Logger) *httptest.Server {
+    proxy := goproxy.NewProxyHttpServer()
+    proxy.OnRequest().DoFunc(logger.OnRequest)
+    proxy.OnResponse().DoFunc(logger.OnResponse)
+    return httptest.NewServer(proxy)
+}
+
+// createProxyClient creates an HTTP client that uses the given proxy
+func createProxyClient(proxyURL string) *http.Client {
+    proxyURLParsed, _ := url.Parse(proxyURL)
+    tr := &http.Transport{
+        Proxy: http.ProxyURL(proxyURLParsed),
+    }
+    return &http.Client{Transport: tr}
+}
+
+func TestHarLoggerBasicFunctionality(t *testing.T) {
+    testCases := []struct {
+        name           string
+        method         string
+        body           string
+        contentType    string
+        expectedMethod string
+    }{
+        {
+            name:           "GET Request",
+            method:         http.MethodGet,
+            expectedMethod: http.MethodGet,
+        },
+        {
+            name:           "POST Request",
+            method:         http.MethodPost,
+            body:           `{"test":"data"}`,
+            contentType:    "application/json",
+            expectedMethod: http.MethodPost,
+        },
+    }
+
+    for _, tc := range testCases {
+        t.Run(tc.name, func(t *testing.T) {
+            var wg sync.WaitGroup
+            wg.Add(1)
+
+            var exportedEntries []Entry
+            exportFunc := func(entries []Entry) {
+                exportedEntries = append(exportedEntries, entries...)
+                wg.Done()
+            }
+
+            logger := NewLogger(exportFunc, WithExportThreshold(1)) // Export after each request
+            defer logger.Stop()
+
+            background := httptest.NewServer(ConstantHandler("hello world"))
+            defer background.Close()
+
+            proxyServer := createTestProxy(logger)
+            defer proxyServer.Close()
+
+            client := createProxyClient(proxyServer.URL)
+
+            req, err := http.NewRequestWithContext(
+                context.Background(),
+                tc.method,
+                background.URL,
+                strings.NewReader(tc.body),
+            )
+            require.NoError(t, err, "Should create request")
+
+            if tc.contentType != "" {
+                req.Header.Set("Content-Type", tc.contentType)
+            }
+
+            resp, err := client.Do(req)
+            require.NoError(t, err, "Should send request successfully")
+            defer resp.Body.Close()
+
+            bodyBytes, err := io.ReadAll(resp.Body)
+            require.NoError(t, err, "Should read response body")
+
+            body := string(bodyBytes)
+            assert.Equal(t, "hello world", body, "Response body should match")
+
+            wg.Wait() // Wait for export to complete
+
+            assert.Len(t, exportedEntries, 1, "Should have exactly one exported entry")
+            assert.Equal(t, tc.expectedMethod, exportedEntries[0].Request.Method, "Request method should match")
+        })
+    }
+}
+
+func TestLoggerThresholdExport(t *testing.T) {
+    var wg sync.WaitGroup
+    var exports [][]Entry
+    var mtx sync.Mutex
+    wg.Add(3) // Expect 3 exports (3,3,1)
+
+    exportFunc := func(entries []Entry) {
+        mtx.Lock()
+        exports = append(exports, entries)
+        mtx.Unlock()
+
+        t.Logf("Export occurred with %d entries", len(entries))
+        wg.Done()
+    }
+
+    threshold := 3
+    logger := NewLogger(exportFunc, WithExportThreshold(threshold))
+
+    background := httptest.NewServer(ConstantHandler("test"))
+    defer background.Close()
+    proxyServer := createTestProxy(logger)
+    defer proxyServer.Close()
+    client := createProxyClient(proxyServer.URL)
+
+    // Send 7 requests
+    for i := 0; i < 7; i++ {
+        req, err := http.NewRequestWithContext(
+            context.Background(),
+            http.MethodGet,
+            background.URL,
+            nil,
+        )
+        require.NoError(t, err)
+
+        resp, err := client.Do(req)
+        require.NoError(t, err)
+        resp.Body.Close()
+    }  
+
+    // Call Stop to trigger final export of remaining entries
+    logger.Stop()
+    wg.Wait()
+
+    require.Equal(t, 3, len(exports), "should have 3 export batches")
+
+    // Count batches by size
+    batchCounts := make(map[int]int)
+    for _, batch := range exports {
+        batchCounts[len(batch)]++
+    }
+
+    // Check batch sizes
+    assert.Equal(t, 2, batchCounts[threshold], "should have two batches of threshold size")
+    assert.Equal(t, 1, batchCounts[1], "should have one batch with 1 entry")
+}
+
+func TestHarLoggerExportInterval(t *testing.T) {
+    var wg sync.WaitGroup
+    var mtx sync.Mutex
+    var exports [][]Entry
+    wg.Add(1) // Expect 1 export with all entries
+
+   exportFunc := func(entries []Entry) {
+        mtx.Lock()
+        exports = append(exports, entries)
+        mtx.Unlock()
+
+        t.Logf("Export occurred with %d entries", len(entries))
+        wg.Done()
+    } 
+
+    logger := NewLogger(exportFunc, WithExportInterval(time.Second))
+
+    background := httptest.NewServer(ConstantHandler("test"))
+    defer background.Close()
+    proxyServer := createTestProxy(logger)
+    defer proxyServer.Close()
+    client := createProxyClient(proxyServer.URL)
+
+    // Send 3 requests
+    for i := 0; i < 3; i++ {
+        req, err := http.NewRequestWithContext(
+            context.Background(),
+            http.MethodGet,
+            background.URL,
+            nil,
+        )
+        require.NoError(t, err)
+
+        resp, err := client.Do(req)
+        require.NoError(t, err)
+        resp.Body.Close()
+    } 
+
+    wg.Wait()
+    logger.Stop()
+
+    require.Equal(t, 1, len(exports), "should have 1 export batch")
+    assert.Equal(t, 3, len(exports[0]), "Should have exported 3 entries")
+}
+