From 1c4a034fc6476e64505ef24ebfea4e2bfe132753 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Wed, 9 Apr 2025 21:40:53 +0000 Subject: [PATCH 01/60] chore: add go.sum file and update dependencies --- go.mod | 8 ++++++++ go.sum | 10 ++++++++++ 2 files changed, 18 insertions(+) create mode 100644 go.sum diff --git a/go.mod b/go.mod index d384985..0396592 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,11 @@ module github.com/Mathious6/harkit go 1.24.2 + +require github.com/stretchr/testify v1.10.0 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..713a0b4 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From a4da780a189f0986e34d12d3a9fe5271c1977aa3 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Wed, 9 Apr 2025 21:41:24 +0000 Subject: [PATCH 02/60] feat: implement request conversion functions for HAR file generation --- converter/request.go | 160 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 converter/request.go diff --git a/converter/request.go b/converter/request.go new file mode 100644 index 0000000..cae9d39 --- /dev/null +++ b/converter/request.go @@ -0,0 +1,160 @@ +package converter + +import ( + "bytes" + "io" + "net/http" + "net/url" + "strings" + + "github.com/Mathious6/harkit/harfile" +) + +func FromHTTPRequest(req *http.Request) (harfile.Request, error) { + headers, headersSize := convertHeaders(req) + postData, bodySize, err := extractPostData(req) + if err != nil { + return harfile.Request{}, err + } + + return harfile.Request{ + Method: req.Method, + URL: req.URL.String(), + HTTPVersion: req.Proto, + Cookies: convertCookies(req.Cookies()), + Headers: headers, + QueryString: convertQueryParams(req.URL), + PostData: postData, + HeadersSize: headersSize, + BodySize: bodySize, + Comment: "Generated from FromHTTPRequest", + }, nil +} + +func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { + harCookies := make([]*harfile.Cookie, len(cookies)) + + for i, cookie := range cookies { + harCookies[i] = &harfile.Cookie{ + Name: cookie.Name, + Value: cookie.Value, + } + } + + return harCookies +} + +func convertHeaders(req *http.Request) ([]*harfile.NameValuePair, int64) { + harHeaders := make([]*harfile.NameValuePair, 0, len(req.Header)) + + // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) + seen := make(map[string]bool) + for _, name := range req.Header.Values("Header-Order") { + if values := req.Header.Values(name); len(values) > 0 { + for _, value := range values { + harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + } + seen[http.CanonicalHeaderKey(name)] = true + } + } + + for name, values := range req.Header { + if seen[name] || strings.EqualFold(name, "Header-Order") { + continue + } + for _, value := range values { + harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + } + } + + headersSize := int64(0) + for _, h := range harHeaders { + headersSize += int64(len(h.Name) + len(h.Value)) + } + + return harHeaders, headersSize +} + +func convertQueryParams(u *url.URL) []*harfile.NameValuePair { + var result []*harfile.NameValuePair + + for key, values := range u.Query() { + for _, value := range values { + result = append(result, &harfile.NameValuePair{Name: key, Value: value}) + } + } + + return result +} + +func extractPostData(req *http.Request) (*harfile.PostData, int64, error) { + if req.Body == nil || req.ContentLength == 0 { + return nil, int64(req.ContentLength), nil + } + + buf, err := io.ReadAll(req.Body) + if err != nil { + return nil, 0, err + } + defer req.Body.Close() + req.Body = io.NopCloser(bytes.NewBuffer(buf)) + + mimeType := req.Header.Get("Content-Type") + postData := &harfile.PostData{MimeType: mimeType} + bodySize := int64(len(buf)) + + if strings.HasPrefix(mimeType, "application/x-www-form-urlencoded") { + text := string(buf) + pairs := strings.SplitSeq(text, "&") + + for pair := range pairs { + nv := strings.SplitN(pair, "=", 2) + if len(nv) == 2 { + name, value := nv[0], nv[1] + postData.Params = append(postData.Params, &harfile.Param{Name: name, Value: value}) + } + } + + return postData, bodySize, nil + } + + if strings.HasPrefix(mimeType, "multipart/form-data") { + err := req.ParseMultipartForm(32 << 20) // 32 MB limit + if err != nil { + return nil, 0, err + } + + for name, values := range req.MultipartForm.Value { + for _, value := range values { + postData.Params = append(postData.Params, &harfile.Param{Name: name, Value: value}) + } + } + + for name, files := range req.MultipartForm.File { + for _, fileHeader := range files { + file, err := fileHeader.Open() + if err != nil { + return nil, 0, err + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return nil, 0, err + } + + postData.Params = append(postData.Params, &harfile.Param{ + Name: name, + FileName: fileHeader.Filename, + ContentType: fileHeader.Header.Get("Content-Type"), + Value: string(content), + }) + } + } + + return postData, bodySize, nil + } + + postData.Text = string(buf) + return postData, bodySize, nil +} From a310dae7333fdf6f5a88f75e151461b2cf8df381 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Wed, 9 Apr 2025 21:41:33 +0000 Subject: [PATCH 03/60] feat: add unit tests for HTTP request conversion in the converter package --- converter/request_test.go | 211 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 converter/request_test.go diff --git a/converter/request_test.go b/converter/request_test.go new file mode 100644 index 0000000..9972396 --- /dev/null +++ b/converter/request_test.go @@ -0,0 +1,211 @@ +package converter_test + +import ( + "bytes" + "io" + "mime/multipart" + "net/http" + "strings" + "testing" + + "github.com/Mathious6/harkit/converter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + METHOD = http.MethodPost + URL = "https://example.com/api?foo=bar" + PROTOCOL = "HTTP/1.1" + + HEADER_ORDER_KEY = "Header-Order" + CONTENT_TYPE_KEY = "Content-Type" + COOKIE_KEY = "Cookie" + + HEADER1_NAME = "NaMe1" + HEADER1_VALUE = "value1" + HEADER2_NAME = "nAmE2" + HEADER2_VALUE = "value2" + + COOKIE_NAME = "name" + COOKIE_VALUE = "value" + + URL_CONTENT_TYPE = "application/x-www-form-urlencoded" + BODY_URL = "foo=bar" + + JSON_CONTENT_TYPE = "application/json" + BODY_JSON = `{"foo":"bar"}` + + PART1_NAME = "name1" + PART1_VALUE = "value1" + PART2_NAME = "file" + PART2_VALUE = "content" + PART2_FILENAME = "test.txt" + PART2_CONTENT_TYPE = "application/octet-stream" +) + +func TestConverter_GivenMethod_WhenConvertingHTTPRequest_ThenMethodShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Equal(t, METHOD, result.Method, "HAR method <> request method") +} + +func TestConverter_GivenURL_WhenConvertingHTTPRequest_ThenURLShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Equal(t, URL, result.URL, "HAR URL <> request URL") +} + +func TestConverter_GivenProtocol_WhenConvertingHTTPRequest_ThenProtocolShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Equal(t, PROTOCOL, result.HTTPVersion, "HAR protocol <> request protocol") +} + +func TestConverter_GivenCookies_WhenConvertingHTTPRequest_ThenCookiesShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.Cookies, 1, "HAR should contain 1 cookie") + assert.Equal(t, COOKIE_NAME, result.Cookies[0].Name, "HAR cookie name <> request cookie name") + assert.Equal(t, COOKIE_VALUE, result.Cookies[0].Value, "HAR cookie value <> request cookie value") +} + +func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersShouldBeCorrectAndOrdered(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.Headers, 3, "HAR should contain 7 headers") + assert.Equal(t, HEADER2_NAME, result.Headers[0].Name, "HAR header name <> request header name") + assert.Equal(t, HEADER2_VALUE, result.Headers[0].Value, "HAR header value <> request header value") + assert.Equal(t, HEADER1_NAME, result.Headers[1].Name, "HAR header name <> request header name") + assert.Equal(t, HEADER1_VALUE, result.Headers[1].Value, "HAR header value <> request header value") +} + +func TestConverter_GivenURLWithQueryString_WhenConvertingHTTPRequest_ThenQueryStringShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.QueryString, 1, "HAR should contain 1 query string parameters") + assert.Equal(t, "foo", result.QueryString[0].Name, "HAR query string name <> request query string name") + assert.Equal(t, "bar", result.QueryString[0].Value, "HAR query string value <> request query string value") +} + +func TestConverter_GivenURLEncodedBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { + req := createRequest(t, strings.NewReader(BODY_URL), URL_CONTENT_TYPE) + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.PostData.Params, 1, "HAR should contain 1 post data parameters") + assert.Empty(t, result.PostData.Text, "HAR should have no post data text") + assert.Equal(t, "foo", result.PostData.Params[0].Name, "HAR post data name <> request post data name") + assert.Equal(t, "bar", result.PostData.Params[0].Value, "HAR post data value <> request post data value") + assert.Equal(t, URL_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") +} + +func TestConverter_GivenJSONBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { + req := createRequest(t, strings.NewReader(BODY_JSON), JSON_CONTENT_TYPE) + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.PostData.Params, 0, "HAR should contain 0 post data parameters") + assert.NotEmpty(t, result.PostData.Text, "HAR should have post data text") + assert.Equal(t, BODY_JSON, result.PostData.Text, "HAR post data text <> request post data text") + assert.Equal(t, JSON_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") +} + +func TestConverter_GivenMultipartBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { + body, contentType := createMultipartBody() + req := createRequest(t, &body, contentType) + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Len(t, result.PostData.Params, 2, "HAR should contain 2 post data parameters") + assert.Empty(t, result.PostData.Text, "HAR should have no post data text") + assert.Equal(t, PART1_NAME, result.PostData.Params[0].Name, "HAR post data name <> request post data name") + assert.Equal(t, PART1_VALUE, result.PostData.Params[0].Value, "HAR post data value <> request post data value") + assert.Equal(t, PART2_NAME, result.PostData.Params[1].Name, "HAR post data name <> request post data name") + assert.Equal(t, PART2_VALUE, result.PostData.Params[1].Value, "HAR post data value <> request post data value") + assert.Equal(t, PART2_FILENAME, result.PostData.Params[1].FileName, "HAR post data filename <> request post data filename") + assert.Equal(t, PART2_CONTENT_TYPE, result.PostData.Params[1].ContentType, "HAR post data content type <> request post data content type") + assert.Equal(t, contentType, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") +} + +func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersSizeShouldBeCorrect(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + expectedSize := len(HEADER1_NAME) + len(HEADER1_VALUE) + expectedSize += len(HEADER2_NAME) + len(HEADER2_VALUE) + expectedSize += len(COOKIE_KEY) + len(COOKIE_NAME+"="+COOKIE_VALUE) + assert.Equal(t, int64(expectedSize), result.HeadersSize, "HAR header size <> request header size") +} + +func TestConverter_GivenBody_WhenConvertingHTTPRequest_ThenBodySizeShouldBeCorrect(t *testing.T) { + req := createRequest(t, strings.NewReader(BODY_URL), URL_CONTENT_TYPE) + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + expectedSize := int64(len(BODY_URL)) + assert.Equal(t, expectedSize, result.BodySize, "HAR body size <> request body size") +} + +// createRequest creates and returns a new HTTP request with the specified body and content type. +// It sets various headers including cookies, custom headers, and content type if provided. +// The function also ensures the request protocol is set and validates the request creation. +func createRequest(t *testing.T, body io.Reader, contentType string) *http.Request { + req, err := http.NewRequest(METHOD, URL, body) + require.NoError(t, err) + + req.Proto = PROTOCOL + + req.Header.Add(COOKIE_KEY, COOKIE_NAME+"="+COOKIE_VALUE) + req.Header.Add(HEADER1_NAME, HEADER1_VALUE) + req.Header.Add(HEADER2_NAME, HEADER2_VALUE) + + req.Header.Add(HEADER_ORDER_KEY, HEADER2_NAME) + req.Header.Add(HEADER_ORDER_KEY, HEADER1_NAME) + + if contentType != "" { + req.Header.Add(CONTENT_TYPE_KEY, contentType) + } + + return req +} + +// createMultipartBody constructs a multipart HTTP request body with predefined fields and file content. +// It returns the body as a bytes.Buffer and the corresponding Content-Type header value. +func createMultipartBody() (body bytes.Buffer, contentType string) { + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + _ = writer.WriteField(PART1_NAME, PART1_VALUE) + + fileWriter, _ := writer.CreateFormFile(PART2_NAME, PART2_FILENAME) + _, _ = fileWriter.Write([]byte(PART2_VALUE)) + + writer.Close() + + return buf, writer.FormDataContentType() +} From 429f56f6c9a701a09a4a853f7f9ba3ec56bf8da6 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 01:24:00 +0000 Subject: [PATCH 04/60] fix: update comments for clarity in HAR and Cookie structures --- harfile/har.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/harfile/har.go b/harfile/har.go index 446a4a3..4df02a6 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -6,7 +6,7 @@ import "time" // HAR parent container for log. type HAR struct { - Log *Log `json:"log"` // + Log *Log `json:"log"` // Log represents the root of exported data. } // Log represents the root of exported data. @@ -99,14 +99,14 @@ type Response struct { // Cookie contains list of all cookies (used in [Request] and [Response] // objects). type Cookie struct { - Name string `json:"name"` // The name of the cookie. - Value string `json:"value"` // The cookie value. - Path string `json:"path,omitempty"` // The path pertaining to the cookie. - Domain string `json:"domain,omitempty"` // The host of the cookie. - Expires string `json:"expires,omitempty"` // Cookie expiration time. (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00). - HTTPOnly bool `json:"httpOnly"` // Set to true if the cookie is HTTP only, false otherwise. - Secure bool `json:"secure"` // True if the cookie was transmitted over ssl, false otherwise. - Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. + Name string `json:"name"` // The name of the cookie. + Value string `json:"value"` // The cookie value. + Path string `json:"path,omitempty"` // The path pertaining to the cookie. + Domain string `json:"domain,omitempty"` // The host of the cookie. + Expires string `json:"expires,omitempty"` // Cookie expiration time. (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.123+02:00). + HTTPOnly bool `json:"httpOnly,omitempty"` // Set to true if the cookie is HTTP only, false otherwise. + Secure bool `json:"secure,omitempty"` // True if the cookie was transmitted over ssl, false otherwise. + Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } // NameValuePair describes a name/value pair. From 46f9ace13f2b3d0bf07664664c89ea0aaec7548a Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 02:26:50 +0000 Subject: [PATCH 05/60] fix: cookie and header conversion functions and header size computing --- converter/common.go | 58 +++++++++++++++++++++++ converter/request.go | 98 +++++++++++++++------------------------ converter/request_test.go | 19 ++++++-- 3 files changed, 110 insertions(+), 65 deletions(-) create mode 100644 converter/common.go diff --git a/converter/common.go b/converter/common.go new file mode 100644 index 0000000..c4f953b --- /dev/null +++ b/converter/common.go @@ -0,0 +1,58 @@ +package converter + +import ( + "net/http" + "strings" + "time" + + "github.com/Mathious6/harkit/harfile" +) + +func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { + harCookies := make([]*harfile.Cookie, len(cookies)) + + for i, cookie := range cookies { + var expires string + if !cookie.Expires.IsZero() { + expires = cookie.Expires.Format(time.RFC3339Nano) + } + + harCookies[i] = &harfile.Cookie{ + Name: cookie.Name, + Value: cookie.Value, + Path: cookie.Path, + Domain: cookie.Domain, + Expires: expires, + HTTPOnly: cookie.HttpOnly, + Secure: cookie.Secure, + } + } + + return harCookies +} + +func convertHeaders(header http.Header) []*harfile.NameValuePair { + harHeaders := make([]*harfile.NameValuePair, 0, len(header)) + + // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) + seen := make(map[string]bool) + for _, name := range header.Values("Header-Order") { + if values := header.Values(name); len(values) > 0 { + for _, value := range values { + harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + } + seen[http.CanonicalHeaderKey(name)] = true + } + } + + for name, values := range header { + if seen[name] || strings.EqualFold(name, "Header-Order") { + continue + } + for _, value := range values { + harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + } + } + + return harHeaders +} diff --git a/converter/request.go b/converter/request.go index cae9d39..37cc6e7 100644 --- a/converter/request.go +++ b/converter/request.go @@ -2,6 +2,7 @@ package converter import ( "bytes" + "errors" "io" "net/http" "net/url" @@ -10,14 +11,19 @@ import ( "github.com/Mathious6/harkit/harfile" ) -func FromHTTPRequest(req *http.Request) (harfile.Request, error) { - headers, headersSize := convertHeaders(req) - postData, bodySize, err := extractPostData(req) +func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { + if req == nil { + return nil, errors.New("request cannot be nil") + } + + headers := convertHeaders(req.Header) + + postData, err := extractPostData(req) if err != nil { - return harfile.Request{}, err + return nil, err } - return harfile.Request{ + return &harfile.Request{ Method: req.Method, URL: req.URL.String(), HTTPVersion: req.Proto, @@ -25,56 +31,12 @@ func FromHTTPRequest(req *http.Request) (harfile.Request, error) { Headers: headers, QueryString: convertQueryParams(req.URL), PostData: postData, - HeadersSize: headersSize, - BodySize: bodySize, + HeadersSize: computeRequestHeadersSize(req, headers), + BodySize: req.ContentLength, Comment: "Generated from FromHTTPRequest", }, nil } -func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { - harCookies := make([]*harfile.Cookie, len(cookies)) - - for i, cookie := range cookies { - harCookies[i] = &harfile.Cookie{ - Name: cookie.Name, - Value: cookie.Value, - } - } - - return harCookies -} - -func convertHeaders(req *http.Request) ([]*harfile.NameValuePair, int64) { - harHeaders := make([]*harfile.NameValuePair, 0, len(req.Header)) - - // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) - seen := make(map[string]bool) - for _, name := range req.Header.Values("Header-Order") { - if values := req.Header.Values(name); len(values) > 0 { - for _, value := range values { - harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) - } - seen[http.CanonicalHeaderKey(name)] = true - } - } - - for name, values := range req.Header { - if seen[name] || strings.EqualFold(name, "Header-Order") { - continue - } - for _, value := range values { - harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) - } - } - - headersSize := int64(0) - for _, h := range harHeaders { - headersSize += int64(len(h.Name) + len(h.Value)) - } - - return harHeaders, headersSize -} - func convertQueryParams(u *url.URL) []*harfile.NameValuePair { var result []*harfile.NameValuePair @@ -87,21 +49,20 @@ func convertQueryParams(u *url.URL) []*harfile.NameValuePair { return result } -func extractPostData(req *http.Request) (*harfile.PostData, int64, error) { +func extractPostData(req *http.Request) (*harfile.PostData, error) { if req.Body == nil || req.ContentLength == 0 { - return nil, int64(req.ContentLength), nil + return nil, nil } buf, err := io.ReadAll(req.Body) if err != nil { - return nil, 0, err + return nil, err } defer req.Body.Close() req.Body = io.NopCloser(bytes.NewBuffer(buf)) mimeType := req.Header.Get("Content-Type") postData := &harfile.PostData{MimeType: mimeType} - bodySize := int64(len(buf)) if strings.HasPrefix(mimeType, "application/x-www-form-urlencoded") { text := string(buf) @@ -115,13 +76,13 @@ func extractPostData(req *http.Request) (*harfile.PostData, int64, error) { } } - return postData, bodySize, nil + return postData, nil } if strings.HasPrefix(mimeType, "multipart/form-data") { err := req.ParseMultipartForm(32 << 20) // 32 MB limit if err != nil { - return nil, 0, err + return nil, err } for name, values := range req.MultipartForm.Value { @@ -134,13 +95,13 @@ func extractPostData(req *http.Request) (*harfile.PostData, int64, error) { for _, fileHeader := range files { file, err := fileHeader.Open() if err != nil { - return nil, 0, err + return nil, err } defer file.Close() content, err := io.ReadAll(file) if err != nil { - return nil, 0, err + return nil, err } postData.Params = append(postData.Params, &harfile.Param{ @@ -152,9 +113,24 @@ func extractPostData(req *http.Request) (*harfile.PostData, int64, error) { } } - return postData, bodySize, nil + return postData, nil } postData.Text = string(buf) - return postData, bodySize, nil + return postData, nil +} + +func computeRequestHeadersSize(req *http.Request, harHeaders []*harfile.NameValuePair) int64 { + headersSize := 0 + + requestLine := req.Method + " " + req.URL.RequestURI() + " " + req.Proto + "\r\n" + headersSize += len(requestLine) + + for _, header := range harHeaders { + headerLine := header.Name + ": " + header.Value + "\r\n" + headersSize += len(headerLine) + } + + headersSize += len("\r\n\r\n") + return int64(headersSize) } diff --git a/converter/request_test.go b/converter/request_test.go index 9972396..2ea834c 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -16,6 +16,7 @@ import ( const ( METHOD = http.MethodPost URL = "https://example.com/api?foo=bar" + URI = "/api?foo=bar" PROTOCOL = "HTTP/1.1" HEADER_ORDER_KEY = "Header-Order" @@ -155,10 +156,7 @@ func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersSizeShouldB result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - expectedSize := len(HEADER1_NAME) + len(HEADER1_VALUE) - expectedSize += len(HEADER2_NAME) + len(HEADER2_VALUE) - expectedSize += len(COOKIE_KEY) + len(COOKIE_NAME+"="+COOKIE_VALUE) - assert.Equal(t, int64(expectedSize), result.HeadersSize, "HAR header size <> request header size") + assert.Equal(t, computeHeadersSize(), result.HeadersSize, "HAR header size <> request header size") } func TestConverter_GivenBody_WhenConvertingHTTPRequest_ThenBodySizeShouldBeCorrect(t *testing.T) { @@ -194,6 +192,19 @@ func createRequest(t *testing.T, body io.Reader, contentType string) *http.Reque return req } +// computeHeadersSize calculates the total size of HTTP headers in bytes. +// It sums up the lengths of the HTTP request line, individual headers, +// and the terminating double CRLF sequence. +func computeHeadersSize() int64 { + headersSize := len(METHOD + " " + URI + " " + PROTOCOL + "\r\n") + headersSize += len(HEADER2_NAME + ": " + HEADER2_VALUE + "\r\n") + headersSize += len(HEADER1_NAME + ": " + HEADER1_VALUE + "\r\n") + headersSize += len(COOKIE_KEY + ": " + COOKIE_NAME + "=" + COOKIE_VALUE + "\r\n") + headersSize += len("\r\n\r\n") + + return int64(headersSize) +} + // createMultipartBody constructs a multipart HTTP request body with predefined fields and file content. // It returns the body as a bytes.Buffer and the corresponding Content-Type header value. func createMultipartBody() (body bytes.Buffer, contentType string) { From 4378af2852fa14601b00a74d0fb03a34f4145b29 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 03:23:27 +0000 Subject: [PATCH 06/60] refactor: replace string literals with constants for main keys in converter --- converter/common.go | 12 +++- converter/request.go | 4 +- converter/request_test.go | 139 ++++++++++++++++++++------------------ 3 files changed, 84 insertions(+), 71 deletions(-) diff --git a/converter/common.go b/converter/common.go index c4f953b..812df71 100644 --- a/converter/common.go +++ b/converter/common.go @@ -8,6 +8,14 @@ import ( "github.com/Mathious6/harkit/harfile" ) +const ( + HeaderOrderKey = "Header-Order" + ContentTypeKey = "Content-Type" + CookieKey = "Cookie" + SetCookieKey = "Set-Cookie" + LocationKey = "Location" +) + func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { harCookies := make([]*harfile.Cookie, len(cookies)) @@ -36,7 +44,7 @@ func convertHeaders(header http.Header) []*harfile.NameValuePair { // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) seen := make(map[string]bool) - for _, name := range header.Values("Header-Order") { + for _, name := range header.Values(HeaderOrderKey) { if values := header.Values(name); len(values) > 0 { for _, value := range values { harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) @@ -46,7 +54,7 @@ func convertHeaders(header http.Header) []*harfile.NameValuePair { } for name, values := range header { - if seen[name] || strings.EqualFold(name, "Header-Order") { + if seen[name] || strings.EqualFold(name, HeaderOrderKey) { continue } for _, value := range values { diff --git a/converter/request.go b/converter/request.go index 37cc6e7..3106725 100644 --- a/converter/request.go +++ b/converter/request.go @@ -61,7 +61,7 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { defer req.Body.Close() req.Body = io.NopCloser(bytes.NewBuffer(buf)) - mimeType := req.Header.Get("Content-Type") + mimeType := req.Header.Get(ContentTypeKey) postData := &harfile.PostData{MimeType: mimeType} if strings.HasPrefix(mimeType, "application/x-www-form-urlencoded") { @@ -107,7 +107,7 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { postData.Params = append(postData.Params, &harfile.Param{ Name: name, FileName: fileHeader.Filename, - ContentType: fileHeader.Header.Get("Content-Type"), + ContentType: fileHeader.Header.Get(ContentTypeKey), Value: string(content), }) } diff --git a/converter/request_test.go b/converter/request_test.go index 2ea834c..546285e 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -14,44 +14,49 @@ import ( ) const ( - METHOD = http.MethodPost - URL = "https://example.com/api?foo=bar" - URI = "/api?foo=bar" - PROTOCOL = "HTTP/1.1" - - HEADER_ORDER_KEY = "Header-Order" - CONTENT_TYPE_KEY = "Content-Type" - COOKIE_KEY = "Cookie" - - HEADER1_NAME = "NaMe1" - HEADER1_VALUE = "value1" - HEADER2_NAME = "nAmE2" - HEADER2_VALUE = "value2" - - COOKIE_NAME = "name" - COOKIE_VALUE = "value" - - URL_CONTENT_TYPE = "application/x-www-form-urlencoded" - BODY_URL = "foo=bar" - - JSON_CONTENT_TYPE = "application/json" - BODY_JSON = `{"foo":"bar"}` - - PART1_NAME = "name1" - PART1_VALUE = "value1" - PART2_NAME = "file" - PART2_VALUE = "content" - PART2_FILENAME = "test.txt" - PART2_CONTENT_TYPE = "application/octet-stream" + REQ_METHOD = http.MethodPost + REQ_URL = "https://example.com/api?foo=bar" + REQ_URI = "/api?foo=bar" + REQ_PROTOCOL = "HTTP/1.1" + + REQ_HEADER1_NAME = "NaMe1" + REQ_HEADER1_VALUE = "value1" + REQ_HEADER2_NAME = "nAmE2" + REQ_HEADER2_VALUE = "value2" + + REQ_COOKIE_NAME = "name" + REQ_COOKIE_VALUE = "value" + + REQ_URL_CONTENT_TYPE = "application/x-www-form-urlencoded" + REQ_BODY_URL = "foo=bar" + + REQ_JSON_CONTENT_TYPE = "application/json" + REQ_BODY_JSON = `{"foo":"bar"}` + + REQ_PART1_NAME = "name1" + REQ_PART1_VALUE = "value1" + REQ_PART2_NAME = "file" + REQ_PART2_VALUE = "content" + REQ_PART2_FILENAME = "test.txt" + REQ_PART2_CONTENT_TYPE = "application/octet-stream" ) +func TestConverter_GivenNilRequest_WhenConvertingHTTPRequest_ThenErrorShouldBeReturned(t *testing.T) { + req := (*http.Request)(nil) + + result, err := converter.FromHTTPRequest(req) + + assert.Error(t, err, "Error should be returned when request is nil") + assert.Nil(t, result, "HAR should be nil when request is nil") +} + func TestConverter_GivenMethod_WhenConvertingHTTPRequest_ThenMethodShouldBeCorrect(t *testing.T) { req := createRequest(t, nil, "") result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, METHOD, result.Method, "HAR method <> request method") + assert.Equal(t, REQ_METHOD, result.Method, "HAR method <> request method") } func TestConverter_GivenURL_WhenConvertingHTTPRequest_ThenURLShouldBeCorrect(t *testing.T) { @@ -60,7 +65,7 @@ func TestConverter_GivenURL_WhenConvertingHTTPRequest_ThenURLShouldBeCorrect(t * result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, URL, result.URL, "HAR URL <> request URL") + assert.Equal(t, REQ_URL, result.URL, "HAR URL <> request URL") } func TestConverter_GivenProtocol_WhenConvertingHTTPRequest_ThenProtocolShouldBeCorrect(t *testing.T) { @@ -69,7 +74,7 @@ func TestConverter_GivenProtocol_WhenConvertingHTTPRequest_ThenProtocolShouldBeC result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, PROTOCOL, result.HTTPVersion, "HAR protocol <> request protocol") + assert.Equal(t, REQ_PROTOCOL, result.HTTPVersion, "HAR protocol <> request protocol") } func TestConverter_GivenCookies_WhenConvertingHTTPRequest_ThenCookiesShouldBeCorrect(t *testing.T) { @@ -79,8 +84,8 @@ func TestConverter_GivenCookies_WhenConvertingHTTPRequest_ThenCookiesShouldBeCor require.NoError(t, err) assert.Len(t, result.Cookies, 1, "HAR should contain 1 cookie") - assert.Equal(t, COOKIE_NAME, result.Cookies[0].Name, "HAR cookie name <> request cookie name") - assert.Equal(t, COOKIE_VALUE, result.Cookies[0].Value, "HAR cookie value <> request cookie value") + assert.Equal(t, REQ_COOKIE_NAME, result.Cookies[0].Name, "HAR cookie name <> request cookie name") + assert.Equal(t, REQ_COOKIE_VALUE, result.Cookies[0].Value, "HAR cookie value <> request cookie value") } func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersShouldBeCorrectAndOrdered(t *testing.T) { @@ -89,11 +94,11 @@ func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersShouldBeCor result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Len(t, result.Headers, 3, "HAR should contain 7 headers") - assert.Equal(t, HEADER2_NAME, result.Headers[0].Name, "HAR header name <> request header name") - assert.Equal(t, HEADER2_VALUE, result.Headers[0].Value, "HAR header value <> request header value") - assert.Equal(t, HEADER1_NAME, result.Headers[1].Name, "HAR header name <> request header name") - assert.Equal(t, HEADER1_VALUE, result.Headers[1].Value, "HAR header value <> request header value") + assert.Len(t, result.Headers, 3, "HAR should contain 3 headers") + assert.Equal(t, REQ_HEADER2_NAME, result.Headers[0].Name, "HAR header name <> request header name") + assert.Equal(t, REQ_HEADER2_VALUE, result.Headers[0].Value, "HAR header value <> request header value") + assert.Equal(t, REQ_HEADER1_NAME, result.Headers[1].Name, "HAR header name <> request header name") + assert.Equal(t, REQ_HEADER1_VALUE, result.Headers[1].Value, "HAR header value <> request header value") } func TestConverter_GivenURLWithQueryString_WhenConvertingHTTPRequest_ThenQueryStringShouldBeCorrect(t *testing.T) { @@ -108,7 +113,7 @@ func TestConverter_GivenURLWithQueryString_WhenConvertingHTTPRequest_ThenQuerySt } func TestConverter_GivenURLEncodedBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { - req := createRequest(t, strings.NewReader(BODY_URL), URL_CONTENT_TYPE) + req := createRequest(t, strings.NewReader(REQ_BODY_URL), REQ_URL_CONTENT_TYPE) result, err := converter.FromHTTPRequest(req) require.NoError(t, err) @@ -117,19 +122,19 @@ func TestConverter_GivenURLEncodedBody_WhenConvertingHTTPRequest_ThenPostDataSho assert.Empty(t, result.PostData.Text, "HAR should have no post data text") assert.Equal(t, "foo", result.PostData.Params[0].Name, "HAR post data name <> request post data name") assert.Equal(t, "bar", result.PostData.Params[0].Value, "HAR post data value <> request post data value") - assert.Equal(t, URL_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") + assert.Equal(t, REQ_URL_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") } func TestConverter_GivenJSONBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { - req := createRequest(t, strings.NewReader(BODY_JSON), JSON_CONTENT_TYPE) + req := createRequest(t, strings.NewReader(REQ_BODY_JSON), REQ_JSON_CONTENT_TYPE) result, err := converter.FromHTTPRequest(req) require.NoError(t, err) assert.Len(t, result.PostData.Params, 0, "HAR should contain 0 post data parameters") assert.NotEmpty(t, result.PostData.Text, "HAR should have post data text") - assert.Equal(t, BODY_JSON, result.PostData.Text, "HAR post data text <> request post data text") - assert.Equal(t, JSON_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") + assert.Equal(t, REQ_BODY_JSON, result.PostData.Text, "HAR post data text <> request post data text") + assert.Equal(t, REQ_JSON_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") } func TestConverter_GivenMultipartBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { @@ -141,12 +146,12 @@ func TestConverter_GivenMultipartBody_WhenConvertingHTTPRequest_ThenPostDataShou assert.Len(t, result.PostData.Params, 2, "HAR should contain 2 post data parameters") assert.Empty(t, result.PostData.Text, "HAR should have no post data text") - assert.Equal(t, PART1_NAME, result.PostData.Params[0].Name, "HAR post data name <> request post data name") - assert.Equal(t, PART1_VALUE, result.PostData.Params[0].Value, "HAR post data value <> request post data value") - assert.Equal(t, PART2_NAME, result.PostData.Params[1].Name, "HAR post data name <> request post data name") - assert.Equal(t, PART2_VALUE, result.PostData.Params[1].Value, "HAR post data value <> request post data value") - assert.Equal(t, PART2_FILENAME, result.PostData.Params[1].FileName, "HAR post data filename <> request post data filename") - assert.Equal(t, PART2_CONTENT_TYPE, result.PostData.Params[1].ContentType, "HAR post data content type <> request post data content type") + assert.Equal(t, REQ_PART1_NAME, result.PostData.Params[0].Name, "HAR post data name <> request post data name") + assert.Equal(t, REQ_PART1_VALUE, result.PostData.Params[0].Value, "HAR post data value <> request post data value") + assert.Equal(t, REQ_PART2_NAME, result.PostData.Params[1].Name, "HAR post data name <> request post data name") + assert.Equal(t, REQ_PART2_VALUE, result.PostData.Params[1].Value, "HAR post data value <> request post data value") + assert.Equal(t, REQ_PART2_FILENAME, result.PostData.Params[1].FileName, "HAR post data filename <> request post data filename") + assert.Equal(t, REQ_PART2_CONTENT_TYPE, result.PostData.Params[1].ContentType, "HAR post data content type <> request post data content type") assert.Equal(t, contentType, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") } @@ -160,12 +165,12 @@ func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersSizeShouldB } func TestConverter_GivenBody_WhenConvertingHTTPRequest_ThenBodySizeShouldBeCorrect(t *testing.T) { - req := createRequest(t, strings.NewReader(BODY_URL), URL_CONTENT_TYPE) + req := createRequest(t, strings.NewReader(REQ_BODY_URL), REQ_URL_CONTENT_TYPE) result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - expectedSize := int64(len(BODY_URL)) + expectedSize := int64(len(REQ_BODY_URL)) assert.Equal(t, expectedSize, result.BodySize, "HAR body size <> request body size") } @@ -173,20 +178,20 @@ func TestConverter_GivenBody_WhenConvertingHTTPRequest_ThenBodySizeShouldBeCorre // It sets various headers including cookies, custom headers, and content type if provided. // The function also ensures the request protocol is set and validates the request creation. func createRequest(t *testing.T, body io.Reader, contentType string) *http.Request { - req, err := http.NewRequest(METHOD, URL, body) + req, err := http.NewRequest(REQ_METHOD, REQ_URL, body) require.NoError(t, err) - req.Proto = PROTOCOL + req.Proto = REQ_PROTOCOL - req.Header.Add(COOKIE_KEY, COOKIE_NAME+"="+COOKIE_VALUE) - req.Header.Add(HEADER1_NAME, HEADER1_VALUE) - req.Header.Add(HEADER2_NAME, HEADER2_VALUE) + req.Header.Add(converter.CookieKey, REQ_COOKIE_NAME+"="+REQ_COOKIE_VALUE) + req.Header.Add(REQ_HEADER1_NAME, REQ_HEADER1_VALUE) + req.Header.Add(REQ_HEADER2_NAME, REQ_HEADER2_VALUE) - req.Header.Add(HEADER_ORDER_KEY, HEADER2_NAME) - req.Header.Add(HEADER_ORDER_KEY, HEADER1_NAME) + req.Header.Add(converter.HeaderOrderKey, REQ_HEADER2_NAME) + req.Header.Add(converter.HeaderOrderKey, REQ_HEADER1_NAME) if contentType != "" { - req.Header.Add(CONTENT_TYPE_KEY, contentType) + req.Header.Add(converter.ContentTypeKey, contentType) } return req @@ -196,10 +201,10 @@ func createRequest(t *testing.T, body io.Reader, contentType string) *http.Reque // It sums up the lengths of the HTTP request line, individual headers, // and the terminating double CRLF sequence. func computeHeadersSize() int64 { - headersSize := len(METHOD + " " + URI + " " + PROTOCOL + "\r\n") - headersSize += len(HEADER2_NAME + ": " + HEADER2_VALUE + "\r\n") - headersSize += len(HEADER1_NAME + ": " + HEADER1_VALUE + "\r\n") - headersSize += len(COOKIE_KEY + ": " + COOKIE_NAME + "=" + COOKIE_VALUE + "\r\n") + headersSize := len(REQ_METHOD + " " + REQ_URI + " " + REQ_PROTOCOL + "\r\n") + headersSize += len(REQ_HEADER2_NAME + ": " + REQ_HEADER2_VALUE + "\r\n") + headersSize += len(REQ_HEADER1_NAME + ": " + REQ_HEADER1_VALUE + "\r\n") + headersSize += len(converter.CookieKey + ": " + REQ_COOKIE_NAME + "=" + REQ_COOKIE_VALUE + "\r\n") headersSize += len("\r\n\r\n") return int64(headersSize) @@ -211,10 +216,10 @@ func createMultipartBody() (body bytes.Buffer, contentType string) { var buf bytes.Buffer writer := multipart.NewWriter(&buf) - _ = writer.WriteField(PART1_NAME, PART1_VALUE) + _ = writer.WriteField(REQ_PART1_NAME, REQ_PART1_VALUE) - fileWriter, _ := writer.CreateFormFile(PART2_NAME, PART2_FILENAME) - _, _ = fileWriter.Write([]byte(PART2_VALUE)) + fileWriter, _ := writer.CreateFormFile(REQ_PART2_NAME, REQ_PART2_FILENAME) + _, _ = fileWriter.Write([]byte(REQ_PART2_VALUE)) writer.Close() From b2da8e5532287008433ea0705fbc49f624a4dd70 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 03:44:10 +0000 Subject: [PATCH 07/60] feat: implement FromHTTPResponse function and associated tests for HTTP response conversion --- converter/response.go | 60 +++++++++++++ converter/response_test.go | 167 +++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 converter/response.go create mode 100644 converter/response_test.go diff --git a/converter/response.go b/converter/response.go new file mode 100644 index 0000000..517463d --- /dev/null +++ b/converter/response.go @@ -0,0 +1,60 @@ +package converter + +import ( + "bytes" + "errors" + "io" + "net/http" + + "github.com/Mathious6/harkit/harfile" +) + +func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { + if resp == nil { + return nil, errors.New("response cannot be nil") + } + + content, err := buildContent(resp) + if err != nil { + return nil, err + } + + return &harfile.Response{ + Status: int64(resp.StatusCode), + StatusText: http.StatusText(resp.StatusCode), + HTTPVersion: resp.Proto, + Cookies: convertCookies(resp.Cookies()), + Headers: convertHeaders(resp.Header), + Content: content, + RedirectURL: locateRedirectURL(resp), + HeadersSize: -1, + BodySize: resp.ContentLength, + Comment: "Generated from FromHTTPResponse", + }, nil +} + +func locateRedirectURL(resp *http.Response) string { + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + if loc, err := resp.Location(); err == nil { + return loc.String() + } + } + return "" +} + +func buildContent(resp *http.Response) (*harfile.Content, error) { + buf, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + resp.Body = io.NopCloser(bytes.NewBuffer(buf)) + + return &harfile.Content{ + Size: int64(len(buf)), + Compression: 0, + MimeType: resp.Header.Get(ContentTypeKey), + Text: string(buf), + Encoding: "", + }, nil +} diff --git a/converter/response_test.go b/converter/response_test.go new file mode 100644 index 0000000..aaf6aea --- /dev/null +++ b/converter/response_test.go @@ -0,0 +1,167 @@ +package converter_test + +import ( + "bytes" + "io" + "net/http" + "testing" + "time" + + "github.com/Mathious6/harkit/converter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + RESP_OK_CODE = http.StatusOK + RESP_FOUND_CODE = http.StatusFound + RESP_PROTOCOL = "HTTP/1.1" + + RESP_HEADER_NAME = "Name" + RESP_HEADER_VALUE = "value" + + RESP_COOKIE_NAME = "name" + RESP_COOKIE_VALUE = "value" + RESP_COOKIE_PATH = "/" + RESP_COOKIE_DOMAIN = "example.com" + RESP_COOKIE_EXPIRES = "Mon, 12 May 2025 00:00:00 GMT" + RESP_COOKIE_HTTPONLY = true + RESP_COOKIE_SECURE = true + + RESP_BODY_TEXT = "response" + RESP_CONTENT_TYPE = "text/plain" + + RESP_LOCATION = "https://example.com/redirect" +) + +func TestConverter_GivenNilResponse_WhenConvertingHTTPResponse_ThenErrorShouldBeReturned(t *testing.T) { + resp := (*http.Response)(nil) + + result, err := converter.FromHTTPResponse(resp) + + assert.Error(t, err, "Error should be returned when response is nil") + assert.Nil(t, result, "HAR should be nil when response is nil") +} + +func TestConverter_GivenStatusCode_WhenConvertingHTTPResponse_ThenStatusShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, int64(resp.StatusCode), result.Status, "HAR status <> response status") +} + +func TestConverter_GivenStatusText_WhenConvertingHTTPResponse_ThenStatusTextShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, http.StatusText(resp.StatusCode), result.StatusText, "HAR status text <> response status text") +} + +func TestConverter_GivenProtocol_WhenConvertingHTTPResponse_ThenProtocolShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, resp.Proto, result.HTTPVersion, "HAR protocol <> response protocol") +} + +func TestConverter_GivenCookies_WhenConvertingHTTPResponse_ThenCookiesShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + exceptedExpires, _ := time.Parse(time.RFC1123, RESP_COOKIE_EXPIRES) + exceptedExpiresStr := exceptedExpires.Format(time.RFC3339Nano) + assert.Len(t, result.Cookies, 1, "HAR should contain 1 cookie") + assert.Equal(t, RESP_COOKIE_NAME, result.Cookies[0].Name, "HAR cookie name <> response cookie name") + assert.Equal(t, RESP_COOKIE_VALUE, result.Cookies[0].Value, "HAR cookie value <> response cookie value") + assert.Equal(t, RESP_COOKIE_PATH, result.Cookies[0].Path, "HAR cookie path <> response cookie path") + assert.Equal(t, RESP_COOKIE_DOMAIN, result.Cookies[0].Domain, "HAR cookie domain <> response cookie domain") + assert.Equal(t, exceptedExpiresStr, result.Cookies[0].Expires, "HAR cookie expires <> response cookie expires") + assert.Equal(t, RESP_COOKIE_HTTPONLY, result.Cookies[0].HTTPOnly, "HAR cookie httpOnly <> response cookie httpOnly") + assert.Equal(t, RESP_COOKIE_SECURE, result.Cookies[0].Secure, "HAR cookie secure <> response cookie secure") +} + +func TestConverter_GivenHeaders_WhenConvertingHTTPResponse_ThenHeadersShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Len(t, result.Headers, 2, "HAR should contain 2 header") + assert.Equal(t, RESP_HEADER_NAME, result.Headers[0].Name, "HAR header name <> response header name") + assert.Equal(t, RESP_HEADER_VALUE, result.Headers[0].Value, "HAR header value <> response header value") +} + +func TestConverter_GivenBody_WhenConvertingHTTPResponse_ThenContentShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, bytes.NewBufferString(RESP_BODY_TEXT), RESP_CONTENT_TYPE) + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, int64(len(RESP_BODY_TEXT)), result.Content.Size, "HAR content size <> response body size") + assert.Equal(t, RESP_CONTENT_TYPE, result.Content.MimeType, "HAR content mime type <> response content mime type") + assert.Equal(t, RESP_BODY_TEXT, result.Content.Text, "HAR content text <> response body text") +} + +func TestConverter_GivenRedirect_WhenConvertingHTTPResponse_ThenRedirectURLShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_FOUND_CODE, nil, "") + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, RESP_LOCATION, result.RedirectURL, "HAR redirect URL <> response location header") +} + +func TestConverter_GivenBody_WhenConvertingHTTPResponse_ThenBodySizeShouldBeCorrect(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, bytes.NewBufferString(RESP_BODY_TEXT), RESP_CONTENT_TYPE) + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Equal(t, int64(len(RESP_BODY_TEXT)), result.BodySize, "HAR body size <> response body size") +} + +func createResponse(t *testing.T, statusCode int, body io.Reader, contentType string) *http.Response { + var buf []byte + var err error + if body != nil { + buf, err = io.ReadAll(body) + require.NoError(t, err) + } + + resp := &http.Response{ + StatusCode: statusCode, + Status: http.StatusText(statusCode), + Proto: RESP_PROTOCOL, + Header: make(http.Header), + ContentLength: int64(len(buf)), + Body: io.NopCloser(bytes.NewBuffer(buf)), + } + + cookie := RESP_COOKIE_NAME + "=" + RESP_COOKIE_VALUE + cookie += ";path=" + RESP_COOKIE_PATH + cookie += ";domain=" + RESP_COOKIE_DOMAIN + cookie += ";expires=" + RESP_COOKIE_EXPIRES + cookie += ";httponly" + cookie += ";secure" + + resp.Header.Add(RESP_HEADER_NAME, RESP_HEADER_VALUE) + resp.Header.Add(converter.SetCookieKey, cookie) + + if contentType != "" { + resp.Header.Set(converter.ContentTypeKey, contentType) + } + + if statusCode >= 300 && statusCode < 400 { + resp.Header.Set(converter.LocationKey, RESP_LOCATION) + } + + return resp +} From 41052e4dbe5748967b76bc7c197ed31886ac9198 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 03:44:33 +0000 Subject: [PATCH 08/60] feat: implement BuildHAR function to create HAR format from HTTP request and response --- converter/har.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 converter/har.go diff --git a/converter/har.go b/converter/har.go new file mode 100644 index 0000000..7533731 --- /dev/null +++ b/converter/har.go @@ -0,0 +1,40 @@ +package converter + +import ( + "net/http" + "time" + + "github.com/Mathious6/harkit/harfile" +) + +func BuildHAR(req *http.Request, resp *http.Response) (*harfile.HAR, error) { + harReq, err := FromHTTPRequest(req) + if err != nil { + return nil, err + } + + harResp, err := FromHTTPResponse(resp) + if err != nil { + return nil, err + } + + return &harfile.HAR{ + Log: &harfile.Log{ + Version: "1.2", + Creator: &harfile.Creator{ + Name: "harkit", + Version: "0.2.0", + }, + Entries: []*harfile.Entry{ + { + StartedDateTime: time.Now(), + Time: 0, + Request: harReq, + Response: harResp, + Cache: &harfile.Cache{}, + Timings: &harfile.Timings{}, + }, + }, + }, + }, nil +} From 1dd631a37221af5dbde88f306fc9d01a418b9b1a Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 10 Apr 2025 03:45:18 +0000 Subject: [PATCH 09/60] feat: add example main.go for HAR file generation and management --- .gitignore | 1 + README.md | 3 +++ example/main.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 .gitignore create mode 100644 example/main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8659d3d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.har diff --git a/README.md b/README.md index 25389fe..9034e70 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ # HAR file management library A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. + +- [ ] Check for case sensitivity in HeaderOrder keys +- [ ] Add dynamic versioning to the library diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..9526a69 --- /dev/null +++ b/example/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + + "github.com/Mathious6/harkit/converter" +) + +func main() { + req, _ := http.NewRequest(http.MethodPost, "https://httpbin.org/post?source=harkit", nil) + req.Header.Add("user-agent", "harkit-example") + req.Header.Add("accept", "application/json") + req.Header.Add(converter.HeaderOrderKey, "accept") // TIPS: Header order is important for some TLS clients. + req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) + + resp, _ := http.DefaultClient.Do(req) + + har, err := converter.BuildHAR(req, resp) + if err != nil { + panic(err) + } + + jsonBytes, _ := json.MarshalIndent(har, "", " ") + _ = os.WriteFile("main.har", jsonBytes, 0644) + fmt.Println("✅ HAR file saved as main.har") +} From a275c774b8c7c401accf6acdbbece06a8abff156 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 15:29:26 +0000 Subject: [PATCH 10/60] refactor: replace net/http with bogdanfinn/fhttp --- converter/common.go | 2 +- converter/har.go | 2 +- converter/request.go | 2 +- converter/request_test.go | 2 +- converter/response.go | 2 +- converter/response_test.go | 2 +- go.mod | 14 +++++++++++++- go.sum | 24 ++++++++++++++++++++++++ 8 files changed, 43 insertions(+), 7 deletions(-) diff --git a/converter/common.go b/converter/common.go index 812df71..872a524 100644 --- a/converter/common.go +++ b/converter/common.go @@ -1,11 +1,11 @@ package converter import ( - "net/http" "strings" "time" "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" ) const ( diff --git a/converter/har.go b/converter/har.go index 7533731..daf084a 100644 --- a/converter/har.go +++ b/converter/har.go @@ -1,10 +1,10 @@ package converter import ( - "net/http" "time" "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" ) func BuildHAR(req *http.Request, resp *http.Response) (*harfile.HAR, error) { diff --git a/converter/request.go b/converter/request.go index 3106725..65945f6 100644 --- a/converter/request.go +++ b/converter/request.go @@ -4,11 +4,11 @@ import ( "bytes" "errors" "io" - "net/http" "net/url" "strings" "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" ) func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { diff --git a/converter/request_test.go b/converter/request_test.go index 546285e..5dfdccd 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -4,11 +4,11 @@ import ( "bytes" "io" "mime/multipart" - "net/http" "strings" "testing" "github.com/Mathious6/harkit/converter" + http "github.com/bogdanfinn/fhttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/converter/response.go b/converter/response.go index 517463d..82ad985 100644 --- a/converter/response.go +++ b/converter/response.go @@ -4,9 +4,9 @@ import ( "bytes" "errors" "io" - "net/http" "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" ) func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { diff --git a/converter/response_test.go b/converter/response_test.go index aaf6aea..a3c94e8 100644 --- a/converter/response_test.go +++ b/converter/response_test.go @@ -3,11 +3,11 @@ package converter_test import ( "bytes" "io" - "net/http" "testing" "time" "github.com/Mathious6/harkit/converter" + http "github.com/bogdanfinn/fhttp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/go.mod b/go.mod index 0396592..b975c3d 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,22 @@ module github.com/Mathious6/harkit go 1.24.2 -require github.com/stretchr/testify v1.10.0 +require ( + github.com/bogdanfinn/fhttp v0.5.36 + github.com/stretchr/testify v1.10.0 +) require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/bogdanfinn/utls v1.6.5 // indirect + github.com/cloudflare/circl v1.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/quic-go/quic-go v0.48.1 // indirect + golang.org/x/crypto v0.29.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.20.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 713a0b4..4a53819 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,33 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/bogdanfinn/fhttp v0.5.36 h1:t1sO/EkO4K40QD/Ti8f6t80leZIdh2AaeLfN7dMvjH8= +github.com/bogdanfinn/fhttp v0.5.36/go.mod h1:BlcawVfXJ4uhk5yyNGOOY2bwo8UmMi6ccMszP1KGLkU= +github.com/bogdanfinn/utls v1.6.5 h1:rVMQvhyN3zodLxKFWMRLt19INGBCZ/OM2/vBWPNIt1w= +github.com/bogdanfinn/utls v1.6.5/go.mod h1:czcHxHGsc1q9NjgWSeSinQZzn6MR76zUmGVIGanSXO0= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= +github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 47bf11569a51afc49f023289e941ef8934cd35b5 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 15:57:05 +0000 Subject: [PATCH 11/60] fix: return empty QueryString[] instead of nil --- converter/request.go | 3 +-- converter/response.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/converter/request.go b/converter/request.go index 65945f6..30df469 100644 --- a/converter/request.go +++ b/converter/request.go @@ -33,12 +33,11 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { PostData: postData, HeadersSize: computeRequestHeadersSize(req, headers), BodySize: req.ContentLength, - Comment: "Generated from FromHTTPRequest", }, nil } func convertQueryParams(u *url.URL) []*harfile.NameValuePair { - var result []*harfile.NameValuePair + result := make([]*harfile.NameValuePair, 0) for key, values := range u.Query() { for _, value := range values { diff --git a/converter/response.go b/converter/response.go index 82ad985..b0279d2 100644 --- a/converter/response.go +++ b/converter/response.go @@ -29,7 +29,6 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { RedirectURL: locateRedirectURL(resp), HeadersSize: -1, BodySize: resp.ContentLength, - Comment: "Generated from FromHTTPResponse", }, nil } From 778025e82e4e5378e1336c16edf2d78011a42288 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 15:57:43 +0000 Subject: [PATCH 12/60] feat: update example main.go to use tls_client --- .gitignore | 1 + example/go.mod | 24 ++++++++++++++++++++++++ example/go.sum | 36 ++++++++++++++++++++++++++++++++++++ example/main.go | 23 ++++++++++++++++++----- 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 example/go.mod create mode 100644 example/go.sum diff --git a/.gitignore b/.gitignore index 8659d3d..f210504 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ +example-harkit *.har diff --git a/example/go.mod b/example/go.mod new file mode 100644 index 0000000..c6210d7 --- /dev/null +++ b/example/go.mod @@ -0,0 +1,24 @@ +module github.com/Mathious6/example-harkit + +go 1.24.2 + +require ( + github.com/Mathious6/harkit v0.1.1 + github.com/bogdanfinn/fhttp v0.5.36 + github.com/bogdanfinn/tls-client v1.9.1 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/bogdanfinn/utls v1.6.5 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/quic-go/quic-go v0.50.1 // indirect + github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect +) + +replace github.com/Mathious6/harkit => ../ diff --git a/example/go.sum b/example/go.sum new file mode 100644 index 0000000..c893b0c --- /dev/null +++ b/example/go.sum @@ -0,0 +1,36 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/bogdanfinn/fhttp v0.5.36 h1:t1sO/EkO4K40QD/Ti8f6t80leZIdh2AaeLfN7dMvjH8= +github.com/bogdanfinn/fhttp v0.5.36/go.mod h1:BlcawVfXJ4uhk5yyNGOOY2bwo8UmMi6ccMszP1KGLkU= +github.com/bogdanfinn/tls-client v1.9.1 h1:Br0WkKL+/7Q9FSNM1zBMdlYXW8bm+XXGMn9iyb9a/7Y= +github.com/bogdanfinn/tls-client v1.9.1/go.mod h1:ehNITC7JBFeh6S7QNWtfD+PBKm0RsqvizAyyij2d/6g= +github.com/bogdanfinn/utls v1.6.5 h1:rVMQvhyN3zodLxKFWMRLt19INGBCZ/OM2/vBWPNIt1w= +github.com/bogdanfinn/utls v1.6.5/go.mod h1:czcHxHGsc1q9NjgWSeSinQZzn6MR76zUmGVIGanSXO0= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= +github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= +github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/example/main.go b/example/main.go index 9526a69..23f3101 100644 --- a/example/main.go +++ b/example/main.go @@ -3,20 +3,33 @@ package main import ( "encoding/json" "fmt" - "net/http" "os" "github.com/Mathious6/harkit/converter" + http "github.com/bogdanfinn/fhttp" + tls_client "github.com/bogdanfinn/tls-client" ) func main() { - req, _ := http.NewRequest(http.MethodPost, "https://httpbin.org/post?source=harkit", nil) + jar := tls_client.NewCookieJar() + options := []tls_client.HttpClientOption{ + tls_client.WithCookieJar(jar), + tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO: REMOVE BEFORE PUSHING + } + + client, _ := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) + + req, _ := http.NewRequest(http.MethodGet, "https://tls.peet.ws/api/all", nil) + req.Header.Add("accept", "*/*") req.Header.Add("user-agent", "harkit-example") - req.Header.Add("accept", "application/json") - req.Header.Add(converter.HeaderOrderKey, "accept") // TIPS: Header order is important for some TLS clients. + req.Header.Add(http.HeaderOrderKey, "user-agent") req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) - resp, _ := http.DefaultClient.Do(req) + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() har, err := converter.BuildHAR(req, resp) if err != nil { From 018968beaf475e71afa46b075136878d44ff49c7 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 18:30:50 +0000 Subject: [PATCH 13/60] chore: add launch configuration for debugging example/main.go --- .vscode/launch.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..2e33b1c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug example/main.go", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/example/main.go" + } + ] +} From 63fb4eea98208ad92ef98688429e3137a2987487 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 18:31:30 +0000 Subject: [PATCH 14/60] fix: replace HeaderOrderKey with http.HeaderOrderKey in common and request test files --- converter/common.go | 8 ++++---- converter/request_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/converter/common.go b/converter/common.go index 872a524..989a861 100644 --- a/converter/common.go +++ b/converter/common.go @@ -9,7 +9,6 @@ import ( ) const ( - HeaderOrderKey = "Header-Order" ContentTypeKey = "Content-Type" CookieKey = "Cookie" SetCookieKey = "Set-Cookie" @@ -44,17 +43,18 @@ func convertHeaders(header http.Header) []*harfile.NameValuePair { // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) seen := make(map[string]bool) - for _, name := range header.Values(HeaderOrderKey) { + for _, name := range header.Values(http.HeaderOrderKey) { if values := header.Values(name); len(values) > 0 { + name = http.CanonicalHeaderKey(name) for _, value := range values { harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) } - seen[http.CanonicalHeaderKey(name)] = true + seen[name] = true } } for name, values := range header { - if seen[name] || strings.EqualFold(name, HeaderOrderKey) { + if seen[name] || strings.EqualFold(name, http.HeaderOrderKey) { continue } for _, value := range values { diff --git a/converter/request_test.go b/converter/request_test.go index 5dfdccd..c3662ba 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -187,8 +187,8 @@ func createRequest(t *testing.T, body io.Reader, contentType string) *http.Reque req.Header.Add(REQ_HEADER1_NAME, REQ_HEADER1_VALUE) req.Header.Add(REQ_HEADER2_NAME, REQ_HEADER2_VALUE) - req.Header.Add(converter.HeaderOrderKey, REQ_HEADER2_NAME) - req.Header.Add(converter.HeaderOrderKey, REQ_HEADER1_NAME) + req.Header.Add(http.HeaderOrderKey, REQ_HEADER2_NAME) + req.Header.Add(http.HeaderOrderKey, REQ_HEADER1_NAME) if contentType != "" { req.Header.Add(converter.ContentTypeKey, contentType) From 03b040a05dad71e1989a5cfff2a638d5e86907d7 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 18:54:21 +0000 Subject: [PATCH 15/60] feat: add getServerIPAddress function to retrieve server IP address in BuildHAR --- converter/har.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/converter/har.go b/converter/har.go index daf084a..7abb04c 100644 --- a/converter/har.go +++ b/converter/har.go @@ -1,6 +1,8 @@ package converter import ( + "context" + "net" "time" "github.com/Mathious6/harkit/harfile" @@ -31,6 +33,7 @@ func BuildHAR(req *http.Request, resp *http.Response) (*harfile.HAR, error) { Time: 0, Request: harReq, Response: harResp, + ServerIPAddress: getServerIPAddress(req.Host), Cache: &harfile.Cache{}, Timings: &harfile.Timings{}, }, @@ -38,3 +41,11 @@ func BuildHAR(req *http.Request, resp *http.Response) (*harfile.HAR, error) { }, }, nil } + +func getServerIPAddress(host string) string { + ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), host) + if err != nil || len(ipAddress) == 0 { + return "" + } + return ipAddress[0].IP.String() +} From f5641f4e45f281b01bbd972c35b5eecefe5a5b47 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:43:34 +0000 Subject: [PATCH 16/60] chore: add .DS_Store to .gitignore to prevent tracking of macOS system files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f210504..787a256 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ example-harkit *.har + +.DS_Store From 22c74a0bce88827fe87730a39f078e3286a2e570 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:44:49 +0000 Subject: [PATCH 17/60] feat: add Save method to HAR struct for exporting to JSON file --- harfile/har.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/harfile/har.go b/harfile/har.go index 4df02a6..8cf4f68 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -2,7 +2,11 @@ // See: http://www.softwareishard.com/blog/har-12-spec/ package harfile -import "time" +import ( + "encoding/json" + "os" + "time" +) // HAR parent container for log. type HAR struct { @@ -172,3 +176,17 @@ type Timings struct { Ssl float64 `json:"ssl,omitempty,omitzero"` // Time required for SSL/TLS negotiation. If this field is defined then the time is also included in the connect field (to ensure backward compatibility with HAR 1.1). Use -1 if the timing does not apply to the current request. Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } + +func (h *HAR) Save(filename string) error { + jsonBytes, err := json.MarshalIndent(h, "", " ") + if err != nil { + return err + } + + err = os.WriteFile("main.har", jsonBytes, 0644) + if err != nil { + return err + } + + return nil +} From b7c9efaae3e703bdd0edfd5dee4c5a87c16ce2fb Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:45:19 +0000 Subject: [PATCH 18/60] fix: update Save method in HAR struct to use provided filename for output --- harfile/har.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harfile/har.go b/harfile/har.go index 8cf4f68..d6dc21a 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -183,7 +183,7 @@ func (h *HAR) Save(filename string) error { return err } - err = os.WriteFile("main.har", jsonBytes, 0644) + err = os.WriteFile(filename, jsonBytes, 0644) if err != nil { return err } From 95135446f283939f482c301f3ee8a35689fca218 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:45:35 +0000 Subject: [PATCH 19/60] feat: implement EntryBuilder and HARHandler for managing HAR entries and saving to file --- converter/har.go | 51 -------------------------------- harhandler/entry_builder.go | 59 +++++++++++++++++++++++++++++++++++++ harhandler/har_handler.go | 29 ++++++++++++++++++ 3 files changed, 88 insertions(+), 51 deletions(-) delete mode 100644 converter/har.go create mode 100644 harhandler/entry_builder.go create mode 100644 harhandler/har_handler.go diff --git a/converter/har.go b/converter/har.go deleted file mode 100644 index 7abb04c..0000000 --- a/converter/har.go +++ /dev/null @@ -1,51 +0,0 @@ -package converter - -import ( - "context" - "net" - "time" - - "github.com/Mathious6/harkit/harfile" - http "github.com/bogdanfinn/fhttp" -) - -func BuildHAR(req *http.Request, resp *http.Response) (*harfile.HAR, error) { - harReq, err := FromHTTPRequest(req) - if err != nil { - return nil, err - } - - harResp, err := FromHTTPResponse(resp) - if err != nil { - return nil, err - } - - return &harfile.HAR{ - Log: &harfile.Log{ - Version: "1.2", - Creator: &harfile.Creator{ - Name: "harkit", - Version: "0.2.0", - }, - Entries: []*harfile.Entry{ - { - StartedDateTime: time.Now(), - Time: 0, - Request: harReq, - Response: harResp, - ServerIPAddress: getServerIPAddress(req.Host), - Cache: &harfile.Cache{}, - Timings: &harfile.Timings{}, - }, - }, - }, - }, nil -} - -func getServerIPAddress(host string) string { - ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), host) - if err != nil || len(ipAddress) == 0 { - return "" - } - return ipAddress[0].IP.String() -} diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go new file mode 100644 index 0000000..e8ca72d --- /dev/null +++ b/harhandler/entry_builder.go @@ -0,0 +1,59 @@ +package harhandler + +import ( + "context" + "net" + "time" + + "github.com/Mathious6/harkit/converter" + "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" +) + +type EntryBuilder struct { + entry *harfile.Entry +} + +func NewEntry() *EntryBuilder { + return &EntryBuilder{ + entry: &harfile.Entry{ + StartedDateTime: time.Now(), + Cache: &harfile.Cache{}, + Timings: &harfile.Timings{}, + }, + } +} + +func (e *EntryBuilder) AddRequest(req *http.Request) error { + harReq, err := converter.FromHTTPRequest(req) + if err != nil { + return err + } + e.entry.Request = harReq + return nil +} + +func (e *EntryBuilder) AddResponse(resp *http.Response) error { + harResp, err := converter.FromHTTPResponse(resp) + if err != nil { + return err + } + e.entry.Response = harResp + return nil +} + +func (e *EntryBuilder) Build() *harfile.Entry { + e.entry.ServerIPAddress = getServerIPAddress(e.entry.Request.URL) + + return e.entry +} + +func getServerIPAddress(url string) string { + host, _, err := net.SplitHostPort(url) + + ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), host) + if err != nil || len(ipAddress) == 0 { + return "" + } + return ipAddress[0].IP.String() +} diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go new file mode 100644 index 0000000..7cde857 --- /dev/null +++ b/harhandler/har_handler.go @@ -0,0 +1,29 @@ +package harhandler + +import "github.com/Mathious6/harkit/harfile" + +type HARHandler struct { + log *harfile.Log +} + +func NewHandler() *HARHandler { + return &HARHandler{ + log: &harfile.Log{ + Version: "1.2", + Creator: &harfile.Creator{ + Name: "harkit", + Version: "0.2.0", + }, + Entries: []*harfile.Entry{}, + }, + } +} + +func (h *HARHandler) AddEntry(entry *harfile.Entry) { + h.log.Entries = append(h.log.Entries, entry) +} + +func (h *HARHandler) Save(filename string) error { + har := &harfile.HAR{Log: h.log} + return har.Save(filename) +} From b89a8d8527beb86527351588fe93ad330bf012e2 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:53:49 +0000 Subject: [PATCH 20/60] fix: update locateRedirectURL function to return a pointer to the redirect URL --- converter/response.go | 7 ++++--- harfile/har.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/converter/response.go b/converter/response.go index b0279d2..9478c27 100644 --- a/converter/response.go +++ b/converter/response.go @@ -32,13 +32,14 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { }, nil } -func locateRedirectURL(resp *http.Response) string { +func locateRedirectURL(resp *http.Response) *string { if resp.StatusCode >= 300 && resp.StatusCode < 400 { if loc, err := resp.Location(); err == nil { - return loc.String() + url := loc.String() + return &url } } - return "" + return nil } func buildContent(resp *http.Response) (*harfile.Content, error) { diff --git a/harfile/har.go b/harfile/har.go index d6dc21a..e6a7010 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -94,7 +94,7 @@ type Response struct { Cookies []*Cookie `json:"cookies"` // List of cookie objects. Headers []*NameValuePair `json:"headers"` // List of header objects. Content *Content `json:"content"` // Details about the response body. - RedirectURL string `json:"redirectURL"` // Redirection target URL from the Location response header. + RedirectURL *string `json:"redirectURL"` // Redirection target URL from the Location response header. HeadersSize int64 `json:"headersSize"` // Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body. Set to -1 if the info is not available. BodySize int64 `json:"bodySize"` // Size of the received response body in bytes. Set to zero in case of responses coming from the cache (304). Set to -1 if the info is not available. Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. From 32b27e1b4fdf0ab9c4501acb432137385c4f8f71 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 19:57:13 +0000 Subject: [PATCH 21/60] fix: correct header names in request constants and dereference redirect URL in response test --- converter/request_test.go | 4 ++-- converter/response_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/converter/request_test.go b/converter/request_test.go index c3662ba..ab80dde 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -19,9 +19,9 @@ const ( REQ_URI = "/api?foo=bar" REQ_PROTOCOL = "HTTP/1.1" - REQ_HEADER1_NAME = "NaMe1" + REQ_HEADER1_NAME = "Name1" REQ_HEADER1_VALUE = "value1" - REQ_HEADER2_NAME = "nAmE2" + REQ_HEADER2_NAME = "Name2" REQ_HEADER2_VALUE = "value2" REQ_COOKIE_NAME = "name" diff --git a/converter/response_test.go b/converter/response_test.go index a3c94e8..0c58668 100644 --- a/converter/response_test.go +++ b/converter/response_test.go @@ -116,7 +116,7 @@ func TestConverter_GivenRedirect_WhenConvertingHTTPResponse_ThenRedirectURLShoul result, err := converter.FromHTTPResponse(resp) require.NoError(t, err) - assert.Equal(t, RESP_LOCATION, result.RedirectURL, "HAR redirect URL <> response location header") + assert.Equal(t, RESP_LOCATION, *result.RedirectURL, "HAR redirect URL <> response location header") } func TestConverter_GivenBody_WhenConvertingHTTPResponse_ThenBodySizeShouldBeCorrect(t *testing.T) { From 0b1791146eabbfcd0bea78481e5f172ce12d878d Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 20:11:33 +0000 Subject: [PATCH 22/60] fix: refactor getServerIPAddress function to use url.Parse for improved URL handling --- harhandler/entry_builder.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index e8ca72d..cc2af61 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -3,6 +3,7 @@ package harhandler import ( "context" "net" + "net/url" "time" "github.com/Mathious6/harkit/converter" @@ -48,10 +49,14 @@ func (e *EntryBuilder) Build() *harfile.Entry { return e.entry } -func getServerIPAddress(url string) string { - host, _, err := net.SplitHostPort(url) +func getServerIPAddress(reqUrl string) string { + url, err := url.Parse(reqUrl) + if err != nil { + return "" + } - ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), host) + // WARN: This is a blocking call and may take time to resolve + ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), url.Hostname()) if err != nil || len(ipAddress) == 0 { return "" } From 5dbad432c00424374dd7e5e7e2ed9aab55a38c10 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 20:33:43 +0000 Subject: [PATCH 23/60] feat: add Total method to Timings struct for calculating total request/response time --- harfile/har.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/harfile/har.go b/harfile/har.go index e6a7010..b920271 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -177,6 +177,20 @@ type Timings struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } +// Total returns the total time of the request/response round trip. +func (t *Timings) Total() float64 { + sum := 0.0 + sum += t.Blocked + sum += t.DNS + sum += t.Connect + sum += t.Send + sum += t.Wait + sum += t.Receive + sum += t.Ssl + return sum +} + +// Save saves the HAR data to a file in JSON format under the specified filename. func (h *HAR) Save(filename string) error { jsonBytes, err := json.MarshalIndent(h, "", " ") if err != nil { From 2b54fb2071839bdd6f2ca071dc2ff6edf675e5f3 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 20:44:56 +0000 Subject: [PATCH 24/60] fix: initialize Timings in NewEntry and update response handling to calculate total time --- harhandler/entry_builder.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index cc2af61..09896c2 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -19,8 +19,13 @@ func NewEntry() *EntryBuilder { return &EntryBuilder{ entry: &harfile.Entry{ StartedDateTime: time.Now(), + Time: -1, Cache: &harfile.Cache{}, - Timings: &harfile.Timings{}, + Timings: &harfile.Timings{ + Send: -1, + Wait: -1, + Receive: -1, + }, }, } } @@ -40,6 +45,10 @@ func (e *EntryBuilder) AddResponse(resp *http.Response) error { return err } e.entry.Response = harResp + + e.entry.Timings.Receive = float64(time.Since(e.entry.StartedDateTime).Milliseconds()) + e.entry.Time = e.entry.Timings.Total() + return nil } From 81a3fb6702fa25db6bf4d8645d5ca6923b7b61f0 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 21:19:35 +0000 Subject: [PATCH 25/60] fix: update HARHandler to use dynamic version from harkit package --- harhandler/har_handler.go | 7 +++++-- version.go | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 version.go diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index 7cde857..062d0d8 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -1,6 +1,9 @@ package harhandler -import "github.com/Mathious6/harkit/harfile" +import ( + "github.com/Mathious6/harkit" + "github.com/Mathious6/harkit/harfile" +) type HARHandler struct { log *harfile.Log @@ -12,7 +15,7 @@ func NewHandler() *HARHandler { Version: "1.2", Creator: &harfile.Creator{ Name: "harkit", - Version: "0.2.0", + Version: harkit.Version, }, Entries: []*harfile.Entry{}, }, diff --git a/version.go b/version.go new file mode 100644 index 0000000..d4548c2 --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package harkit + +var Version = "v0.2.0" From 2abccfb3b0e93906838019939a4292e06627b560 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 21:42:22 +0000 Subject: [PATCH 26/60] fix: update Total method in Timings struct to ignore negative values for accurate time calculation --- harfile/har.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/harfile/har.go b/harfile/har.go index b920271..4ca2ccf 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -178,15 +178,14 @@ type Timings struct { } // Total returns the total time of the request/response round trip. +// Ignoring -1 values which indicate that the timing does not apply to the current request. func (t *Timings) Total() float64 { sum := 0.0 - sum += t.Blocked - sum += t.DNS - sum += t.Connect - sum += t.Send - sum += t.Wait - sum += t.Receive - sum += t.Ssl + for _, v := range []float64{t.Blocked, t.DNS, t.Connect, t.Send, t.Wait, t.Receive, t.Ssl} { + if v > 0 { + sum += v + } + } return sum } From e8b75812d88db005a8073536e05f811e5216b501 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 21:42:38 +0000 Subject: [PATCH 27/60] fix: update EntryBuilder and HARHandler to conditionally resolve server IP address --- harhandler/entry_builder.go | 7 ++++--- harhandler/har_handler.go | 23 ++++++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 09896c2..46e855a 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -52,9 +52,10 @@ func (e *EntryBuilder) AddResponse(resp *http.Response) error { return nil } -func (e *EntryBuilder) Build() *harfile.Entry { - e.entry.ServerIPAddress = getServerIPAddress(e.entry.Request.URL) - +func (e *EntryBuilder) build(resolveIP bool) *harfile.Entry { + if resolveIP && e.entry.Request != nil { + e.entry.ServerIPAddress = getServerIPAddress(e.entry.Request.URL) + } return e.entry } diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index 062d0d8..be883b0 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -5,12 +5,22 @@ import ( "github.com/Mathious6/harkit/harfile" ) +type HandlerOption func(*HARHandler) + type HARHandler struct { log *harfile.Log + + resolveIPAddress bool +} + +func WithServerIPAddress() HandlerOption { + return func(h *HARHandler) { + h.resolveIPAddress = true + } } -func NewHandler() *HARHandler { - return &HARHandler{ +func NewHandler(opts ...HandlerOption) *HARHandler { + h := &HARHandler{ log: &harfile.Log{ Version: "1.2", Creator: &harfile.Creator{ @@ -20,9 +30,16 @@ func NewHandler() *HARHandler { Entries: []*harfile.Entry{}, }, } + + for _, opt := range opts { + opt(h) + } + + return h } -func (h *HARHandler) AddEntry(entry *harfile.Entry) { +func (h *HARHandler) AddEntry(builder *EntryBuilder) { + entry := builder.build(h.resolveIPAddress) h.log.Entries = append(h.log.Entries, entry) } From e1729698ddc87d217faffeecb41b9674226d7c1e Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 22:14:38 +0000 Subject: [PATCH 28/60] refactor: streamline main function by removing unused imports and simplifying request handling --- example/main.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/example/main.go b/example/main.go index 23f3101..0892004 100644 --- a/example/main.go +++ b/example/main.go @@ -1,42 +1,43 @@ package main import ( - "encoding/json" - "fmt" - "os" - - "github.com/Mathious6/harkit/converter" + "github.com/Mathious6/harkit/harhandler" http "github.com/bogdanfinn/fhttp" tls_client "github.com/bogdanfinn/tls-client" ) func main() { - jar := tls_client.NewCookieJar() - options := []tls_client.HttpClientOption{ - tls_client.WithCookieJar(jar), - tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO: REMOVE BEFORE PUSHING - } + handler := harhandler.NewHandler() + + client, _ := tls_client.NewHttpClient( + tls_client.NewNoopLogger(), + tls_client.WithCookieJar(tls_client.NewCookieJar()), + // tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this + ) - client, _ := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...) + entry := harhandler.NewEntry() - req, _ := http.NewRequest(http.MethodGet, "https://tls.peet.ws/api/all", nil) + req, _ := http.NewRequest(http.MethodPost, "https://httpbin.org/post?source=harkit", nil) + req.Header.Add("host", "httpbin.org") + req.Header.Add("accept-encoding", "gzip, deflate, br") req.Header.Add("accept", "*/*") req.Header.Add("user-agent", "harkit-example") + req.Header.Add(http.HeaderOrderKey, "host") + req.Header.Add(http.HeaderOrderKey, "accept-encoding") + req.Header.Add(http.HeaderOrderKey, "accept") req.Header.Add(http.HeaderOrderKey, "user-agent") + req.Header.Add(http.HeaderOrderKey, "cookie") req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) + _ = entry.AddRequest(req) resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() + _ = entry.AddResponse(resp) - har, err := converter.BuildHAR(req, resp) - if err != nil { - panic(err) - } + handler.AddEntry(entry) - jsonBytes, _ := json.MarshalIndent(har, "", " ") - _ = os.WriteFile("main.har", jsonBytes, 0644) - fmt.Println("✅ HAR file saved as main.har") + handler.Save("example.har") } From 1a33e4a1610312185dcd9d3b9a6de70816572015 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 22:38:45 +0000 Subject: [PATCH 29/60] fix: update PostData struct to use omitempty for params and text fields --- harfile/har.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/harfile/har.go b/harfile/har.go index 4ca2ccf..2405e9f 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -121,10 +121,11 @@ type NameValuePair struct { } // PostData describes posted data, if any (embedded in [Request] object). +// Text and params fields are mutually exclusive. type PostData struct { MimeType string `json:"mimeType"` // Mime type of posted data. - Params []*Param `json:"params"` // List of posted parameters (in case of URL encoded parameters). - Text string `json:"text"` // Plain text posted data + Params []*Param `json:"params,omitempty"` // List of posted parameters (in case of URL encoded parameters). + Text string `json:"text,omitempty"` // Plain text posted data Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } From 50eb5c2d45b08f89d4a17253f6dd27a9bf656a99 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Sat, 12 Apr 2025 22:38:55 +0000 Subject: [PATCH 30/60] refactor: update request handling to use constants and streamline code structure --- example/main.go | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/example/main.go b/example/main.go index 0892004..995cff5 100644 --- a/example/main.go +++ b/example/main.go @@ -1,11 +1,18 @@ package main import ( + "fmt" + "io" + "net/url" + "strings" + "github.com/Mathious6/harkit/harhandler" http "github.com/bogdanfinn/fhttp" tls_client "github.com/bogdanfinn/tls-client" ) +const URL = "https://httpbin.org/post?source=harkit" + func main() { handler := harhandler.NewHandler() @@ -15,29 +22,37 @@ func main() { // tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this ) - entry := harhandler.NewEntry() + // 1. Form URL-encoded + form := url.Values{} + form.Set("name", "Pierre") + form.Set("role", "developer") + sendRequest(client, handler, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) + fmt.Println("✅ Form URL-encoded request sent") - req, _ := http.NewRequest(http.MethodPost, "https://httpbin.org/post?source=harkit", nil) - req.Header.Add("host", "httpbin.org") - req.Header.Add("accept-encoding", "gzip, deflate, br") - req.Header.Add("accept", "*/*") - req.Header.Add("user-agent", "harkit-example") - req.Header.Add(http.HeaderOrderKey, "host") - req.Header.Add(http.HeaderOrderKey, "accept-encoding") - req.Header.Add(http.HeaderOrderKey, "accept") - req.Header.Add(http.HeaderOrderKey, "user-agent") - req.Header.Add(http.HeaderOrderKey, "cookie") + // 2. JSON + jsonBody := `{"name":"Pierre","role":"developer"}` + sendRequest(client, handler, "application/json", strings.NewReader(jsonBody)) + fmt.Println("✅ JSON request sent") + + handler.Save("example.har") +} + +func sendRequest(c tls_client.HttpClient, h *harhandler.HARHandler, contentType string, body io.Reader) { + req, _ := http.NewRequest(http.MethodPost, URL, body) + req.Header.Set("Content-Type", contentType) + req.Header.Set("Accept", "*/*") + req.Header.Set("User-Agent", "harkit-example") req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) + + entry := harhandler.NewEntry() _ = entry.AddRequest(req) - resp, err := client.Do(req) + resp, err := c.Do(req) if err != nil { panic(err) } defer resp.Body.Close() _ = entry.AddResponse(resp) - handler.AddEntry(entry) - - handler.Save("example.har") + h.AddEntry(entry) } From 211f10915cb1297373a288ff220904d9a8175a3d Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 01:26:37 +0000 Subject: [PATCH 31/60] fix: update JSON formatting in Save method to use consistent indentation and append newline --- harfile/har.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/harfile/har.go b/harfile/har.go index 2405e9f..1536b84 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -192,11 +192,13 @@ func (t *Timings) Total() float64 { // Save saves the HAR data to a file in JSON format under the specified filename. func (h *HAR) Save(filename string) error { - jsonBytes, err := json.MarshalIndent(h, "", " ") + jsonBytes, err := json.MarshalIndent(h, "", " ") if err != nil { return err } + jsonBytes = append(jsonBytes, '\n') + err = os.WriteFile(filename, jsonBytes, 0644) if err != nil { return err From 628b378c76190fc886ac156d0449fbb27f9dc611 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 02:52:48 +0000 Subject: [PATCH 32/60] fix: update headers handling to include Content-Length in request and response conversions --- converter/common.go | 38 +++++++++++++++++++++++++------------- converter/request.go | 2 +- converter/request_test.go | 25 +++++++++++++++++++++++-- converter/response.go | 2 +- converter/response_test.go | 36 ++++++++++++++++++++++++++++++------ 5 files changed, 80 insertions(+), 23 deletions(-) diff --git a/converter/common.go b/converter/common.go index 989a861..3c11945 100644 --- a/converter/common.go +++ b/converter/common.go @@ -1,6 +1,7 @@ package converter import ( + "fmt" "strings" "time" @@ -9,10 +10,11 @@ import ( ) const ( - ContentTypeKey = "Content-Type" - CookieKey = "Cookie" - SetCookieKey = "Set-Cookie" - LocationKey = "Location" + ContentLengthKey = "Content-Length" + ContentTypeKey = "Content-Type" + CookieKey = "Cookie" + SetCookieKey = "Set-Cookie" + LocationKey = "Location" ) func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { @@ -38,22 +40,32 @@ func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { return harCookies } -func convertHeaders(header http.Header) []*harfile.NameValuePair { - harHeaders := make([]*harfile.NameValuePair, 0, len(header)) +func convertHeaders(header http.Header, contentLength int64) []*harfile.NameValuePair { + // By default, client adds Content-Length header later on, so we need to add it here + // We clone the header to avoid modifying the original one to avoid side effects + clonedHeader := header.Clone() + if contentLength > 0 && clonedHeader.Get(ContentLengthKey) == "" { + clonedHeader.Set(ContentLengthKey, fmt.Sprintf("%d", contentLength)) + } - // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) + harHeaders := make([]*harfile.NameValuePair, 0, len(clonedHeader)) seen := make(map[string]bool) - for _, name := range header.Values(http.HeaderOrderKey) { - if values := header.Values(name); len(values) > 0 { - name = http.CanonicalHeaderKey(name) + + // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) + order := clonedHeader.Values(http.HeaderOrderKey) + for _, name := range order { + canonical := http.CanonicalHeaderKey(name) + values := clonedHeader.Values(name) + + if len(values) > 0 { for _, value := range values { - harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + harHeaders = append(harHeaders, &harfile.NameValuePair{Name: canonical, Value: value}) } - seen[name] = true + seen[canonical] = true } } - for name, values := range header { + for name, values := range clonedHeader { if seen[name] || strings.EqualFold(name, http.HeaderOrderKey) { continue } diff --git a/converter/request.go b/converter/request.go index 30df469..31d559e 100644 --- a/converter/request.go +++ b/converter/request.go @@ -16,7 +16,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { return nil, errors.New("request cannot be nil") } - headers := convertHeaders(req.Header) + headers := convertHeaders(req.Header, req.ContentLength) postData, err := extractPostData(req) if err != nil { diff --git a/converter/request_test.go b/converter/request_test.go index ab80dde..7c41824 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -30,8 +30,9 @@ const ( REQ_URL_CONTENT_TYPE = "application/x-www-form-urlencoded" REQ_BODY_URL = "foo=bar" - REQ_JSON_CONTENT_TYPE = "application/json" - REQ_BODY_JSON = `{"foo":"bar"}` + REQ_JSON_CONTENT_TYPE = "application/json" + REQ_BODY_JSON = `{"foo":"bar"}` + REQ_JSON_CONTENT_LENGTH_VALUE = "13" REQ_PART1_NAME = "name1" REQ_PART1_VALUE = "value1" @@ -112,6 +113,15 @@ func TestConverter_GivenURLWithQueryString_WhenConvertingHTTPRequest_ThenQuerySt assert.Equal(t, "bar", result.QueryString[0].Value, "HAR query string value <> request query string value") } +func TestConverter_GivenEmptyBody_WhenConvertingHTTPRequest_ThenContentLengthShouldNotBeSet(t *testing.T) { + req := createRequest(t, nil, "") + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.NotEqual(t, converter.ContentLengthKey, result.Headers[2].Name, "HAR content length header value should not be set") +} + func TestConverter_GivenURLEncodedBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { req := createRequest(t, strings.NewReader(REQ_BODY_URL), REQ_URL_CONTENT_TYPE) @@ -137,6 +147,16 @@ func TestConverter_GivenJSONBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeC assert.Equal(t, REQ_JSON_CONTENT_TYPE, result.PostData.MimeType, "HAR post data mime type <> request post data mime type") } +func TestConverter_GivenJSONBody_WhenConvertingHTTPRequest_ThenContentLengthShouldBeSet(t *testing.T) { + req := createRequest(t, strings.NewReader(REQ_BODY_JSON), REQ_JSON_CONTENT_TYPE) + + result, err := converter.FromHTTPRequest(req) + require.NoError(t, err) + + assert.Equal(t, converter.ContentLengthKey, result.Headers[2].Name, "HAR content length header name <> request content length header name") + assert.Equal(t, REQ_JSON_CONTENT_LENGTH_VALUE, result.Headers[2].Value, "HAR content length header value <> request content length header value") +} + func TestConverter_GivenMultipartBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { body, contentType := createMultipartBody() req := createRequest(t, &body, contentType) @@ -189,6 +209,7 @@ func createRequest(t *testing.T, body io.Reader, contentType string) *http.Reque req.Header.Add(http.HeaderOrderKey, REQ_HEADER2_NAME) req.Header.Add(http.HeaderOrderKey, REQ_HEADER1_NAME) + req.Header.Add(http.HeaderOrderKey, converter.ContentLengthKey) if contentType != "" { req.Header.Add(converter.ContentTypeKey, contentType) diff --git a/converter/response.go b/converter/response.go index 9478c27..79d30a7 100644 --- a/converter/response.go +++ b/converter/response.go @@ -24,7 +24,7 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { StatusText: http.StatusText(resp.StatusCode), HTTPVersion: resp.Proto, Cookies: convertCookies(resp.Cookies()), - Headers: convertHeaders(resp.Header), + Headers: convertHeaders(resp.Header, resp.ContentLength), Content: content, RedirectURL: locateRedirectURL(resp), HeadersSize: -1, diff --git a/converter/response_test.go b/converter/response_test.go index 0c58668..236fd11 100644 --- a/converter/response_test.go +++ b/converter/response_test.go @@ -28,8 +28,9 @@ const ( RESP_COOKIE_HTTPONLY = true RESP_COOKIE_SECURE = true - RESP_BODY_TEXT = "response" - RESP_CONTENT_TYPE = "text/plain" + RESP_BODY_TEXT = "response" + RESP_CONTENT_TYPE = "text/plain" + RESP_JSON_CONTENT_LENGTH_VALUE = "8" RESP_LOCATION = "https://example.com/redirect" ) @@ -128,6 +129,25 @@ func TestConverter_GivenBody_WhenConvertingHTTPResponse_ThenBodySizeShouldBeCorr assert.Equal(t, int64(len(RESP_BODY_TEXT)), result.BodySize, "HAR body size <> response body size") } +func TestConverter_GivenEmptyBody_WhenConvertingHTTPResponse_ThenContentLengthShouldNotBeSet(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, bytes.NewBufferString(""), RESP_CONTENT_TYPE) + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Len(t, result.Headers, 3, "HAR should contain 3 header") +} + +func TestConverter_GivenBody_WhenConvertingHTTPResponse_ThenContentLengthShouldBeSet(t *testing.T) { + resp := createResponse(t, RESP_OK_CODE, bytes.NewBufferString(RESP_BODY_TEXT), RESP_CONTENT_TYPE) + + result, err := converter.FromHTTPResponse(resp) + require.NoError(t, err) + + assert.Len(t, result.Headers, 4, "HAR should contain 4 header") + assert.Equal(t, RESP_JSON_CONTENT_LENGTH_VALUE, result.Headers[3].Value, "HAR content length <> response content length") +} + func createResponse(t *testing.T, statusCode int, body io.Reader, contentType string) *http.Response { var buf []byte var err error @@ -152,15 +172,19 @@ func createResponse(t *testing.T, statusCode int, body io.Reader, contentType st cookie += ";httponly" cookie += ";secure" - resp.Header.Add(RESP_HEADER_NAME, RESP_HEADER_VALUE) - resp.Header.Add(converter.SetCookieKey, cookie) + resp.Header.Append(RESP_HEADER_NAME, RESP_HEADER_VALUE) + resp.Header.Append(converter.SetCookieKey, cookie) if contentType != "" { - resp.Header.Set(converter.ContentTypeKey, contentType) + resp.Header.Append(converter.ContentTypeKey, contentType) + } + + if len(buf) > 0 { + resp.Header.Append(converter.ContentLengthKey, RESP_JSON_CONTENT_LENGTH_VALUE) } if statusCode >= 300 && statusCode < 400 { - resp.Header.Set(converter.LocationKey, RESP_LOCATION) + resp.Header.Append(converter.LocationKey, RESP_LOCATION) } return resp From c7c9d12615e69bfa7c4c8742cad620e26412a0c4 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 03:00:12 +0000 Subject: [PATCH 33/60] fix: update response handling to use content size instead of Content-Length for headers and body size --- converter/response.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/converter/response.go b/converter/response.go index 79d30a7..7250941 100644 --- a/converter/response.go +++ b/converter/response.go @@ -24,11 +24,11 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { StatusText: http.StatusText(resp.StatusCode), HTTPVersion: resp.Proto, Cookies: convertCookies(resp.Cookies()), - Headers: convertHeaders(resp.Header, resp.ContentLength), + Headers: convertHeaders(resp.Header, content.Size), Content: content, RedirectURL: locateRedirectURL(resp), HeadersSize: -1, - BodySize: resp.ContentLength, + BodySize: content.Size, }, nil } From 205319746e320280c538baf6fc76aa24624dda42 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 03:26:42 +0000 Subject: [PATCH 34/60] fix: update header size calculation to use single CRLF instead of double CRLF --- converter/request.go | 2 +- converter/request_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/converter/request.go b/converter/request.go index 31d559e..abc79b5 100644 --- a/converter/request.go +++ b/converter/request.go @@ -130,6 +130,6 @@ func computeRequestHeadersSize(req *http.Request, harHeaders []*harfile.NameValu headersSize += len(headerLine) } - headersSize += len("\r\n\r\n") + headersSize += len("\r\n") return int64(headersSize) } diff --git a/converter/request_test.go b/converter/request_test.go index 7c41824..5ca4f62 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -226,7 +226,7 @@ func computeHeadersSize() int64 { headersSize += len(REQ_HEADER2_NAME + ": " + REQ_HEADER2_VALUE + "\r\n") headersSize += len(REQ_HEADER1_NAME + ": " + REQ_HEADER1_VALUE + "\r\n") headersSize += len(converter.CookieKey + ": " + REQ_COOKIE_NAME + "=" + REQ_COOKIE_VALUE + "\r\n") - headersSize += len("\r\n\r\n") + headersSize += len("\r\n") return int64(headersSize) } From 3790505033405436880c9e12f720d0f0f2710c25 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 04:49:47 +0000 Subject: [PATCH 35/60] fix: enhance AddRequest method to merge cookies and update cookie header handling --- harhandler/entry_builder.go | 64 ++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 46e855a..eb405ad 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/url" + "strings" "time" "github.com/Mathious6/harkit/converter" @@ -30,11 +31,16 @@ func NewEntry() *EntryBuilder { } } -func (e *EntryBuilder) AddRequest(req *http.Request) error { +func (e *EntryBuilder) AddRequest(req *http.Request, cookies []*http.Cookie) error { harReq, err := converter.FromHTTPRequest(req) if err != nil { return err } + + // If our client has a cookie jar, we need to merge the cookies + harReq.Headers = mergeCookieHeader(harReq.Headers, cookies) + harReq.Cookies = mergeCookies(harReq.Cookies, cookies) + e.entry.Request = harReq return nil } @@ -72,3 +78,59 @@ func getServerIPAddress(reqUrl string) string { } return ipAddress[0].IP.String() } + +func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Cookie { + if len(new) == 0 { + return existing + } + + merged := make([]*harfile.Cookie, 0, len(existing)+len(new)) + merged = append(merged, existing...) + + for _, cookie := range new { + merged = append(merged, &harfile.Cookie{ + Name: cookie.Name, + Value: cookie.Value, + Path: cookie.Path, + Domain: cookie.Domain, + Expires: cookie.RawExpires, + HTTPOnly: cookie.HttpOnly, + Secure: cookie.Secure, + }) + } + + return merged + +} + +func mergeCookieHeader(existing []*harfile.NameValuePair, new []*http.Cookie) []*harfile.NameValuePair { + if len(new) == 0 { + return existing + } + + var cookieHeader strings.Builder + for i, c := range new { + if i > 0 { + cookieHeader.WriteString("; ") + } + cookieHeader.WriteString(c.Name + "=" + c.Value) + } + + found := false + for _, header := range existing { + if strings.EqualFold(header.Name, "Cookie") { + header.Value = header.Value + "; " + cookieHeader.String() + found = true + break + } + } + + if !found { + existing = append(existing, &harfile.NameValuePair{ + Name: "Cookie", + Value: cookieHeader.String(), + }) + } + + return existing +} From 3659ae7d6c1dfaa9995f0c9519102e812da95d58 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 04:50:01 +0000 Subject: [PATCH 36/60] fix: refactor request handling to streamline GET and POST methods and update URL usage --- example/main.go | 73 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/example/main.go b/example/main.go index 995cff5..ef88c8f 100644 --- a/example/main.go +++ b/example/main.go @@ -11,7 +11,7 @@ import ( tls_client "github.com/bogdanfinn/tls-client" ) -const URL = "https://httpbin.org/post?source=harkit" +const URL = "https://httpbin.org" func main() { handler := harhandler.NewHandler() @@ -19,33 +19,84 @@ func main() { client, _ := tls_client.NewHttpClient( tls_client.NewNoopLogger(), tls_client.WithCookieJar(tls_client.NewCookieJar()), - // tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this + tls_client.WithNotFollowRedirects(), + tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this ) - // 1. Form URL-encoded + // 1. Get with query parameters + sendGetRequest(client, handler, URL+"/get?name=pierre&role=developer") + fmt.Println("✅ Parameters sent") + + // 2. Set cookies + sendGetRequest(client, handler, URL+"/cookies/set?name=pierre&role=developer") + fmt.Println("✅ Cookies set") + + // 3. Form URL-encoded form := url.Values{} form.Set("name", "Pierre") form.Set("role", "developer") - sendRequest(client, handler, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) + sendPostRequest(client, handler, URL+"/post", "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) fmt.Println("✅ Form URL-encoded request sent") - // 2. JSON + // 4. JSON jsonBody := `{"name":"Pierre","role":"developer"}` - sendRequest(client, handler, "application/json", strings.NewReader(jsonBody)) + sendPostRequest(client, handler, URL+"/post", "application/json", strings.NewReader(jsonBody)) fmt.Println("✅ JSON request sent") handler.Save("example.har") } -func sendRequest(c tls_client.HttpClient, h *harhandler.HARHandler, contentType string, body io.Reader) { +func sendGetRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL string) { + req, _ := http.NewRequest(http.MethodGet, URL, nil) + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "httpbin.org") + req.Header.Add("User-Agent", "harkit-example") + req.Header.Add("Accept-Encoding", "gzip, deflate, br") + + req.Header.Add(http.HeaderOrderKey, "accept") + req.Header.Add(http.HeaderOrderKey, "host") + req.Header.Add(http.HeaderOrderKey, "user-agent") + req.Header.Add(http.HeaderOrderKey, "accept-encoding") + + entry := harhandler.NewEntry() + + urlParsed, _ := url.Parse(URL) + cookies := c.GetCookies(urlParsed) + _ = entry.AddRequest(req, cookies) + + resp, err := c.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + _ = entry.AddResponse(resp) + + h.AddEntry(entry) +} + +func sendPostRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL string, contentType string, body io.Reader) { req, _ := http.NewRequest(http.MethodPost, URL, body) - req.Header.Set("Content-Type", contentType) - req.Header.Set("Accept", "*/*") - req.Header.Set("User-Agent", "harkit-example") + req.Header.Add("Accept", "*/*") + req.Header.Add("Content-Type", contentType) + req.Header.Add("Host", "httpbin.org") + req.Header.Add("User-Agent", "harkit-example") + req.Header.Add("Accept-Encoding", "gzip, deflate, br") + req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) + req.Header.Add(http.HeaderOrderKey, "accept") + req.Header.Add(http.HeaderOrderKey, "content-length") + req.Header.Add(http.HeaderOrderKey, "content-type") + req.Header.Add(http.HeaderOrderKey, "cookie") + req.Header.Add(http.HeaderOrderKey, "host") + req.Header.Add(http.HeaderOrderKey, "user-agent") + req.Header.Add(http.HeaderOrderKey, "accept-encoding") + entry := harhandler.NewEntry() - _ = entry.AddRequest(req) + + urlParsed, _ := url.Parse(URL) + cookies := c.GetCookies(urlParsed) + _ = entry.AddRequest(req, cookies) resp, err := c.Do(req) if err != nil { From 80c0fad1964aa69b6eff22f99dec067961733593 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 18:24:05 +0000 Subject: [PATCH 37/60] fix: update HAR entry handling to use NVPair type and improve cookie management --- converter/common.go | 8 ++-- converter/request.go | 8 ++-- example/main.go | 8 +--- harfile/har.go | 80 +++++++++++++++++----------------- harhandler/entry_builder.go | 86 +++++++++++++++++++------------------ harhandler/har_handler.go | 2 +- 6 files changed, 94 insertions(+), 98 deletions(-) diff --git a/converter/common.go b/converter/common.go index 3c11945..8b2509a 100644 --- a/converter/common.go +++ b/converter/common.go @@ -40,7 +40,7 @@ func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { return harCookies } -func convertHeaders(header http.Header, contentLength int64) []*harfile.NameValuePair { +func convertHeaders(header http.Header, contentLength int64) []*harfile.NVPair { // By default, client adds Content-Length header later on, so we need to add it here // We clone the header to avoid modifying the original one to avoid side effects clonedHeader := header.Clone() @@ -48,7 +48,7 @@ func convertHeaders(header http.Header, contentLength int64) []*harfile.NameValu clonedHeader.Set(ContentLengthKey, fmt.Sprintf("%d", contentLength)) } - harHeaders := make([]*harfile.NameValuePair, 0, len(clonedHeader)) + harHeaders := make([]*harfile.NVPair, 0, len(clonedHeader)) seen := make(map[string]bool) // Used to sort headers in HAR file if needed (e.g. https://github.com/bogdanfinn/tls-client) @@ -59,7 +59,7 @@ func convertHeaders(header http.Header, contentLength int64) []*harfile.NameValu if len(values) > 0 { for _, value := range values { - harHeaders = append(harHeaders, &harfile.NameValuePair{Name: canonical, Value: value}) + harHeaders = append(harHeaders, &harfile.NVPair{Name: canonical, Value: value}) } seen[canonical] = true } @@ -70,7 +70,7 @@ func convertHeaders(header http.Header, contentLength int64) []*harfile.NameValu continue } for _, value := range values { - harHeaders = append(harHeaders, &harfile.NameValuePair{Name: name, Value: value}) + harHeaders = append(harHeaders, &harfile.NVPair{Name: name, Value: value}) } } diff --git a/converter/request.go b/converter/request.go index abc79b5..2de8f76 100644 --- a/converter/request.go +++ b/converter/request.go @@ -36,12 +36,12 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { }, nil } -func convertQueryParams(u *url.URL) []*harfile.NameValuePair { - result := make([]*harfile.NameValuePair, 0) +func convertQueryParams(u *url.URL) []*harfile.NVPair { + result := make([]*harfile.NVPair, 0) for key, values := range u.Query() { for _, value := range values { - result = append(result, &harfile.NameValuePair{Name: key, Value: value}) + result = append(result, &harfile.NVPair{Name: key, Value: value}) } } @@ -119,7 +119,7 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { return postData, nil } -func computeRequestHeadersSize(req *http.Request, harHeaders []*harfile.NameValuePair) int64 { +func computeRequestHeadersSize(req *http.Request, harHeaders []*harfile.NVPair) int64 { headersSize := 0 requestLine := req.Method + " " + req.URL.RequestURI() + " " + req.Proto + "\r\n" diff --git a/example/main.go b/example/main.go index ef88c8f..6f035ad 100644 --- a/example/main.go +++ b/example/main.go @@ -58,11 +58,9 @@ func sendGetRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL strin req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() - urlParsed, _ := url.Parse(URL) cookies := c.GetCookies(urlParsed) - _ = entry.AddRequest(req, cookies) + entry, _ := harhandler.NewEntryWithRequest(req, cookies) resp, err := c.Do(req) if err != nil { @@ -92,11 +90,9 @@ func sendPostRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL stri req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() - urlParsed, _ := url.Parse(URL) cookies := c.GetCookies(urlParsed) - _ = entry.AddRequest(req, cookies) + entry, _ := harhandler.NewEntryWithRequest(req, cookies) resp, err := c.Do(req) if err != nil { diff --git a/harfile/har.go b/harfile/har.go index 1536b84..95135bb 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -1,4 +1,5 @@ // Package harfile provides types for working with HAR (HTTP Archive) 1.2 files. +// // See: http://www.softwareishard.com/blog/har-12-spec/ package harfile @@ -46,19 +47,18 @@ type Page struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// PageTimings describes timings for various events (states) fired during the -// page load. All times are specified in milliseconds. If a time info is not -// available appropriate field is set to -1. +// PageTimings describes timings for various events (states) fired during the page load. All times +// are specified in milliseconds. If a time info is not available appropriate field is set to -1. type PageTimings struct { OnContentLoad float64 `json:"onContentLoad,omitempty,omitzero"` // Content of the page loaded. Number of milliseconds since page load started (page.startedDateTime). Use -1 if the timing does not apply to the current request. OnLoad float64 `json:"onLoad,omitempty,omitzero"` // Page is loaded (onLoad event fired). Number of milliseconds since page load started (page.startedDateTime). Use -1 if the timing does not apply to the current request. Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// Entry represents an array with all exported HTTP requests. Sorting entries -// by startedDateTime (starting from the oldest) is preferred way how to export -// data since it can make importing faster. However the reader application -// should always make sure the array is sorted (if required for the import). +// Entry represents an array with all exported HTTP requests. Sorting entries by startedDateTime +// (starting from the oldest) is preferred way how to export data since it can make importing +// faster. However the reader application should always make sure the array is sorted (if required +// for the import). type Entry struct { Pageref string `json:"pageref,omitempty"` // Reference to the parent page. Leave out this field if the application does not support grouping by pages. StartedDateTime time.Time `json:"startedDateTime"` // Date and time stamp of the request start (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD). @@ -74,34 +74,33 @@ type Entry struct { // Request contains detailed info about performed request. type Request struct { - Method string `json:"method"` // Request method (GET, POST, ...). - URL string `json:"url"` // Absolute URL of the request (fragments are not included). - HTTPVersion string `json:"httpVersion"` // Request HTTP Version. - Cookies []*Cookie `json:"cookies"` // List of cookie objects. - Headers []*NameValuePair `json:"headers"` // List of header objects. - QueryString []*NameValuePair `json:"queryString"` // List of query parameter objects. - PostData *PostData `json:"postData,omitempty"` // Posted data info. - HeadersSize int64 `json:"headersSize"` // Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body. Set to -1 if the info is not available. - BodySize int64 `json:"bodySize"` // Size of the request body (POST data payload) in bytes. Set to -1 if the info is not available. - Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. + Method string `json:"method"` // Request method (GET, POST, ...). + URL string `json:"url"` // Absolute URL of the request (fragments are not included). + HTTPVersion string `json:"httpVersion"` // Request HTTP Version. + Cookies []*Cookie `json:"cookies"` // List of cookie objects. + Headers []*NVPair `json:"headers"` // List of header objects. + QueryString []*NVPair `json:"queryString"` // List of query parameter objects. + PostData *PostData `json:"postData,omitempty"` // Posted data info. + HeadersSize int64 `json:"headersSize"` // Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body. Set to -1 if the info is not available. + BodySize int64 `json:"bodySize"` // Size of the request body (POST data payload) in bytes. Set to -1 if the info is not available. + Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } // Response contains detailed info about the response. type Response struct { - Status int64 `json:"status"` // Response status. - StatusText string `json:"statusText"` // Response status description. - HTTPVersion string `json:"httpVersion"` // Response HTTP Version. - Cookies []*Cookie `json:"cookies"` // List of cookie objects. - Headers []*NameValuePair `json:"headers"` // List of header objects. - Content *Content `json:"content"` // Details about the response body. - RedirectURL *string `json:"redirectURL"` // Redirection target URL from the Location response header. - HeadersSize int64 `json:"headersSize"` // Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body. Set to -1 if the info is not available. - BodySize int64 `json:"bodySize"` // Size of the received response body in bytes. Set to zero in case of responses coming from the cache (304). Set to -1 if the info is not available. - Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. -} - -// Cookie contains list of all cookies (used in [Request] and [Response] -// objects). + Status int64 `json:"status"` // Response status. + StatusText string `json:"statusText"` // Response status description. + HTTPVersion string `json:"httpVersion"` // Response HTTP Version. + Cookies []*Cookie `json:"cookies"` // List of cookie objects. + Headers []*NVPair `json:"headers"` // List of header objects. + Content *Content `json:"content"` // Details about the response body. + RedirectURL *string `json:"redirectURL"` // Redirection target URL from the Location response header. + HeadersSize int64 `json:"headersSize"` // Total number of bytes from the start of the HTTP response message until (and including) the double CRLF before the body. Set to -1 if the info is not available. + BodySize int64 `json:"bodySize"` // Size of the received response body in bytes. Set to zero in case of responses coming from the cache (304). Set to -1 if the info is not available. + Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. +} + +// Cookie contains list of all cookies (used in [Request] and [Response] objects). type Cookie struct { Name string `json:"name"` // The name of the cookie. Value string `json:"value"` // The cookie value. @@ -113,15 +112,15 @@ type Cookie struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// NameValuePair describes a name/value pair. -type NameValuePair struct { +// NVPair describes a name/value pair. +type NVPair struct { Name string `json:"name"` // Name of the pair. Value string `json:"value"` // Value of the pair. Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// PostData describes posted data, if any (embedded in [Request] object). -// Text and params fields are mutually exclusive. +// PostData describes posted data, if any (embedded in [Request] object). Text and params fields are +// mutually exclusive. type PostData struct { MimeType string `json:"mimeType"` // Mime type of posted data. Params []*Param `json:"params,omitempty"` // List of posted parameters (in case of URL encoded parameters). @@ -138,8 +137,7 @@ type Param struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// Content describes details about response content (embedded in [Response] -// object). +// Content describes details about response content (embedded in [Response] object). type Content struct { Size int64 `json:"size"` // Length of the returned content in bytes. Should be equal to response.bodySize if there is no compression and bigger when the content has been compressed. Compression int64 `json:"compression,omitempty"` // Number of bytes saved. Leave out this field if the information is not available. @@ -165,8 +163,8 @@ type CacheData struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// Timings describes various phases within request-response round trip. All -// times are specified in milliseconds. +// Timings describes various phases within request-response round trip. All times are specified in +// milliseconds. type Timings struct { Blocked float64 `json:"blocked,omitempty,omitzero"` // Time spent in a queue waiting for a network connection. Use -1 if the timing does not apply to the current request. DNS float64 `json:"dns,omitempty,omitzero"` // DNS resolution time. The time required to resolve a host name. Use -1 if the timing does not apply to the current request. @@ -178,8 +176,8 @@ type Timings struct { Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. } -// Total returns the total time of the request/response round trip. -// Ignoring -1 values which indicate that the timing does not apply to the current request. +// Total returns the total time of the request/response round trip. Ignoring -1 values which +// indicate that the timing does not apply to the current request. func (t *Timings) Total() float64 { sum := 0.0 for _, v := range []float64{t.Blocked, t.DNS, t.Connect, t.Send, t.Wait, t.Receive, t.Ssl} { diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index eb405ad..3798b18 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -1,3 +1,5 @@ +// Package harhandler provides functionality to build HAR entries from bogdanfinn/fhttp requests +// and responses. package harhandler import ( @@ -12,15 +14,28 @@ import ( http "github.com/bogdanfinn/fhttp" ) +// EntryBuilder builds a HAR entry from a bogdanfinn/fhttp request and optionally a response. type EntryBuilder struct { entry *harfile.Entry } -func NewEntry() *EntryBuilder { +// NewEntryWithRequest creates a new EntryBuilder with the given request and cookies. It +// immediately attaches the request as HAR request in the underlying entry and merge cookies (if +// any) into the request headers and cookies. +func NewEntryWithRequest(req *http.Request, cookies []*http.Cookie) (*EntryBuilder, error) { + harReq, err := converter.FromHTTPRequest(req) + if err != nil { + return nil, err + } + + harReq.Headers = mergeCookieHeader(harReq.Headers, cookies) + harReq.Cookies = mergeCookies(harReq.Cookies, cookies) + return &EntryBuilder{ entry: &harfile.Entry{ StartedDateTime: time.Now(), Time: -1, + Request: harReq, Cache: &harfile.Cache{}, Timings: &harfile.Timings{ Send: -1, @@ -28,23 +43,10 @@ func NewEntry() *EntryBuilder { Receive: -1, }, }, - } -} - -func (e *EntryBuilder) AddRequest(req *http.Request, cookies []*http.Cookie) error { - harReq, err := converter.FromHTTPRequest(req) - if err != nil { - return err - } - - // If our client has a cookie jar, we need to merge the cookies - harReq.Headers = mergeCookieHeader(harReq.Headers, cookies) - harReq.Cookies = mergeCookies(harReq.Cookies, cookies) - - e.entry.Request = harReq - return nil + }, nil } +// AddResponse attaches an HTTP response to the entry and updates timing information. func (e *EntryBuilder) AddResponse(resp *http.Response) error { harResp, err := converter.FromHTTPResponse(resp) if err != nil { @@ -58,27 +60,31 @@ func (e *EntryBuilder) AddResponse(resp *http.Response) error { return nil } -func (e *EntryBuilder) build(resolveIP bool) *harfile.Entry { +// Build finalizes the HAR entry. If resolveIP is true, the server IP address will be resolved and +// stored in the entry. +func (e *EntryBuilder) Build(resolveIP bool) *harfile.Entry { if resolveIP && e.entry.Request != nil { e.entry.ServerIPAddress = getServerIPAddress(e.entry.Request.URL) } return e.entry } +// getServerIPAddress resolves the first IP address for the given URL and returns an empty string if +// resolution fails. This is a blocking call and may take time to resolve. func getServerIPAddress(reqUrl string) string { - url, err := url.Parse(reqUrl) + parsedUrl, err := url.Parse(reqUrl) if err != nil { return "" } - // WARN: This is a blocking call and may take time to resolve - ipAddress, err := net.DefaultResolver.LookupIPAddr(context.Background(), url.Hostname()) - if err != nil || len(ipAddress) == 0 { + ips, err := net.DefaultResolver.LookupIPAddr(context.Background(), parsedUrl.Hostname()) + if err != nil || len(ips) == 0 { return "" } - return ipAddress[0].IP.String() + return ips[0].IP.String() } +// mergeCookies appends new cookies to the existing cookie slice. func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Cookie { if len(new) == 0 { return existing @@ -87,15 +93,15 @@ func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Coo merged := make([]*harfile.Cookie, 0, len(existing)+len(new)) merged = append(merged, existing...) - for _, cookie := range new { + for _, c := range new { merged = append(merged, &harfile.Cookie{ - Name: cookie.Name, - Value: cookie.Value, - Path: cookie.Path, - Domain: cookie.Domain, - Expires: cookie.RawExpires, - HTTPOnly: cookie.HttpOnly, - Secure: cookie.Secure, + Name: c.Name, + Value: c.Value, + Path: c.Path, + Domain: c.Domain, + Expires: c.RawExpires, + HTTPOnly: c.HttpOnly, + Secure: c.Secure, }) } @@ -103,7 +109,9 @@ func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Coo } -func mergeCookieHeader(existing []*harfile.NameValuePair, new []*http.Cookie) []*harfile.NameValuePair { +// mergeCookieHeader merges new cookies into an existing "Cookie" header, or adds a new "Cookie" +// header if none exists. +func mergeCookieHeader(existing []*harfile.NVPair, new []*http.Cookie) []*harfile.NVPair { if len(new) == 0 { return existing } @@ -116,21 +124,15 @@ func mergeCookieHeader(existing []*harfile.NameValuePair, new []*http.Cookie) [] cookieHeader.WriteString(c.Name + "=" + c.Value) } - found := false for _, header := range existing { if strings.EqualFold(header.Name, "Cookie") { header.Value = header.Value + "; " + cookieHeader.String() - found = true - break + return existing } } - if !found { - existing = append(existing, &harfile.NameValuePair{ - Name: "Cookie", - Value: cookieHeader.String(), - }) - } - - return existing + return append(existing, &harfile.NVPair{ + Name: "Cookie", + Value: cookieHeader.String(), + }) } diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index be883b0..9e42c83 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -39,7 +39,7 @@ func NewHandler(opts ...HandlerOption) *HARHandler { } func (h *HARHandler) AddEntry(builder *EntryBuilder) { - entry := builder.build(h.resolveIPAddress) + entry := builder.Build(h.resolveIPAddress) h.log.Entries = append(h.log.Entries, entry) } From c21726b62aebfd34026bb4326c15a80641c74b78 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 18:48:10 +0000 Subject: [PATCH 38/60] fix: update comments for clarity and improve variable naming in entry builder --- README.md | 14 +++++- example/main.go | 2 +- harhandler/entry_builder.go | 88 ++++++++++++++++++------------------- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 9034e70..69bb0ed 100644 --- a/README.md +++ b/README.md @@ -2,5 +2,15 @@ A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. -- [ ] Check for case sensitivity in HeaderOrder keys -- [ ] Add dynamic versioning to the library +- [X] Check for case sensitivity in HeaderOrder keys +- [ ] Add Remote Add Address (proxy used) +- [ ] Fix bodySize + +## How to compile + +### MacOS + +```bash +cd example +GOOS=darwin GOARCH=amd64 go build example/main.go +``` diff --git a/example/main.go b/example/main.go index 6f035ad..fd6c06c 100644 --- a/example/main.go +++ b/example/main.go @@ -20,7 +20,7 @@ func main() { tls_client.NewNoopLogger(), tls_client.WithCookieJar(tls_client.NewCookieJar()), tls_client.WithNotFollowRedirects(), - tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this + // tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this ) // 1. Get with query parameters diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 3798b18..5c96076 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -20,16 +20,16 @@ type EntryBuilder struct { } // NewEntryWithRequest creates a new EntryBuilder with the given request and cookies. It -// immediately attaches the request as HAR request in the underlying entry and merge cookies (if -// any) into the request headers and cookies. -func NewEntryWithRequest(req *http.Request, cookies []*http.Cookie) (*EntryBuilder, error) { +// immediately attaches the request as a HAR request in the underlying entry, merging cookies +// into both the request headers and cookie list. +func NewEntryWithRequest(req *http.Request, additionalCookies []*http.Cookie) (*EntryBuilder, error) { harReq, err := converter.FromHTTPRequest(req) if err != nil { return nil, err } - harReq.Headers = mergeCookieHeader(harReq.Headers, cookies) - harReq.Cookies = mergeCookies(harReq.Cookies, cookies) + harReq.Headers = mergeCookieHeader(harReq.Headers, additionalCookies) + harReq.Cookies = mergeCookies(harReq.Cookies, additionalCookies) return &EntryBuilder{ entry: &harfile.Entry{ @@ -47,54 +47,54 @@ func NewEntryWithRequest(req *http.Request, cookies []*http.Cookie) (*EntryBuild } // AddResponse attaches an HTTP response to the entry and updates timing information. -func (e *EntryBuilder) AddResponse(resp *http.Response) error { +func (b *EntryBuilder) AddResponse(resp *http.Response) error { harResp, err := converter.FromHTTPResponse(resp) if err != nil { return err } - e.entry.Response = harResp + b.entry.Response = harResp - e.entry.Timings.Receive = float64(time.Since(e.entry.StartedDateTime).Milliseconds()) - e.entry.Time = e.entry.Timings.Total() + b.entry.Timings.Receive = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) + b.entry.Time = b.entry.Timings.Total() return nil } // Build finalizes the HAR entry. If resolveIP is true, the server IP address will be resolved and // stored in the entry. -func (e *EntryBuilder) Build(resolveIP bool) *harfile.Entry { - if resolveIP && e.entry.Request != nil { - e.entry.ServerIPAddress = getServerIPAddress(e.entry.Request.URL) +func (b *EntryBuilder) Build(resolveIP bool) *harfile.Entry { + if resolveIP && b.entry.Request != nil { + b.entry.ServerIPAddress = resolveServerIPAddress(b.entry.Request.URL) } - return e.entry + return b.entry } -// getServerIPAddress resolves the first IP address for the given URL and returns an empty string if -// resolution fails. This is a blocking call and may take time to resolve. -func getServerIPAddress(reqUrl string) string { - parsedUrl, err := url.Parse(reqUrl) +// resolveServerIPAddress resolves the first IP address for the given URL. Returns an empty +// string if resolution fails. This is a blocking call and may take time to resolve. +func resolveServerIPAddress(rawURL string) string { + parsedURL, err := url.Parse(rawURL) if err != nil { return "" } - ips, err := net.DefaultResolver.LookupIPAddr(context.Background(), parsedUrl.Hostname()) - if err != nil || len(ips) == 0 { + ipAddrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), parsedURL.Hostname()) + if err != nil || len(ipAddrs) == 0 { return "" } - return ips[0].IP.String() + return ipAddrs[0].IP.String() } -// mergeCookies appends new cookies to the existing cookie slice. -func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Cookie { - if len(new) == 0 { - return existing +// mergeCookies appends new cookies into an existing cookie slice. +func mergeCookies(existingCookies []*harfile.Cookie, newCookies []*http.Cookie) []*harfile.Cookie { + if len(newCookies) == 0 { + return existingCookies } - merged := make([]*harfile.Cookie, 0, len(existing)+len(new)) - merged = append(merged, existing...) + combined := make([]*harfile.Cookie, 0, len(existingCookies)+len(newCookies)) + combined = append(combined, existingCookies...) - for _, c := range new { - merged = append(merged, &harfile.Cookie{ + for _, c := range newCookies { + combined = append(combined, &harfile.Cookie{ Name: c.Name, Value: c.Value, Path: c.Path, @@ -104,35 +104,33 @@ func mergeCookies(existing []*harfile.Cookie, new []*http.Cookie) []*harfile.Coo Secure: c.Secure, }) } - - return merged - + return combined } // mergeCookieHeader merges new cookies into an existing "Cookie" header, or adds a new "Cookie" -// header if none exists. -func mergeCookieHeader(existing []*harfile.NVPair, new []*http.Cookie) []*harfile.NVPair { - if len(new) == 0 { - return existing +// header if one doesn't already exist. +func mergeCookieHeader(existingHeaders []*harfile.NVPair, newCookies []*http.Cookie) []*harfile.NVPair { + if len(newCookies) == 0 { + return existingHeaders } - var cookieHeader strings.Builder - for i, c := range new { + var mergedCookieValue strings.Builder + for i, c := range newCookies { if i > 0 { - cookieHeader.WriteString("; ") + mergedCookieValue.WriteString("; ") } - cookieHeader.WriteString(c.Name + "=" + c.Value) + mergedCookieValue.WriteString(c.Name + "=" + c.Value) } - for _, header := range existing { - if strings.EqualFold(header.Name, "Cookie") { - header.Value = header.Value + "; " + cookieHeader.String() - return existing + for _, hdr := range existingHeaders { + if strings.EqualFold(hdr.Name, "Cookie") { + hdr.Value += "; " + mergedCookieValue.String() + return existingHeaders } } - return append(existing, &harfile.NVPair{ + return append(existingHeaders, &harfile.NVPair{ Name: "Cookie", - Value: cookieHeader.String(), + Value: mergedCookieValue.String(), }) } From cd21c9018d405b8849534f9473bd49e0c439ffc7 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 15 Apr 2025 22:33:09 +0000 Subject: [PATCH 39/60] fix: enhance NewEntryWithRequest to clone request and preserve body, update comments for clarity --- converter/request.go | 2 +- converter/response.go | 2 +- harhandler/entry_builder.go | 94 ++++++++++++++----------------------- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/converter/request.go b/converter/request.go index 2de8f76..cf3277d 100644 --- a/converter/request.go +++ b/converter/request.go @@ -58,7 +58,7 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { return nil, err } defer req.Body.Close() - req.Body = io.NopCloser(bytes.NewBuffer(buf)) + req.Body = io.NopCloser(bytes.NewReader(buf)) mimeType := req.Header.Get(ContentTypeKey) postData := &harfile.PostData{MimeType: mimeType} diff --git a/converter/response.go b/converter/response.go index 7250941..a4439b0 100644 --- a/converter/response.go +++ b/converter/response.go @@ -48,7 +48,7 @@ func buildContent(resp *http.Response) (*harfile.Content, error) { return nil, err } defer resp.Body.Close() - resp.Body = io.NopCloser(bytes.NewBuffer(buf)) + resp.Body = io.NopCloser(bytes.NewReader(buf)) return &harfile.Content{ Size: int64(len(buf)), diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 5c96076..28bc69d 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -1,12 +1,14 @@ // Package harhandler provides functionality to build HAR entries from bogdanfinn/fhttp requests -// and responses. +// and responses. It ensures correct extraction of request and response data into the HAR format, +// including body content, headers, cookies, and timing information. package harhandler import ( + "bytes" "context" + "io" "net" "net/url" - "strings" "time" "github.com/Mathious6/harkit/converter" @@ -19,18 +21,23 @@ type EntryBuilder struct { entry *harfile.Entry } -// NewEntryWithRequest creates a new EntryBuilder with the given request and cookies. It -// immediately attaches the request as a HAR request in the underlying entry, merging cookies -// into both the request headers and cookie list. +// NewEntryWithRequest creates a new EntryBuilder from the given HTTP request and optional cookies. +// It clones the request and preserves the request body to ensure that the original request remains +// usable. The body is read once, stored in memory, and restored on both the original and cloned +// request. Additional cookies are added to the cloned request before it is transformed into a +// HAR-compatible structure. func NewEntryWithRequest(req *http.Request, additionalCookies []*http.Cookie) (*EntryBuilder, error) { - harReq, err := converter.FromHTTPRequest(req) + clonedReq, _ := cloneRequestPreserveBody(req) + + for _, c := range additionalCookies { + clonedReq.AddCookie(c) + } + + harReq, err := converter.FromHTTPRequest(clonedReq) if err != nil { return nil, err } - harReq.Headers = mergeCookieHeader(harReq.Headers, additionalCookies) - harReq.Cookies = mergeCookies(harReq.Cookies, additionalCookies) - return &EntryBuilder{ entry: &harfile.Entry{ StartedDateTime: time.Now(), @@ -46,7 +53,8 @@ func NewEntryWithRequest(req *http.Request, additionalCookies []*http.Cookie) (* }, nil } -// AddResponse attaches an HTTP response to the entry and updates timing information. +// AddResponse attaches an HTTP response to the HAR entry and records the time elapsed since the +// request was initiated. This sets the response block and populates the receive timing. func (b *EntryBuilder) AddResponse(resp *http.Response) error { harResp, err := converter.FromHTTPResponse(resp) if err != nil { @@ -60,8 +68,8 @@ func (b *EntryBuilder) AddResponse(resp *http.Response) error { return nil } -// Build finalizes the HAR entry. If resolveIP is true, the server IP address will be resolved and -// stored in the entry. +// Build finalizes and returns the HAR entry. If resolveIP is true, it attempts to resolve the +// server's IP address using a DNS lookup based on the request URL. func (b *EntryBuilder) Build(resolveIP bool) *harfile.Entry { if resolveIP && b.entry.Request != nil { b.entry.ServerIPAddress = resolveServerIPAddress(b.entry.Request.URL) @@ -69,8 +77,8 @@ func (b *EntryBuilder) Build(resolveIP bool) *harfile.Entry { return b.entry } -// resolveServerIPAddress resolves the first IP address for the given URL. Returns an empty -// string if resolution fails. This is a blocking call and may take time to resolve. +// resolveServerIPAddress performs a DNS lookup on the given URL and returns the first resolved +// IP address as a string. Returns an empty string on failure. This is a blocking operation. func resolveServerIPAddress(rawURL string) string { parsedURL, err := url.Parse(rawURL) if err != nil { @@ -84,53 +92,23 @@ func resolveServerIPAddress(rawURL string) string { return ipAddrs[0].IP.String() } -// mergeCookies appends new cookies into an existing cookie slice. -func mergeCookies(existingCookies []*harfile.Cookie, newCookies []*http.Cookie) []*harfile.Cookie { - if len(newCookies) == 0 { - return existingCookies - } - - combined := make([]*harfile.Cookie, 0, len(existingCookies)+len(newCookies)) - combined = append(combined, existingCookies...) - - for _, c := range newCookies { - combined = append(combined, &harfile.Cookie{ - Name: c.Name, - Value: c.Value, - Path: c.Path, - Domain: c.Domain, - Expires: c.RawExpires, - HTTPOnly: c.HttpOnly, - Secure: c.Secure, - }) - } - return combined -} - -// mergeCookieHeader merges new cookies into an existing "Cookie" header, or adds a new "Cookie" -// header if one doesn't already exist. -func mergeCookieHeader(existingHeaders []*harfile.NVPair, newCookies []*http.Cookie) []*harfile.NVPair { - if len(newCookies) == 0 { - return existingHeaders +// cloneRequestPreserveBody clones an HTTP request and preserves its body by reading it into memory +// and assigning new readers to both the original and cloned request. This allows for safe reuse of +// both requests without consuming the body multiple times. +func cloneRequestPreserveBody(req *http.Request) (*http.Request, error) { + if req.Body == nil { + return req.Clone(req.Context()), nil } - var mergedCookieValue strings.Builder - for i, c := range newCookies { - if i > 0 { - mergedCookieValue.WriteString("; ") - } - mergedCookieValue.WriteString(c.Name + "=" + c.Value) + buf, err := io.ReadAll(req.Body) + if err != nil { + return nil, err } + defer req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(buf)) - for _, hdr := range existingHeaders { - if strings.EqualFold(hdr.Name, "Cookie") { - hdr.Value += "; " + mergedCookieValue.String() - return existingHeaders - } - } + clonedReq := req.Clone(req.Context()) + clonedReq.Body = io.NopCloser(bytes.NewReader(buf)) - return append(existingHeaders, &harfile.NVPair{ - Name: "Cookie", - Value: mergedCookieValue.String(), - }) + return clonedReq, nil } From 3cdb1e7b2cff7029a59be23a252d4253fc5ad27d Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Wed, 16 Apr 2025 12:35:01 +0000 Subject: [PATCH 40/60] fix: refactor HAR entry creation to use NewEntry and improve request handling --- example/main.go | 16 ++++----- harhandler/entry_builder.go | 68 ++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/example/main.go b/example/main.go index fd6c06c..0f992c8 100644 --- a/example/main.go +++ b/example/main.go @@ -58,16 +58,14 @@ func sendGetRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL strin req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - urlParsed, _ := url.Parse(URL) - cookies := c.GetCookies(urlParsed) - entry, _ := harhandler.NewEntryWithRequest(req, cookies) - + entry := harhandler.NewEntry() resp, err := c.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddResponse(resp) + urlParsed, _ := url.Parse(URL) + _ = entry.AddEntry(req, resp, c.GetCookies(urlParsed)) h.AddEntry(entry) } @@ -90,16 +88,14 @@ func sendPostRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL stri req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - urlParsed, _ := url.Parse(URL) - cookies := c.GetCookies(urlParsed) - entry, _ := harhandler.NewEntryWithRequest(req, cookies) - + entry := harhandler.NewEntry() resp, err := c.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddResponse(resp) + urlParsed, _ := url.Parse(URL) + _ = entry.AddEntry(req, resp, c.GetCookies(urlParsed)) h.AddEntry(entry) } diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 28bc69d..98497fd 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -1,6 +1,7 @@ // Package harhandler provides functionality to build HAR entries from bogdanfinn/fhttp requests -// and responses. It ensures correct extraction of request and response data into the HAR format, -// including body content, headers, cookies, and timing information. +// and responses. It allows deferred HAR construction after the request is executed, ensuring +// accurate extraction of request and response data, including body content, headers, cookies, +// and timing information. package harhandler import ( @@ -16,33 +17,19 @@ import ( http "github.com/bogdanfinn/fhttp" ) -// EntryBuilder builds a HAR entry from a bogdanfinn/fhttp request and optionally a response. +// EntryBuilder builds a HAR entry from a bogdanfinn/fhttp request and response. +// It encapsulates all relevant data including timings, headers, cookies, and body content. type EntryBuilder struct { entry *harfile.Entry } -// NewEntryWithRequest creates a new EntryBuilder from the given HTTP request and optional cookies. -// It clones the request and preserves the request body to ensure that the original request remains -// usable. The body is read once, stored in memory, and restored on both the original and cloned -// request. Additional cookies are added to the cloned request before it is transformed into a -// HAR-compatible structure. -func NewEntryWithRequest(req *http.Request, additionalCookies []*http.Cookie) (*EntryBuilder, error) { - clonedReq, _ := cloneRequestPreserveBody(req) - - for _, c := range additionalCookies { - clonedReq.AddCookie(c) - } - - harReq, err := converter.FromHTTPRequest(clonedReq) - if err != nil { - return nil, err - } - +// NewEntry initializes an empty HAR entry with default fields and timing placeholders. +// Use AddEntry to attach the request and response data once the request is completed. +func NewEntry() *EntryBuilder { return &EntryBuilder{ entry: &harfile.Entry{ StartedDateTime: time.Now(), Time: -1, - Request: harReq, Cache: &harfile.Cache{}, Timings: &harfile.Timings{ Send: -1, @@ -50,26 +37,45 @@ func NewEntryWithRequest(req *http.Request, additionalCookies []*http.Cookie) (* Receive: -1, }, }, - }, nil + } } -// AddResponse attaches an HTTP response to the HAR entry and records the time elapsed since the -// request was initiated. This sets the response block and populates the receive timing. -func (b *EntryBuilder) AddResponse(resp *http.Response) error { +// AddEntry populates the HAR entry with the given HTTP request and response, as well as any +// additional cookies. It clones and restores the request body to prevent side effects. +// This method should be called after the HTTP request is executed to avoid blocking. +func (b *EntryBuilder) AddEntry( + req *http.Request, resp *http.Response, additionalCookies []*http.Cookie, +) error { + b.entry.Timings.Receive = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) + + clonedReq, err := cloneRequestPreserveBody(req) + if err != nil { + return err + } + for _, c := range additionalCookies { + clonedReq.AddCookie(c) + } + + harReq, err := converter.FromHTTPRequest(clonedReq) + if err != nil { + return err + } harResp, err := converter.FromHTTPResponse(resp) if err != nil { return err } + + b.entry.Request = harReq b.entry.Response = harResp - b.entry.Timings.Receive = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) + b.entry.Timings.Wait = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) - b.entry.Timings.Receive b.entry.Time = b.entry.Timings.Total() return nil } -// Build finalizes and returns the HAR entry. If resolveIP is true, it attempts to resolve the -// server's IP address using a DNS lookup based on the request URL. +// Build finalizes and returns the constructed HAR entry. +// If resolveIP is true, a DNS resolution will be performed on the request host. func (b *EntryBuilder) Build(resolveIP bool) *harfile.Entry { if resolveIP && b.entry.Request != nil { b.entry.ServerIPAddress = resolveServerIPAddress(b.entry.Request.URL) @@ -92,9 +98,9 @@ func resolveServerIPAddress(rawURL string) string { return ipAddrs[0].IP.String() } -// cloneRequestPreserveBody clones an HTTP request and preserves its body by reading it into memory -// and assigning new readers to both the original and cloned request. This allows for safe reuse of -// both requests without consuming the body multiple times. +// cloneRequestPreserveBody clones an HTTP request and preserves its body by buffering the content +// into memory. Both the original and the cloned request will be reset with a fresh body reader, +// allowing for safe reuse without data loss. func cloneRequestPreserveBody(req *http.Request) (*http.Request, error) { if req.Body == nil { return req.Clone(req.Context()), nil From 480d58def04166fb778a3231197b65d6ae725d39 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 16 Jun 2025 20:08:02 +0000 Subject: [PATCH 41/60] chore: update Dockerfile to use Microsoft container image, streamline Zsh installation, and enhance plugin setup --- .devcontainer/Dockerfile | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f9e2e18..f5907ee 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,21 +1,17 @@ -FROM golang:1.24 +FROM mcr.microsoft.com/devcontainers/go:1.24 -RUN apt-get update && \ - apt-get install -y zsh tree && \ - rm -rf /var/lib/apt/lists/* - -RUN useradd -m vscode +# This is important to change the user to clone/copy/etc. in good folder USER vscode -WORKDIR /workspace - -ENV PATH=$PATH:/usr/local/go/bin:/go/bin +WORKDIR /home/vscode -RUN go install github.com/go-delve/delve/cmd/dlv@latest -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.0.2 +# Install Oh My Zsh : https://github.com/ohmyzsh/ohmyzsh?tab=readme-ov-file#basic-installation +# We need to use '|| true' to avoid the script to stop the execution if the installation is already done +RUN sh -c "$(wget https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh -O -)" || true -RUN sh -c "$(wget https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh -O -)" && \ - git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions && \ +# Download and install ZSH plugins +RUN git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions && \ git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting && \ git clone https://github.com/MichaelAquilina/zsh-you-should-use.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/you-should-use -COPY --chown=vscode:vscode .zshrc /home/vscode/.zshrc +# Copy .devcontainer/zshrc config to apply plugins/themes/etc. +COPY --chown=vscode:vscode .zshrc .zshrc From 53e64c689598f374b0f71caa90b8797842eea180 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 16 Jun 2025 20:25:14 +0000 Subject: [PATCH 42/60] chore: remove Dockerfile and Zsh configuration, update devcontainer.json to use container image and manage Zsh plugins through features --- .devcontainer/.zshrc | 13 ------------- .devcontainer/Dockerfile | 17 ----------------- .devcontainer/devcontainer.json | 11 +++++++++-- 3 files changed, 9 insertions(+), 32 deletions(-) delete mode 100644 .devcontainer/.zshrc delete mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/.zshrc b/.devcontainer/.zshrc deleted file mode 100644 index 45b1a63..0000000 --- a/.devcontainer/.zshrc +++ /dev/null @@ -1,13 +0,0 @@ -export ZSH="$HOME/.oh-my-zsh" - -ZSH_THEME="robbyrussell" - -plugins=( - zsh-autosuggestions - zsh-syntax-highlighting - you-should-use - git - golang -) - -source $ZSH/oh-my-zsh.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index f5907ee..0000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM mcr.microsoft.com/devcontainers/go:1.24 - -# This is important to change the user to clone/copy/etc. in good folder -USER vscode -WORKDIR /home/vscode - -# Install Oh My Zsh : https://github.com/ohmyzsh/ohmyzsh?tab=readme-ov-file#basic-installation -# We need to use '|| true' to avoid the script to stop the execution if the installation is already done -RUN sh -c "$(wget https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh -O -)" || true - -# Download and install ZSH plugins -RUN git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions && \ - git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting && \ - git clone https://github.com/MichaelAquilina/zsh-you-should-use.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/you-should-use - -# Copy .devcontainer/zshrc config to apply plugins/themes/etc. -COPY --chown=vscode:vscode .zshrc .zshrc diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d1df9fd..0b62f71 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Go", - "dockerFile": "Dockerfile", + "image": "mcr.microsoft.com/devcontainers/go:1.24", "remoteUser": "vscode", "shutdownAction": "stopContainer", "postCreateCommand": "go mod download", @@ -17,5 +17,12 @@ }, "mounts": [ "source=go-modules,target=/go,type=volume" // Keep go modules in a volume - ] + ], + "features": { + "ghcr.io/devcontainers-extra/features/zsh-plugins:0": { + "plugins": "git golang zsh-autosuggestions zsh-syntax-highlighting zsh-you-should-use", + "omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions https://github.com/zsh-users/zsh-syntax-highlighting https://github.com/MichaelAquilina/zsh-you-should-use", + "username": "vscode" + } + } } From 7b4dd881c385dbc1e26e2a132e189bb3be7198c0 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 17 Jun 2025 01:08:43 +0000 Subject: [PATCH 43/60] fix: refactor example main function to improve request handling and add new request methods for GET and POST with query parameters, form data, and JSON --- README.md | 4 +- example/main.go | 113 ++++++++++++++++++++++++++---------- harhandler/entry_builder.go | 11 +--- 3 files changed, 86 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 69bb0ed..3be1de0 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. - [X] Check for case sensitivity in HeaderOrder keys -- [ ] Add Remote Add Address (proxy used) +- [ ] Add Remote Address (proxy used) - [ ] Fix bodySize ## How to compile @@ -12,5 +12,5 @@ A Golang library for parsing and managing HAR (HTTP Archive) files. Provides eas ```bash cd example -GOOS=darwin GOARCH=amd64 go build example/main.go +GOOS=darwin GOARCH=arm64 go build -o example-harkit main.go ``` diff --git a/example/main.go b/example/main.go index 0f992c8..59d98d7 100644 --- a/example/main.go +++ b/example/main.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "io" "net/url" "strings" @@ -20,60 +19,109 @@ func main() { tls_client.NewNoopLogger(), tls_client.WithCookieJar(tls_client.NewCookieJar()), tls_client.WithNotFollowRedirects(), - // tls_client.WithCharlesProxy("127.0.0.1", "8888"), // TODO : do not commit this ) - // 1. Get with query parameters - sendGetRequest(client, handler, URL+"/get?name=pierre&role=developer") - fmt.Println("✅ Parameters sent") + sendGetRequestWithQueryParams(handler, client) + sendGetRequestWithSetCookies(handler, client) + sendPostRequestWithForm(handler, client) + sendPostRequestWithJSON(handler, client) - // 2. Set cookies - sendGetRequest(client, handler, URL+"/cookies/set?name=pierre&role=developer") - fmt.Println("✅ Cookies set") + handler.Save("example.har") +} - // 3. Form URL-encoded - form := url.Values{} - form.Set("name", "Pierre") - form.Set("role", "developer") - sendPostRequest(client, handler, URL+"/post", "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) - fmt.Println("✅ Form URL-encoded request sent") +func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client tls_client.HttpClient) { + req, _ := http.NewRequest(http.MethodGet, URL+"/get?name=pierre&role=developer", nil) + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "httpbin.org") + req.Header.Add("User-Agent", "harkit-example") + req.Header.Add("Accept-Encoding", "gzip, deflate, br") - // 4. JSON - jsonBody := `{"name":"Pierre","role":"developer"}` - sendPostRequest(client, handler, URL+"/post", "application/json", strings.NewReader(jsonBody)) - fmt.Println("✅ JSON request sent") + req.Header.Add(http.HeaderOrderKey, "accept") + req.Header.Add(http.HeaderOrderKey, "host") + req.Header.Add(http.HeaderOrderKey, "user-agent") + req.Header.Add(http.HeaderOrderKey, "accept-encoding") - handler.Save("example.har") + entry := harhandler.NewEntry() + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + _ = entry.AddEntry(req, resp) + + handler.AddEntry(entry) + + fmt.Println("Parameters sent.") +} + +func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client tls_client.HttpClient) { + req, _ := http.NewRequest(http.MethodGet, URL+"/cookies/set?name=pierre&role=developer", nil) + req.Header.Add("Accept", "*/*") + req.Header.Add("Host", "httpbin.org") + req.Header.Add("User-Agent", "harkit-example") + req.Header.Add("Accept-Encoding", "gzip, deflate, br") + + req.Header.Add(http.HeaderOrderKey, "accept") + req.Header.Add(http.HeaderOrderKey, "host") + req.Header.Add(http.HeaderOrderKey, "user-agent") + req.Header.Add(http.HeaderOrderKey, "accept-encoding") + + entry := harhandler.NewEntry() + resp, err := client.Do(req) + if err != nil { + panic(err) + } + defer resp.Body.Close() + _ = entry.AddEntry(req, resp) + + handler.AddEntry(entry) + + fmt.Println("Cookies set.") } -func sendGetRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL string) { - req, _ := http.NewRequest(http.MethodGet, URL, nil) +func sendPostRequestWithForm(handler *harhandler.HARHandler, client tls_client.HttpClient) { + form := url.Values{} + form.Set("name", "Pierre") + form.Set("role", "developer") + body := strings.NewReader(form.Encode()) + + req, _ := http.NewRequest(http.MethodPost, URL+"/post", body) req.Header.Add("Accept", "*/*") + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") + req.AddCookie(&http.Cookie{Name: "example", Value: "cookie"}) + req.Header.Add(http.HeaderOrderKey, "accept") + req.Header.Add(http.HeaderOrderKey, "content-length") + req.Header.Add(http.HeaderOrderKey, "content-type") + req.Header.Add(http.HeaderOrderKey, "cookie") req.Header.Add(http.HeaderOrderKey, "host") req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") entry := harhandler.NewEntry() - resp, err := c.Do(req) + resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - urlParsed, _ := url.Parse(URL) - _ = entry.AddEntry(req, resp, c.GetCookies(urlParsed)) + _ = entry.AddEntry(req, resp) + + handler.AddEntry(entry) - h.AddEntry(entry) + fmt.Println("Form URL-encoded request sent.") } -func sendPostRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL string, contentType string, body io.Reader) { - req, _ := http.NewRequest(http.MethodPost, URL, body) +func sendPostRequestWithJSON(handler *harhandler.HARHandler, client tls_client.HttpClient) { + jsonBody := `{"name":"Pierre","role":"developer"}` + body := strings.NewReader(jsonBody) + + req, _ := http.NewRequest(http.MethodPost, URL+"/post", body) req.Header.Add("Accept", "*/*") - req.Header.Add("Content-Type", contentType) + req.Header.Add("Content-Type", "application/json") req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") @@ -89,13 +137,14 @@ func sendPostRequest(c tls_client.HttpClient, h *harhandler.HARHandler, URL stri req.Header.Add(http.HeaderOrderKey, "accept-encoding") entry := harhandler.NewEntry() - resp, err := c.Do(req) + resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - urlParsed, _ := url.Parse(URL) - _ = entry.AddEntry(req, resp, c.GetCookies(urlParsed)) + _ = entry.AddEntry(req, resp) + + handler.AddEntry(entry) - h.AddEntry(entry) + fmt.Println("JSON request sent.") } diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go index 98497fd..98e6cf1 100644 --- a/harhandler/entry_builder.go +++ b/harhandler/entry_builder.go @@ -40,21 +40,16 @@ func NewEntry() *EntryBuilder { } } -// AddEntry populates the HAR entry with the given HTTP request and response, as well as any -// additional cookies. It clones and restores the request body to prevent side effects. +// AddEntry populates the HAR entry with the given HTTP request and response. +// It clones and restores the request body to prevent side effects. // This method should be called after the HTTP request is executed to avoid blocking. -func (b *EntryBuilder) AddEntry( - req *http.Request, resp *http.Response, additionalCookies []*http.Cookie, -) error { +func (b *EntryBuilder) AddEntry(req *http.Request, resp *http.Response) error { b.entry.Timings.Receive = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) clonedReq, err := cloneRequestPreserveBody(req) if err != nil { return err } - for _, c := range additionalCookies { - clonedReq.AddCookie(c) - } harReq, err := converter.FromHTTPRequest(clonedReq) if err != nil { From 0b3d1cde496683ef37591bda4696bad1117a26e7 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 17 Jun 2025 01:41:19 +0000 Subject: [PATCH 44/60] chore: update devcontainer setup with initialization and post-create scripts, enhance .gitignore for development files, and adjust README for clarity --- .devcontainer/devcontainer.json | 3 ++- .devcontainer/initialize.sh | 13 +++++++++++++ .devcontainer/post-create.sh | 6 ++++++ .gitignore | 5 +++++ .vscode/settings.json | 3 +++ README.md | 9 --------- example/main.go | 1 + 7 files changed, 30 insertions(+), 10 deletions(-) create mode 100755 .devcontainer/initialize.sh create mode 100755 .devcontainer/post-create.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0b62f71..cf56bda 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,8 @@ "image": "mcr.microsoft.com/devcontainers/go:1.24", "remoteUser": "vscode", "shutdownAction": "stopContainer", - "postCreateCommand": "go mod download", + "initializeCommand": "./.devcontainer/initialize.sh", + "postCreateCommand": "./.devcontainer/post-create.sh", "customizations": { "vscode": { "settings": { diff --git a/.devcontainer/initialize.sh b/.devcontainer/initialize.sh new file mode 100755 index 0000000..9e60b18 --- /dev/null +++ b/.devcontainer/initialize.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ "$(uname)" == "Darwin" ]; then + CHARLES_APP=$(mdfind "kMDItemCFBundleIdentifier == 'com.xk72.Charles'" | head -n 1) + + if [ -d "$CHARLES_APP" ]; then + rm -rf .certs + mkdir -p .certs + "$CHARLES_APP/Contents/MacOS/Charles" ssl export .certs/charles-ssl.pem + else + echo "Charles is not installed, skipping certificate export." + fi +fi diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 0000000..ec4b3f3 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +go mod download + +sudo cp .certs/charles-ssl.pem /usr/local/share/ca-certificates/charles-ssl-proxying-certificate.crt +sudo update-ca-certificates diff --git a/.gitignore b/.gitignore index 787a256..7beaffa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,9 @@ +# DEVCONTAINER +.certs + +# DEBUG example-harkit *.har +# OS .DS_Store diff --git a/.vscode/settings.json b/.vscode/settings.json index e5f5ad3..9d78bad 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,9 @@ "files.autoSave": "onFocusChange", "files.insertFinalNewline": true, "files.trimFinalNewlines": true, + "files.exclude": { + "**/.certs": true + }, // GOLANG SETTINGS: "go.toolsManagement.autoUpdate": true, "go.useLanguageServer": true, diff --git a/README.md b/README.md index 3be1de0..58d6507 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,3 @@ A Golang library for parsing and managing HAR (HTTP Archive) files. Provides eas - [X] Check for case sensitivity in HeaderOrder keys - [ ] Add Remote Address (proxy used) - [ ] Fix bodySize - -## How to compile - -### MacOS - -```bash -cd example -GOOS=darwin GOARCH=arm64 go build -o example-harkit main.go -``` diff --git a/example/main.go b/example/main.go index 59d98d7..7f662df 100644 --- a/example/main.go +++ b/example/main.go @@ -19,6 +19,7 @@ func main() { tls_client.NewNoopLogger(), tls_client.WithCookieJar(tls_client.NewCookieJar()), tls_client.WithNotFollowRedirects(), + tls_client.WithCharlesProxy("host.docker.internal", "8888"), ) sendGetRequestWithQueryParams(handler, client) From 44c02748cf97622acb1bd25381cb333d94923174 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Tue, 17 Jun 2025 01:51:07 +0000 Subject: [PATCH 45/60] feat: add proxy support in main function to conditionally use Charles proxy for HTTP requests and implement isProxyRunning function to check proxy availability --- example/main.go | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/example/main.go b/example/main.go index 7f662df..a716e6b 100644 --- a/example/main.go +++ b/example/main.go @@ -2,25 +2,41 @@ package main import ( "fmt" + "net" "net/url" "strings" + "time" "github.com/Mathious6/harkit/harhandler" http "github.com/bogdanfinn/fhttp" tls_client "github.com/bogdanfinn/tls-client" ) -const URL = "https://httpbin.org" +const ( + URL = "https://httpbin.org" + PROXY_HOST = "host.docker.internal" + PROXY_PORT = "8888" +) func main() { handler := harhandler.NewHandler() - client, _ := tls_client.NewHttpClient( - tls_client.NewNoopLogger(), + opts := []tls_client.HttpClientOption{ tls_client.WithCookieJar(tls_client.NewCookieJar()), tls_client.WithNotFollowRedirects(), - tls_client.WithCharlesProxy("host.docker.internal", "8888"), - ) + } + + if isProxyRunning(net.JoinHostPort(PROXY_HOST, PROXY_PORT), 100*time.Millisecond) { + opts = append(opts, tls_client.WithCharlesProxy(PROXY_HOST, PROXY_PORT)) + fmt.Println("Using Charles proxy.") + } else { + fmt.Println("Charles proxy not running, using direct connection.") + } + + client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), opts...) + if err != nil { + panic(err) + } sendGetRequestWithQueryParams(handler, client) sendGetRequestWithSetCookies(handler, client) @@ -149,3 +165,13 @@ func sendPostRequestWithJSON(handler *harhandler.HARHandler, client tls_client.H fmt.Println("JSON request sent.") } + +// isProxyRunning checks if a proxy is running on the given address and port. +func isProxyRunning(address string, timeout time.Duration) bool { + conn, err := net.DialTimeout("tcp", address, timeout) + if err != nil { + return false + } + defer conn.Close() + return true +} From 5aee718b71ebc42ecc0adfbee3015902106f4e21 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Wed, 23 Jul 2025 22:13:27 +0000 Subject: [PATCH 46/60] add: update example to replace tls-client with httpkit and adjust related function signatures --- README.md | 35 +++++++++++++++++++++++++++++++---- example/go.mod | 16 +++++++++++----- example/go.sum | 30 ++++++++++++++++++++---------- example/main.go | 26 +++++++++++++------------- 4 files changed, 75 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 58d6507..29d60e2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,34 @@ # HAR file management library -A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. +A Golang library for parsing and managing HAR (HTTP Archive) files. Provides easy-to-use structs and functions for loading, inspecting, and manipulating HAR files, making HTTP traffic analysis and debugging simpler. This library is designed to be used with the following libraries: -- [X] Check for case sensitivity in HeaderOrder keys -- [ ] Add Remote Address (proxy used) -- [ ] Fix bodySize +* [`bogdanfin/tls-client`](https://github.com/bogdanfinn/tls-client) +* The standard **`net/http`** library +* Other **custom request/response structures** + +## Purpose + +* Provide a **complete and persistent history of requests and responses**. +* Facilitate **monitoring, tracking, and debugging** of request systems. +* Offer a **1:1 equivalent of HAR exports** produced by tools like **Charles Proxy** or **Proxyman** during SSL proxying. + +## Functional Requirements / Specifications + +* **Multi-source compatibility:** Capture and generate HAR files from **tls-client**, **net/http**, or any **custom structs**. +* **Maximum fidelity to the HAR standard:** Match as closely as possible the format and content produced by **Charles Proxy** (as a reference for export quality). +* **Strict header order preservation:** Maintain the exact order of headers as defined by the TLS layer (which Go does not guarantee by default—requires specific handling). +* **Simplified API:** + * Create a **HAR session** via a dedicated function. + * Add a **request** to the HAR. + * Add the **corresponding response** via a complementary function. + * Explicit handling of **requests without responses** (timeouts, cancellations, network errors). +* **Additional fields beyond the HAR standard:** + * **IP address** used during the TLS connection. + * **Session ID** for session monitoring and tracking. + +## Bonus / Potential Extensions + +* **Monitoring integration:** Native export compatible with **Prometheus / Grafana**, or extract metrics directly from HAR files for real-time visualization. +* **Advanced historization:** Automatic HAR file storage in an **S3 bucket**, with associated **metadata/tags**: + * Final request status: **success**, **failure**, **timeout**, **HTTP 5xx**, etc. + * **Category / service / user tagging** for easier identification. diff --git a/example/go.mod b/example/go.mod index c6210d7..71a09b6 100644 --- a/example/go.mod +++ b/example/go.mod @@ -1,24 +1,30 @@ module github.com/Mathious6/example-harkit -go 1.24.2 +go 1.24.3 require ( github.com/Mathious6/harkit v0.1.1 - github.com/bogdanfinn/fhttp v0.5.36 - github.com/bogdanfinn/tls-client v1.9.1 + github.com/Mathious6/httpkit v0.0.0-20250723215844-55768b698fa7 + github.com/bogdanfinn/fhttp v0.6.0 ) require ( + github.com/Dharmey747/quic-go-utls v1.0.3-utls // indirect + github.com/Mathious6/platekit v1.0.0 // indirect github.com/andybalholm/brotli v1.1.1 // indirect - github.com/bogdanfinn/utls v1.6.5 // indirect + github.com/bogdanfinn/utls v1.7.3-barnius // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/quic-go/quic-go v0.50.1 // indirect + github.com/quic-go/qpack v0.5.1 // indirect github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect + go.uber.org/mock v0.5.0 // indirect golang.org/x/crypto v0.37.0 // indirect + golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.39.0 // indirect + golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect + golang.org/x/tools v0.22.0 // indirect ) replace github.com/Mathious6/harkit => ../ diff --git a/example/go.sum b/example/go.sum index c893b0c..ca72a52 100644 --- a/example/go.sum +++ b/example/go.sum @@ -1,11 +1,15 @@ +github.com/Dharmey747/quic-go-utls v1.0.3-utls h1:wqvk69LgFwT6AtTW14ASpRIJkvCaH21dX/U5ov5STr0= +github.com/Dharmey747/quic-go-utls v1.0.3-utls/go.mod h1:lgQoyZzST8vJJQ84eF9Xi2xJJnujoiNk0FGFEyQonG8= +github.com/Mathious6/httpkit v0.0.0-20250723215844-55768b698fa7 h1:hT8GzwGDjNXxOcaZiYI0gcdwo3h4ylDht5JOAzKAZkA= +github.com/Mathious6/httpkit v0.0.0-20250723215844-55768b698fa7/go.mod h1:ZtgfNJPBEngOGIG/rC30Ei1bWb52doOa1qHeX1NTNNg= +github.com/Mathious6/platekit v1.0.0 h1:XoS0C1KiMKZem8mvD2VE1fvwS/1DT9Vc9u/dVdV69zY= +github.com/Mathious6/platekit v1.0.0/go.mod h1:bMXoVaS2ziTocqDm4rzMFuC8eiUFYpUIa+hXHymt8i8= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/bogdanfinn/fhttp v0.5.36 h1:t1sO/EkO4K40QD/Ti8f6t80leZIdh2AaeLfN7dMvjH8= -github.com/bogdanfinn/fhttp v0.5.36/go.mod h1:BlcawVfXJ4uhk5yyNGOOY2bwo8UmMi6ccMszP1KGLkU= -github.com/bogdanfinn/tls-client v1.9.1 h1:Br0WkKL+/7Q9FSNM1zBMdlYXW8bm+XXGMn9iyb9a/7Y= -github.com/bogdanfinn/tls-client v1.9.1/go.mod h1:ehNITC7JBFeh6S7QNWtfD+PBKm0RsqvizAyyij2d/6g= -github.com/bogdanfinn/utls v1.6.5 h1:rVMQvhyN3zodLxKFWMRLt19INGBCZ/OM2/vBWPNIt1w= -github.com/bogdanfinn/utls v1.6.5/go.mod h1:czcHxHGsc1q9NjgWSeSinQZzn6MR76zUmGVIGanSXO0= +github.com/bogdanfinn/fhttp v0.6.0 h1:24JoDnE43tq3RdK99K1M5mxa2JyntKr6WcDsy1KdA0o= +github.com/bogdanfinn/fhttp v0.6.0/go.mod h1:ZR1hRfxsOd/j/C8RnwyNXA90DxkrHB3Y1nuCD1YlbdI= +github.com/bogdanfinn/utls v1.7.3-barnius h1:2p9riIoGHI85eVDebhHm58qLokyJ8bFEn26wg24S1uU= +github.com/bogdanfinn/utls v1.7.3-barnius/go.mod h1:SUn0CoHGVp/akGNuaqh99yvovu64PCP2LbWd3Z/Laic= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,23 +18,29 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q= -github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc= github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/example/main.go b/example/main.go index a716e6b..0e42857 100644 --- a/example/main.go +++ b/example/main.go @@ -8,8 +8,8 @@ import ( "time" "github.com/Mathious6/harkit/harhandler" + "github.com/Mathious6/httpkit" http "github.com/bogdanfinn/fhttp" - tls_client "github.com/bogdanfinn/tls-client" ) const ( @@ -21,32 +21,32 @@ const ( func main() { handler := harhandler.NewHandler() - opts := []tls_client.HttpClientOption{ - tls_client.WithCookieJar(tls_client.NewCookieJar()), - tls_client.WithNotFollowRedirects(), + opts := []httpkit.HttpClientOption{ + httpkit.WithCookieJar(httpkit.NewCookieJar()), + httpkit.WithNotFollowRedirects(), } if isProxyRunning(net.JoinHostPort(PROXY_HOST, PROXY_PORT), 100*time.Millisecond) { - opts = append(opts, tls_client.WithCharlesProxy(PROXY_HOST, PROXY_PORT)) + opts = append(opts, httpkit.WithCharlesProxy(PROXY_HOST, PROXY_PORT)) fmt.Println("Using Charles proxy.") } else { fmt.Println("Charles proxy not running, using direct connection.") } - client, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), opts...) + client, err := httpkit.NewHttpClient(httpkit.NewNoopLogger(), opts...) if err != nil { panic(err) } sendGetRequestWithQueryParams(handler, client) - sendGetRequestWithSetCookies(handler, client) - sendPostRequestWithForm(handler, client) - sendPostRequestWithJSON(handler, client) + // sendGetRequestWithSetCookies(handler, client) + // sendPostRequestWithForm(handler, client) + // sendPostRequestWithJSON(handler, client) handler.Save("example.har") } -func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client tls_client.HttpClient) { +func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/get?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") req.Header.Add("Host", "httpbin.org") @@ -71,7 +71,7 @@ func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client tls_cl fmt.Println("Parameters sent.") } -func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client tls_client.HttpClient) { +func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/cookies/set?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") req.Header.Add("Host", "httpbin.org") @@ -96,7 +96,7 @@ func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client tls_cli fmt.Println("Cookies set.") } -func sendPostRequestWithForm(handler *harhandler.HARHandler, client tls_client.HttpClient) { +func sendPostRequestWithForm(handler *harhandler.HARHandler, client httpkit.HttpClient) { form := url.Values{} form.Set("name", "Pierre") form.Set("role", "developer") @@ -132,7 +132,7 @@ func sendPostRequestWithForm(handler *harhandler.HARHandler, client tls_client.H fmt.Println("Form URL-encoded request sent.") } -func sendPostRequestWithJSON(handler *harhandler.HARHandler, client tls_client.HttpClient) { +func sendPostRequestWithJSON(handler *harhandler.HARHandler, client httpkit.HttpClient) { jsonBody := `{"name":"Pierre","role":"developer"}` body := strings.NewReader(jsonBody) From ebd9d57c55dc4f6211c99a8cff1914e668a128d7 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 24 Jul 2025 12:11:26 +0000 Subject: [PATCH 47/60] refactor: streamline cookie and request handling by introducing helper functions for expiration formatting and query/post data extraction --- converter/common.go | 24 ++++++++++++------------ converter/request.go | 20 +++++++++++++------- converter/response.go | 4 ++-- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/converter/common.go b/converter/common.go index 8b2509a..6567f3d 100644 --- a/converter/common.go +++ b/converter/common.go @@ -19,30 +19,30 @@ const ( func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { harCookies := make([]*harfile.Cookie, len(cookies)) - - for i, cookie := range cookies { - var expires string - if !cookie.Expires.IsZero() { - expires = cookie.Expires.Format(time.RFC3339Nano) - } - - harCookies[i] = &harfile.Cookie{ + for index, cookie := range cookies { + harCookies[index] = &harfile.Cookie{ Name: cookie.Name, Value: cookie.Value, Path: cookie.Path, Domain: cookie.Domain, - Expires: expires, + Expires: formatExpires(cookie.Expires), HTTPOnly: cookie.HttpOnly, Secure: cookie.Secure, } } - return harCookies } +func formatExpires(expires time.Time) string { + if expires.IsZero() { + return "" + } + return expires.Format(time.RFC3339Nano) +} + func convertHeaders(header http.Header, contentLength int64) []*harfile.NVPair { - // By default, client adds Content-Length header later on, so we need to add it here - // We clone the header to avoid modifying the original one to avoid side effects + // By default, client adds Content-Length header later on, so we need to add it here. + // We clone the header to avoid modifying the original one to avoid side effects. clonedHeader := header.Clone() if contentLength > 0 && clonedHeader.Get(ContentLengthKey) == "" { clonedHeader.Set(ContentLengthKey, fmt.Sprintf("%d", contentLength)) diff --git a/converter/request.go b/converter/request.go index cf3277d..8bb9c7a 100644 --- a/converter/request.go +++ b/converter/request.go @@ -11,6 +11,12 @@ import ( http "github.com/bogdanfinn/fhttp" ) +const ( + applicationXWWWFormURLEncoded = "application/x-www-form-urlencoded" + multipartFormData = "multipart/form-data" + maxMultipartFormDataSize = 32 << 20 // 32 MB limit +) + func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { if req == nil { return nil, errors.New("request cannot be nil") @@ -18,7 +24,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { headers := convertHeaders(req.Header, req.ContentLength) - postData, err := extractPostData(req) + postData, err := extractRequestPostData(req) if err != nil { return nil, err } @@ -29,14 +35,14 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { HTTPVersion: req.Proto, Cookies: convertCookies(req.Cookies()), Headers: headers, - QueryString: convertQueryParams(req.URL), + QueryString: convertRequestQueryParams(req.URL), PostData: postData, HeadersSize: computeRequestHeadersSize(req, headers), BodySize: req.ContentLength, }, nil } -func convertQueryParams(u *url.URL) []*harfile.NVPair { +func convertRequestQueryParams(u *url.URL) []*harfile.NVPair { result := make([]*harfile.NVPair, 0) for key, values := range u.Query() { @@ -48,7 +54,7 @@ func convertQueryParams(u *url.URL) []*harfile.NVPair { return result } -func extractPostData(req *http.Request) (*harfile.PostData, error) { +func extractRequestPostData(req *http.Request) (*harfile.PostData, error) { if req.Body == nil || req.ContentLength == 0 { return nil, nil } @@ -63,7 +69,7 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { mimeType := req.Header.Get(ContentTypeKey) postData := &harfile.PostData{MimeType: mimeType} - if strings.HasPrefix(mimeType, "application/x-www-form-urlencoded") { + if strings.HasPrefix(mimeType, applicationXWWWFormURLEncoded) { text := string(buf) pairs := strings.SplitSeq(text, "&") @@ -78,8 +84,8 @@ func extractPostData(req *http.Request) (*harfile.PostData, error) { return postData, nil } - if strings.HasPrefix(mimeType, "multipart/form-data") { - err := req.ParseMultipartForm(32 << 20) // 32 MB limit + if strings.HasPrefix(mimeType, multipartFormData) { + err := req.ParseMultipartForm(maxMultipartFormDataSize) if err != nil { return nil, err } diff --git a/converter/response.go b/converter/response.go index a4439b0..c758eb4 100644 --- a/converter/response.go +++ b/converter/response.go @@ -14,7 +14,7 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { return nil, errors.New("response cannot be nil") } - content, err := buildContent(resp) + content, err := buildResponseContent(resp) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func locateRedirectURL(resp *http.Response) *string { return nil } -func buildContent(resp *http.Response) (*harfile.Content, error) { +func buildResponseContent(resp *http.Response) (*harfile.Content, error) { buf, err := io.ReadAll(resp.Body) if err != nil { return nil, err From d980b91567f1bc14f9dee2dae464491b8b326517 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 24 Jul 2025 13:11:38 +0000 Subject: [PATCH 48/60] refactor: simplify HAR entry handling by removing EntryBuilder and integrating entry creation directly into HARHandler methods --- example/main.go | 44 ++++------- harhandler/entry_builder.go | 115 --------------------------- harhandler/har_handler.go | 126 ++++++++++++++++++++++++++---- harhandler/har_handler_options.go | 9 +++ 4 files changed, 135 insertions(+), 159 deletions(-) delete mode 100644 harhandler/entry_builder.go create mode 100644 harhandler/har_handler_options.go diff --git a/example/main.go b/example/main.go index 0e42857..be549f5 100644 --- a/example/main.go +++ b/example/main.go @@ -19,8 +19,6 @@ const ( ) func main() { - handler := harhandler.NewHandler() - opts := []httpkit.HttpClientOption{ httpkit.WithCookieJar(httpkit.NewCookieJar()), httpkit.WithNotFollowRedirects(), @@ -38,15 +36,15 @@ func main() { panic(err) } - sendGetRequestWithQueryParams(handler, client) - // sendGetRequestWithSetCookies(handler, client) - // sendPostRequestWithForm(handler, client) - // sendPostRequestWithJSON(handler, client) + sendGetRequestWithQueryParams(client) + sendGetRequestWithSetCookies(client) + sendPostRequestWithForm(client) + sendPostRequestWithJSON(client) - handler.Save("example.har") + harhandler.Export(client.GetFlowId(), "example.har") } -func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client httpkit.HttpClient) { +func sendGetRequestWithQueryParams(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/get?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") req.Header.Add("Host", "httpbin.org") @@ -58,20 +56,18 @@ func sendGetRequestWithQueryParams(handler *harhandler.HARHandler, client httpki req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() + sentAt := time.Now() resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddEntry(req, resp) - - handler.AddEntry(entry) + harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) fmt.Println("Parameters sent.") } -func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client httpkit.HttpClient) { +func sendGetRequestWithSetCookies(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/cookies/set?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") req.Header.Add("Host", "httpbin.org") @@ -83,20 +79,18 @@ func sendGetRequestWithSetCookies(handler *harhandler.HARHandler, client httpkit req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() + sentAt := time.Now() resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddEntry(req, resp) - - handler.AddEntry(entry) + harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) fmt.Println("Cookies set.") } -func sendPostRequestWithForm(handler *harhandler.HARHandler, client httpkit.HttpClient) { +func sendPostRequestWithForm(client httpkit.HttpClient) { form := url.Values{} form.Set("name", "Pierre") form.Set("role", "developer") @@ -119,20 +113,18 @@ func sendPostRequestWithForm(handler *harhandler.HARHandler, client httpkit.Http req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() + sentAt := time.Now() resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddEntry(req, resp) - - handler.AddEntry(entry) + harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) fmt.Println("Form URL-encoded request sent.") } -func sendPostRequestWithJSON(handler *harhandler.HARHandler, client httpkit.HttpClient) { +func sendPostRequestWithJSON(client httpkit.HttpClient) { jsonBody := `{"name":"Pierre","role":"developer"}` body := strings.NewReader(jsonBody) @@ -153,15 +145,13 @@ func sendPostRequestWithJSON(handler *harhandler.HARHandler, client httpkit.Http req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") - entry := harhandler.NewEntry() + sentAt := time.Now() resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() - _ = entry.AddEntry(req, resp) - - handler.AddEntry(entry) + harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) fmt.Println("JSON request sent.") } diff --git a/harhandler/entry_builder.go b/harhandler/entry_builder.go deleted file mode 100644 index 98e6cf1..0000000 --- a/harhandler/entry_builder.go +++ /dev/null @@ -1,115 +0,0 @@ -// Package harhandler provides functionality to build HAR entries from bogdanfinn/fhttp requests -// and responses. It allows deferred HAR construction after the request is executed, ensuring -// accurate extraction of request and response data, including body content, headers, cookies, -// and timing information. -package harhandler - -import ( - "bytes" - "context" - "io" - "net" - "net/url" - "time" - - "github.com/Mathious6/harkit/converter" - "github.com/Mathious6/harkit/harfile" - http "github.com/bogdanfinn/fhttp" -) - -// EntryBuilder builds a HAR entry from a bogdanfinn/fhttp request and response. -// It encapsulates all relevant data including timings, headers, cookies, and body content. -type EntryBuilder struct { - entry *harfile.Entry -} - -// NewEntry initializes an empty HAR entry with default fields and timing placeholders. -// Use AddEntry to attach the request and response data once the request is completed. -func NewEntry() *EntryBuilder { - return &EntryBuilder{ - entry: &harfile.Entry{ - StartedDateTime: time.Now(), - Time: -1, - Cache: &harfile.Cache{}, - Timings: &harfile.Timings{ - Send: -1, - Wait: -1, - Receive: -1, - }, - }, - } -} - -// AddEntry populates the HAR entry with the given HTTP request and response. -// It clones and restores the request body to prevent side effects. -// This method should be called after the HTTP request is executed to avoid blocking. -func (b *EntryBuilder) AddEntry(req *http.Request, resp *http.Response) error { - b.entry.Timings.Receive = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) - - clonedReq, err := cloneRequestPreserveBody(req) - if err != nil { - return err - } - - harReq, err := converter.FromHTTPRequest(clonedReq) - if err != nil { - return err - } - harResp, err := converter.FromHTTPResponse(resp) - if err != nil { - return err - } - - b.entry.Request = harReq - b.entry.Response = harResp - - b.entry.Timings.Wait = float64(time.Since(b.entry.StartedDateTime).Milliseconds()) - b.entry.Timings.Receive - b.entry.Time = b.entry.Timings.Total() - - return nil -} - -// Build finalizes and returns the constructed HAR entry. -// If resolveIP is true, a DNS resolution will be performed on the request host. -func (b *EntryBuilder) Build(resolveIP bool) *harfile.Entry { - if resolveIP && b.entry.Request != nil { - b.entry.ServerIPAddress = resolveServerIPAddress(b.entry.Request.URL) - } - return b.entry -} - -// resolveServerIPAddress performs a DNS lookup on the given URL and returns the first resolved -// IP address as a string. Returns an empty string on failure. This is a blocking operation. -func resolveServerIPAddress(rawURL string) string { - parsedURL, err := url.Parse(rawURL) - if err != nil { - return "" - } - - ipAddrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), parsedURL.Hostname()) - if err != nil || len(ipAddrs) == 0 { - return "" - } - return ipAddrs[0].IP.String() -} - -// cloneRequestPreserveBody clones an HTTP request and preserves its body by buffering the content -// into memory. Both the original and the cloned request will be reset with a fresh body reader, -// allowing for safe reuse without data loss. -func cloneRequestPreserveBody(req *http.Request) (*http.Request, error) { - if req.Body == nil { - return req.Clone(req.Context()), nil - } - - buf, err := io.ReadAll(req.Body) - if err != nil { - return nil, err - } - defer req.Body.Close() - req.Body = io.NopCloser(bytes.NewReader(buf)) - - clonedReq := req.Clone(req.Context()) - clonedReq.Body = io.NopCloser(bytes.NewReader(buf)) - - return clonedReq, nil -} diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index 9e42c83..1d5efcc 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -1,11 +1,25 @@ package harhandler import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/url" + "sync" + "time" + "github.com/Mathious6/harkit" + "github.com/Mathious6/harkit/converter" "github.com/Mathious6/harkit/harfile" + http "github.com/bogdanfinn/fhttp" ) -type HandlerOption func(*HARHandler) +var ( + globalHarStorage = make(map[string]*HARHandler) + globalHarStorageMutex = sync.Mutex{} +) type HARHandler struct { log *harfile.Log @@ -13,37 +27,115 @@ type HARHandler struct { resolveIPAddress bool } -func WithServerIPAddress() HandlerOption { - return func(h *HARHandler) { - h.resolveIPAddress = true +func NewHandler(flowId string, opts ...HandlerOption) *HARHandler { + globalHarStorageMutex.Lock() + defer globalHarStorageMutex.Unlock() + + if handler, exists := globalHarStorage[flowId]; exists { + for _, opt := range opts { + opt(handler) + } + return handler } -} -func NewHandler(opts ...HandlerOption) *HARHandler { - h := &HARHandler{ + handler := &HARHandler{ log: &harfile.Log{ Version: "1.2", Creator: &harfile.Creator{ - Name: "harkit", - Version: harkit.Version, + Name: flowId, + Version: fmt.Sprintf("harkit-%s", harkit.Version), }, Entries: []*harfile.Entry{}, }, } for _, opt := range opts { - opt(h) + opt(handler) } - - return h + globalHarStorage[flowId] = handler + return handler } -func (h *HARHandler) AddEntry(builder *EntryBuilder) { - entry := builder.Build(h.resolveIPAddress) - h.log.Entries = append(h.log.Entries, entry) +func (h *HARHandler) Build(sentAt time.Time, req *http.Request, resp *http.Response) error { + timingsReceive := float64(time.Since(sentAt).Milliseconds()) + + clonedReq, err := cloneRequestPreserveBody(req) + if err != nil { + return err + } + harReq, err := converter.FromHTTPRequest(clonedReq) + if err != nil { + return err + } + + harResp, err := converter.FromHTTPResponse(resp) + if err != nil { + return err + } + + timings := &harfile.Timings{ + Send: -1, + Wait: float64(time.Since(sentAt).Milliseconds()) - timingsReceive, + Receive: timingsReceive, + } + + h.log.Entries = append(h.log.Entries, &harfile.Entry{ + StartedDateTime: sentAt, + Time: timings.Total(), + Request: harReq, + Response: harResp, + Cache: &harfile.Cache{}, + Timings: timings, + ServerIPAddress: resolveServerIPAddress(h.resolveIPAddress, harReq.URL), + }) + + return nil } -func (h *HARHandler) Save(filename string) error { - har := &harfile.HAR{Log: h.log} +func Export(flowId string, filename string) error { + globalHarStorageMutex.Lock() + defer globalHarStorageMutex.Unlock() + + har := &harfile.HAR{Log: globalHarStorage[flowId].log} return har.Save(filename) } + +// resolveServerIPAddress performs a DNS lookup on the given URL and returns the first resolved +// IP address as a string. Returns an empty string on failure. This is a blocking operation. +func resolveServerIPAddress(resolve bool, rawURL string) string { + if !resolve { + return "" + } + + parsedURL, err := url.Parse(rawURL) + if err != nil { + return "" + } + + ipAddrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), parsedURL.Hostname()) + if err != nil || len(ipAddrs) == 0 { + return "" + } + return ipAddrs[0].IP.String() +} + +// cloneRequestPreserveBody clones an HTTP request and preserves its body by buffering the content +// into memory. Both the original and the cloned request will be reset with a fresh body reader, +// allowing for safe reuse without data loss. +func cloneRequestPreserveBody(req *http.Request) (*http.Request, error) { + if req.Body == nil { + return req.Clone(req.Context()), nil + } + + buf, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + defer req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(buf)) + + clonedReq := req.Clone(req.Context()) + clonedReq.Body = io.NopCloser(bytes.NewReader(buf)) + + return clonedReq, nil +} diff --git a/harhandler/har_handler_options.go b/harhandler/har_handler_options.go new file mode 100644 index 0000000..495a78c --- /dev/null +++ b/harhandler/har_handler_options.go @@ -0,0 +1,9 @@ +package harhandler + +type HandlerOption func(*HARHandler) + +func WithServerIPAddress() HandlerOption { + return func(h *HARHandler) { + h.resolveIPAddress = true + } +} From bffaefdac8eea63ea574249c91a451dab9e191a5 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Thu, 24 Jul 2025 13:32:56 +0000 Subject: [PATCH 49/60] refactor: replace NewHandler with AddEntry in example functions to streamline HAR entry creation --- example/main.go | 8 ++-- harhandler/har_handler.go | 77 ++++++++++++++++++++++++++------------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/example/main.go b/example/main.go index be549f5..509d412 100644 --- a/example/main.go +++ b/example/main.go @@ -62,7 +62,7 @@ func sendGetRequestWithQueryParams(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) fmt.Println("Parameters sent.") } @@ -85,7 +85,7 @@ func sendGetRequestWithSetCookies(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) fmt.Println("Cookies set.") } @@ -119,7 +119,7 @@ func sendPostRequestWithForm(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) fmt.Println("Form URL-encoded request sent.") } @@ -151,7 +151,7 @@ func sendPostRequestWithJSON(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.NewHandler(client.GetFlowId()).Build(sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) fmt.Println("JSON request sent.") } diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index 1d5efcc..da5d045 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -22,41 +22,74 @@ var ( ) type HARHandler struct { - log *harfile.Log + har *harfile.HAR resolveIPAddress bool } -func NewHandler(flowId string, opts ...HandlerOption) *HARHandler { +func CreateHandler(flowID string, opts ...HandlerOption) (*HARHandler, error) { globalHarStorageMutex.Lock() defer globalHarStorageMutex.Unlock() + if _, ok := globalHarStorage[flowID]; ok { + return nil, fmt.Errorf("handler %q already exists", flowID) + } + handler := newHARHandler(flowID, opts...) + globalHarStorage[flowID] = handler + return handler, nil +} + +func GetHandler(flowID string) (*HARHandler, error) { + globalHarStorageMutex.Lock() + defer globalHarStorageMutex.Unlock() + handler, exists := globalHarStorage[flowID] + if !exists { + return nil, fmt.Errorf("handler %q not found", flowID) + } + return handler, nil +} - if handler, exists := globalHarStorage[flowId]; exists { +func GetOrCreateHandler(flowID string, opts ...HandlerOption) *HARHandler { + if h, err := GetHandler(flowID); err == nil { for _, opt := range opts { - opt(handler) + opt(h) } - return handler + return h } + h, _ := CreateHandler(flowID, opts...) + return h +} - handler := &HARHandler{ - log: &harfile.Log{ - Version: "1.2", - Creator: &harfile.Creator{ - Name: flowId, - Version: fmt.Sprintf("harkit-%s", harkit.Version), +func newHARHandler(flowID string, opts ...HandlerOption) *HARHandler { + h := &HARHandler{ + har: &harfile.HAR{ + Log: &harfile.Log{ + Version: "1.2", + Creator: &harfile.Creator{ + Name: flowID, + Version: fmt.Sprintf("harkit-%s", harkit.Version), + }, + Entries: []*harfile.Entry{}, }, - Entries: []*harfile.Entry{}, }, } - for _, opt := range opts { - opt(handler) + opt(h) } - globalHarStorage[flowId] = handler - return handler + return h +} + +func AddEntry(flowId string, sentAt time.Time, req *http.Request, resp *http.Response) error { + handler := GetOrCreateHandler(flowId) + return handler.AddEntry(sentAt, req, resp) } -func (h *HARHandler) Build(sentAt time.Time, req *http.Request, resp *http.Response) error { +func Export(flowId string, filename string) error { + handler := GetOrCreateHandler(flowId) + delete(globalHarStorage, flowId) + return handler.har.Save(filename) +} + +func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Response) error { timingsReceive := float64(time.Since(sentAt).Milliseconds()) clonedReq, err := cloneRequestPreserveBody(req) @@ -79,7 +112,7 @@ func (h *HARHandler) Build(sentAt time.Time, req *http.Request, resp *http.Respo Receive: timingsReceive, } - h.log.Entries = append(h.log.Entries, &harfile.Entry{ + h.har.Log.Entries = append(h.har.Log.Entries, &harfile.Entry{ StartedDateTime: sentAt, Time: timings.Total(), Request: harReq, @@ -92,14 +125,6 @@ func (h *HARHandler) Build(sentAt time.Time, req *http.Request, resp *http.Respo return nil } -func Export(flowId string, filename string) error { - globalHarStorageMutex.Lock() - defer globalHarStorageMutex.Unlock() - - har := &harfile.HAR{Log: globalHarStorage[flowId].log} - return har.Save(filename) -} - // resolveServerIPAddress performs a DNS lookup on the given URL and returns the first resolved // IP address as a string. Returns an empty string on failure. This is a blocking operation. func resolveServerIPAddress(resolve bool, rawURL string) string { From f6b1e57c26546688d3ab2f027b048769ae4f4e16 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Fri, 25 Jul 2025 16:07:21 +0000 Subject: [PATCH 50/60] feat: add HARVersion constant and enhance documentation for HARHandler and related functions --- harfile/har.go | 2 ++ harhandler/har_handler.go | 41 +++++++++++++++++-------------- harhandler/har_handler_options.go | 2 ++ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/harfile/har.go b/harfile/har.go index 95135bb..514ecfa 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -9,6 +9,8 @@ import ( "time" ) +const HARVersion = "1.2" // HARVersion is the version of the HAR format + // HAR parent container for log. type HAR struct { Log *Log `json:"log"` // Log represents the root of exported data. diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index da5d045..ef39fb8 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -17,20 +17,21 @@ import ( ) var ( - globalHarStorage = make(map[string]*HARHandler) - globalHarStorageMutex = sync.Mutex{} + globalHarStorage = make(map[string]*HARHandler) // globalHarStorage is storing all the HARHandlers for all the flows + globalHarStorageMutex = sync.Mutex{} // globalHarStorageMutex is used to synchronize access to the globalHarStorage map ) +// HARHandler is the main struct that stores the HAR data for a flow type HARHandler struct { - har *harfile.HAR - - resolveIPAddress bool + har *harfile.HAR // har is the HAR data for the flow + resolveIPAddress bool // resolveIPAddress is a flag to resolve the IP address of the server } +// CreateHandler creates a new HARHandler for a flow with the given flowID func CreateHandler(flowID string, opts ...HandlerOption) (*HARHandler, error) { globalHarStorageMutex.Lock() defer globalHarStorageMutex.Unlock() - if _, ok := globalHarStorage[flowID]; ok { + if _, exists := globalHarStorage[flowID]; exists { return nil, fmt.Errorf("handler %q already exists", flowID) } handler := newHARHandler(flowID, opts...) @@ -38,6 +39,7 @@ func CreateHandler(flowID string, opts ...HandlerOption) (*HARHandler, error) { return handler, nil } +// GetHandler gets the HARHandler for a flow with the given flowID func GetHandler(flowID string) (*HARHandler, error) { globalHarStorageMutex.Lock() defer globalHarStorageMutex.Unlock() @@ -48,22 +50,24 @@ func GetHandler(flowID string) (*HARHandler, error) { return handler, nil } +// GetOrCreateHandler gets the HARHandler for a flow with the given flowID, if it doesn't exist, it creates a new one func GetOrCreateHandler(flowID string, opts ...HandlerOption) *HARHandler { - if h, err := GetHandler(flowID); err == nil { + if handler, err := GetHandler(flowID); err == nil { for _, opt := range opts { - opt(h) + opt(handler) } - return h + return handler } - h, _ := CreateHandler(flowID, opts...) - return h + handler, _ := CreateHandler(flowID, opts...) + return handler } +// newHARHandler creates a new HARHandler for a flow with the given flowID and applies the given options func newHARHandler(flowID string, opts ...HandlerOption) *HARHandler { h := &HARHandler{ har: &harfile.HAR{ Log: &harfile.Log{ - Version: "1.2", + Version: harfile.HARVersion, Creator: &harfile.Creator{ Name: flowID, Version: fmt.Sprintf("harkit-%s", harkit.Version), @@ -78,17 +82,18 @@ func newHARHandler(flowID string, opts ...HandlerOption) *HARHandler { return h } +// AddEntry adds a new entry to the HARHandler for a flow with the given flowID, sentAt, request, and response func AddEntry(flowId string, sentAt time.Time, req *http.Request, resp *http.Response) error { - handler := GetOrCreateHandler(flowId) - return handler.AddEntry(sentAt, req, resp) + return GetOrCreateHandler(flowId).AddEntry(sentAt, req, resp) } -func Export(flowId string, filename string) error { - handler := GetOrCreateHandler(flowId) +// Export exports the HAR data for a flow with the given flowID and filename +func Export(flowId, filename string) error { delete(globalHarStorage, flowId) - return handler.har.Save(filename) + return GetOrCreateHandler(flowId).har.Save(filename) } +// AddEntry adds a new entry to the HARHandler for a flow with the given sentAt, request, and response func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Response) error { timingsReceive := float64(time.Since(sentAt).Milliseconds()) @@ -117,7 +122,7 @@ func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Re Time: timings.Total(), Request: harReq, Response: harResp, - Cache: &harfile.Cache{}, + Cache: nil, Timings: timings, ServerIPAddress: resolveServerIPAddress(h.resolveIPAddress, harReq.URL), }) diff --git a/harhandler/har_handler_options.go b/harhandler/har_handler_options.go index 495a78c..e8f9c64 100644 --- a/harhandler/har_handler_options.go +++ b/harhandler/har_handler_options.go @@ -1,7 +1,9 @@ package harhandler +// HandlerOption is a function that can be used to configure a HARHandler type HandlerOption func(*HARHandler) +// WithServerIPAddress is a function that can be used to configure a HARHandler to resolve the IP address of the server func WithServerIPAddress() HandlerOption { return func(h *HARHandler) { h.resolveIPAddress = true From 6f1eb75d9ed3ff9fc2a39890a2755a7b779f9a6b Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:05:12 +0000 Subject: [PATCH 51/60] chore: update version to v1.0.0 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index d4548c2..9d95010 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package harkit -var Version = "v0.2.0" +var Version = "v1.0.0" From 0b896f078039f11f8ba9285dd8879e5e9761343c Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:05:32 +0000 Subject: [PATCH 52/60] chore: update devcontainer.json to include note for future Go image version upgrade (1.25) --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cf56bda..b257b01 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Go", - "image": "mcr.microsoft.com/devcontainers/go:1.24", + "image": "mcr.microsoft.com/devcontainers/go:1.24", // Switch to 1.25 when it's released "remoteUser": "vscode", "shutdownAction": "stopContainer", "initializeCommand": "./.devcontainer/initialize.sh", From 7d3d45dd4b2269f814261a651e4e37e1ac21566e Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:05:52 +0000 Subject: [PATCH 53/60] refactor: standardize HTTP version handling by introducing DefaultRequestHTTPVersion constant and updating request conversion logic --- converter/common.go | 11 ++++++----- converter/request.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/converter/common.go b/converter/common.go index 6567f3d..1b4178e 100644 --- a/converter/common.go +++ b/converter/common.go @@ -10,11 +10,12 @@ import ( ) const ( - ContentLengthKey = "Content-Length" - ContentTypeKey = "Content-Type" - CookieKey = "Cookie" - SetCookieKey = "Set-Cookie" - LocationKey = "Location" + DefaultRequestHTTPVersion = "HTTP/2.0" + ContentLengthKey = "Content-Length" + ContentTypeKey = "Content-Type" + CookieKey = "Cookie" + SetCookieKey = "Set-Cookie" + LocationKey = "Location" ) func convertCookies(cookies []*http.Cookie) []*harfile.Cookie { diff --git a/converter/request.go b/converter/request.go index 8bb9c7a..df764f4 100644 --- a/converter/request.go +++ b/converter/request.go @@ -32,7 +32,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { return &harfile.Request{ Method: req.Method, URL: req.URL.String(), - HTTPVersion: req.Proto, + HTTPVersion: DefaultRequestHTTPVersion, // WARNING: req.Proto is not always accurate Cookies: convertCookies(req.Cookies()), Headers: headers, QueryString: convertRequestQueryParams(req.URL), From 0ea7cfb078751be62d8e982304024a44a1070f8f Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:06:24 +0000 Subject: [PATCH 54/60] feat: enhance HAR time handling by introducing HARTime type and updating time fields in Page and Entry structures --- harfile/har.go | 27 ++++++++++++++++++++------- harhandler/har_handler.go | 5 +++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/harfile/har.go b/harfile/har.go index 514ecfa..34392bd 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -9,7 +9,17 @@ import ( "time" ) -const HARVersion = "1.2" // HARVersion is the version of the HAR format +const ( + HARVersion = "1.2" // HARVersion is the version of the HAR format + HARTimeLayout = "2006-01-02T15:04:05.000-07:00" // REF: "Mon Jan 2 15:04:05 MST 2006" +) + +// HARTime is a wrapper around time.Time that formats the time in the HAR format. +type HARTime time.Time + +func (ht HARTime) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Time(ht).Format(HARTimeLayout) + `"`), nil +} // HAR parent container for log. type HAR struct { @@ -42,7 +52,7 @@ type Browser struct { // Pages represents list of exported pages. type Page struct { - StartedDateTime time.Time `json:"startedDateTime"` // Date and time stamp for the beginning of the page load (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00). + StartedDateTime HARTime `json:"startedDateTime"` // Date and time stamp for the beginning of the page load (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD, e.g. 2009-07-24T19:20:30.45+01:00). ID string `json:"id"` // Unique identifier of a page within the [log]. Entries use it to refer the parent page. Title string `json:"title"` // Page title. PageTimings *PageTimings `json:"pageTimings"` // Detailed timing info about page load. @@ -63,7 +73,7 @@ type PageTimings struct { // for the import). type Entry struct { Pageref string `json:"pageref,omitempty"` // Reference to the parent page. Leave out this field if the application does not support grouping by pages. - StartedDateTime time.Time `json:"startedDateTime"` // Date and time stamp of the request start (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD). + StartedDateTime HARTime `json:"startedDateTime"` // Date and time stamp of the request start (ISO 8601 - YYYY-MM-DDThh:mm:ss.sTZD). Time float64 `json:"time"` // Total elapsed time of the request in milliseconds. This is the sum of all timings available in the timings object (i.e. not including -1 values) . Request *Request `json:"request"` // Detailed info about the request. Response *Response `json:"response"` // Detailed info about the response. @@ -191,16 +201,19 @@ func (t *Timings) Total() float64 { } // Save saves the HAR data to a file in JSON format under the specified filename. +// It uses the json.NewEncoder to encode without escaping HTML. func (h *HAR) Save(filename string) error { - jsonBytes, err := json.MarshalIndent(h, "", " ") + file, err := os.Create(filename) if err != nil { return err } + defer file.Close() - jsonBytes = append(jsonBytes, '\n') + enc := json.NewEncoder(file) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") - err = os.WriteFile(filename, jsonBytes, 0644) - if err != nil { + if err := enc.Encode(h); err != nil { return err } diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index ef39fb8..f66c636 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -89,8 +89,9 @@ func AddEntry(flowId string, sentAt time.Time, req *http.Request, resp *http.Res // Export exports the HAR data for a flow with the given flowID and filename func Export(flowId, filename string) error { + handler := GetOrCreateHandler(flowId) delete(globalHarStorage, flowId) - return GetOrCreateHandler(flowId).har.Save(filename) + return handler.har.Save(filename) } // AddEntry adds a new entry to the HARHandler for a flow with the given sentAt, request, and response @@ -118,7 +119,7 @@ func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Re } h.har.Log.Entries = append(h.har.Log.Entries, &harfile.Entry{ - StartedDateTime: sentAt, + StartedDateTime: harfile.HARTime(sentAt), Time: timings.Total(), Request: harReq, Response: harResp, From 1fa00538e8e506f2c29a2dd46ee44c6e28f962c9 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:26:59 +0000 Subject: [PATCH 55/60] refactor: enhance request handling by introducing protocol header management and updating header processing logic --- converter/request.go | 22 ++++++++++++++++++++-- example/main.go | 8 -------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/converter/request.go b/converter/request.go index df764f4..a7ceba0 100644 --- a/converter/request.go +++ b/converter/request.go @@ -22,6 +22,9 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { return nil, errors.New("request cannot be nil") } + reqProto := DefaultRequestHTTPVersion // WARNING: req.Proto is not always accurate + + protocolHeader := handleProtocolHeader(reqProto, req.Method, *req.URL) headers := convertHeaders(req.Header, req.ContentLength) postData, err := extractRequestPostData(req) @@ -32,9 +35,9 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { return &harfile.Request{ Method: req.Method, URL: req.URL.String(), - HTTPVersion: DefaultRequestHTTPVersion, // WARNING: req.Proto is not always accurate + HTTPVersion: reqProto, Cookies: convertCookies(req.Cookies()), - Headers: headers, + Headers: append(protocolHeader, headers...), QueryString: convertRequestQueryParams(req.URL), PostData: postData, HeadersSize: computeRequestHeadersSize(req, headers), @@ -42,6 +45,21 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { }, nil } +func handleProtocolHeader(proto string, method string, url url.URL) []*harfile.NVPair { + if proto == "HTTP/2.0" { + return []*harfile.NVPair{ + {Name: ":method", Value: method}, + {Name: ":authority", Value: url.Host}, + {Name: ":scheme", Value: url.Scheme}, + {Name: ":path", Value: url.RequestURI()}, + } + } else { + return []*harfile.NVPair{ + {Name: "Host", Value: url.Host}, + } + } +} + func convertRequestQueryParams(u *url.URL) []*harfile.NVPair { result := make([]*harfile.NVPair, 0) diff --git a/example/main.go b/example/main.go index 509d412..db82490 100644 --- a/example/main.go +++ b/example/main.go @@ -47,12 +47,10 @@ func main() { func sendGetRequestWithQueryParams(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/get?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") - req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") req.Header.Add(http.HeaderOrderKey, "accept") - req.Header.Add(http.HeaderOrderKey, "host") req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") @@ -70,12 +68,10 @@ func sendGetRequestWithQueryParams(client httpkit.HttpClient) { func sendGetRequestWithSetCookies(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodGet, URL+"/cookies/set?name=pierre&role=developer", nil) req.Header.Add("Accept", "*/*") - req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") req.Header.Add(http.HeaderOrderKey, "accept") - req.Header.Add(http.HeaderOrderKey, "host") req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") @@ -99,7 +95,6 @@ func sendPostRequestWithForm(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodPost, URL+"/post", body) req.Header.Add("Accept", "*/*") req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") @@ -109,7 +104,6 @@ func sendPostRequestWithForm(client httpkit.HttpClient) { req.Header.Add(http.HeaderOrderKey, "content-length") req.Header.Add(http.HeaderOrderKey, "content-type") req.Header.Add(http.HeaderOrderKey, "cookie") - req.Header.Add(http.HeaderOrderKey, "host") req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") @@ -131,7 +125,6 @@ func sendPostRequestWithJSON(client httpkit.HttpClient) { req, _ := http.NewRequest(http.MethodPost, URL+"/post", body) req.Header.Add("Accept", "*/*") req.Header.Add("Content-Type", "application/json") - req.Header.Add("Host", "httpbin.org") req.Header.Add("User-Agent", "harkit-example") req.Header.Add("Accept-Encoding", "gzip, deflate, br") @@ -141,7 +134,6 @@ func sendPostRequestWithJSON(client httpkit.HttpClient) { req.Header.Add(http.HeaderOrderKey, "content-length") req.Header.Add(http.HeaderOrderKey, "content-type") req.Header.Add(http.HeaderOrderKey, "cookie") - req.Header.Add(http.HeaderOrderKey, "host") req.Header.Add(http.HeaderOrderKey, "user-agent") req.Header.Add(http.HeaderOrderKey, "accept-encoding") From 5c3a7ef958f3c623319a349a5a75a2202e26b829 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:33:42 +0000 Subject: [PATCH 56/60] refactor: improve response handling by introducing protocol header management and updating header processing logic --- converter/request.go | 4 ++-- converter/response.go | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/converter/request.go b/converter/request.go index a7ceba0..f8910fd 100644 --- a/converter/request.go +++ b/converter/request.go @@ -24,7 +24,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { reqProto := DefaultRequestHTTPVersion // WARNING: req.Proto is not always accurate - protocolHeader := handleProtocolHeader(reqProto, req.Method, *req.URL) + protocolHeader := handleRequestProtocolHeader(reqProto, req.Method, *req.URL) headers := convertHeaders(req.Header, req.ContentLength) postData, err := extractRequestPostData(req) @@ -45,7 +45,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { }, nil } -func handleProtocolHeader(proto string, method string, url url.URL) []*harfile.NVPair { +func handleRequestProtocolHeader(proto string, method string, url url.URL) []*harfile.NVPair { if proto == "HTTP/2.0" { return []*harfile.NVPair{ {Name: ":method", Value: method}, diff --git a/converter/response.go b/converter/response.go index c758eb4..27547ad 100644 --- a/converter/response.go +++ b/converter/response.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "io" + "strconv" "github.com/Mathious6/harkit/harfile" http "github.com/bogdanfinn/fhttp" @@ -19,12 +20,15 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { return nil, err } + protocolHeader := handleResponseProtocolHeader(resp.Proto, resp.StatusCode) + headers := convertHeaders(resp.Header, resp.ContentLength) + return &harfile.Response{ Status: int64(resp.StatusCode), StatusText: http.StatusText(resp.StatusCode), HTTPVersion: resp.Proto, Cookies: convertCookies(resp.Cookies()), - Headers: convertHeaders(resp.Header, content.Size), + Headers: append(protocolHeader, headers...), Content: content, RedirectURL: locateRedirectURL(resp), HeadersSize: -1, @@ -32,6 +36,16 @@ func FromHTTPResponse(resp *http.Response) (*harfile.Response, error) { }, nil } +func handleResponseProtocolHeader(proto string, status int) []*harfile.NVPair { + if proto == "HTTP/2.0" { + return []*harfile.NVPair{ + {Name: ":status", Value: strconv.Itoa(status)}, + } + } else { + return []*harfile.NVPair{} + } +} + func locateRedirectURL(resp *http.Response) *string { if resp.StatusCode >= 300 && resp.StatusCode < 400 { if loc, err := resp.Location(); err == nil { From f83c88d45723613d18883a6b0c41aa18333fa290 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 10:51:45 +0000 Subject: [PATCH 57/60] feat: enhance HAR entry logging by including client proxy information in request and response handling --- example/main.go | 8 ++++---- harfile/har.go | 1 + harhandler/har_handler.go | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/example/main.go b/example/main.go index db82490..606ad11 100644 --- a/example/main.go +++ b/example/main.go @@ -60,7 +60,7 @@ func sendGetRequestWithQueryParams(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), client.GetProxy(), sentAt, req, resp) fmt.Println("Parameters sent.") } @@ -81,7 +81,7 @@ func sendGetRequestWithSetCookies(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), client.GetProxy(), sentAt, req, resp) fmt.Println("Cookies set.") } @@ -113,7 +113,7 @@ func sendPostRequestWithForm(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), client.GetProxy(), sentAt, req, resp) fmt.Println("Form URL-encoded request sent.") } @@ -143,7 +143,7 @@ func sendPostRequestWithJSON(client httpkit.HttpClient) { panic(err) } defer resp.Body.Close() - harhandler.AddEntry(client.GetFlowId(), sentAt, req, resp) + harhandler.AddEntry(client.GetFlowId(), client.GetProxy(), sentAt, req, resp) fmt.Println("JSON request sent.") } diff --git a/harfile/har.go b/harfile/har.go index 34392bd..9d1ff99 100644 --- a/harfile/har.go +++ b/harfile/har.go @@ -79,6 +79,7 @@ type Entry struct { Response *Response `json:"response"` // Detailed info about the response. Cache *Cache `json:"cache"` // Info about cache usage. Timings *Timings `json:"timings"` // Detailed timing info about request/response round trip. + ClientProxy string `json:"clientProxy,omitempty"` // NEW: Proxy used by the client to connect to the server. ServerIPAddress string `json:"serverIPAddress,omitempty"` // IP address of the server that was connected (result of DNS resolution). Connection string `json:"connection,omitempty"` // Unique ID of the parent TCP/IP connection, can be the client or server port number. Note that a port number doesn't have to be unique identifier in cases where the port is shared for more connections. If the port isn't available for the application, any other unique connection ID can be used instead (e.g. connection index). Leave out this field if the application doesn't support this info. Comment string `json:"comment,omitempty"` // A comment provided by the user or the application. diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index f66c636..52729c0 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -83,8 +83,8 @@ func newHARHandler(flowID string, opts ...HandlerOption) *HARHandler { } // AddEntry adds a new entry to the HARHandler for a flow with the given flowID, sentAt, request, and response -func AddEntry(flowId string, sentAt time.Time, req *http.Request, resp *http.Response) error { - return GetOrCreateHandler(flowId).AddEntry(sentAt, req, resp) +func AddEntry(flowId string, proxy string, sentAt time.Time, req *http.Request, resp *http.Response) error { + return GetOrCreateHandler(flowId).AddEntry(proxy, sentAt, req, resp) } // Export exports the HAR data for a flow with the given flowID and filename @@ -95,7 +95,7 @@ func Export(flowId, filename string) error { } // AddEntry adds a new entry to the HARHandler for a flow with the given sentAt, request, and response -func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Response) error { +func (h *HARHandler) AddEntry(proxy string, sentAt time.Time, req *http.Request, resp *http.Response) error { timingsReceive := float64(time.Since(sentAt).Milliseconds()) clonedReq, err := cloneRequestPreserveBody(req) @@ -125,6 +125,7 @@ func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Re Response: harResp, Cache: nil, Timings: timings, + ClientProxy: proxy, ServerIPAddress: resolveServerIPAddress(h.resolveIPAddress, harReq.URL), }) @@ -135,7 +136,7 @@ func (h *HARHandler) AddEntry(sentAt time.Time, req *http.Request, resp *http.Re // IP address as a string. Returns an empty string on failure. This is a blocking operation. func resolveServerIPAddress(resolve bool, rawURL string) string { if !resolve { - return "" + return "0.0.0.0" } parsedURL, err := url.Parse(rawURL) From 1c3eeed55934132017d16bffcb53ee4d9ad9e88d Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 21:27:33 +0000 Subject: [PATCH 58/60] refactor: simplify request handling in AddEntry by removing request cloning and directly using the original request --- harhandler/har_handler.go | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/harhandler/har_handler.go b/harhandler/har_handler.go index 52729c0..b40a17b 100644 --- a/harhandler/har_handler.go +++ b/harhandler/har_handler.go @@ -1,10 +1,8 @@ package harhandler import ( - "bytes" "context" "fmt" - "io" "net" "net/url" "sync" @@ -98,11 +96,7 @@ func Export(flowId, filename string) error { func (h *HARHandler) AddEntry(proxy string, sentAt time.Time, req *http.Request, resp *http.Response) error { timingsReceive := float64(time.Since(sentAt).Milliseconds()) - clonedReq, err := cloneRequestPreserveBody(req) - if err != nil { - return err - } - harReq, err := converter.FromHTTPRequest(clonedReq) + harReq, err := converter.FromHTTPRequest(req) if err != nil { return err } @@ -150,24 +144,3 @@ func resolveServerIPAddress(resolve bool, rawURL string) string { } return ipAddrs[0].IP.String() } - -// cloneRequestPreserveBody clones an HTTP request and preserves its body by buffering the content -// into memory. Both the original and the cloned request will be reset with a fresh body reader, -// allowing for safe reuse without data loss. -func cloneRequestPreserveBody(req *http.Request) (*http.Request, error) { - if req.Body == nil { - return req.Clone(req.Context()), nil - } - - buf, err := io.ReadAll(req.Body) - if err != nil { - return nil, err - } - defer req.Body.Close() - req.Body = io.NopCloser(bytes.NewReader(buf)) - - clonedReq := req.Clone(req.Context()) - clonedReq.Body = io.NopCloser(bytes.NewReader(buf)) - - return clonedReq, nil -} From 081dea950d043b31cdaceb1f6af770891636fb31 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 21:28:12 +0000 Subject: [PATCH 59/60] refactor: improve request handling by updating header processing and simplifying body reading logic --- converter/request.go | 52 ++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/converter/request.go b/converter/request.go index f8910fd..9c3c258 100644 --- a/converter/request.go +++ b/converter/request.go @@ -1,7 +1,6 @@ package converter import ( - "bytes" "errors" "io" "net/url" @@ -15,6 +14,13 @@ const ( applicationXWWWFormURLEncoded = "application/x-www-form-urlencoded" multipartFormData = "multipart/form-data" maxMultipartFormDataSize = 32 << 20 // 32 MB limit + + methodKey = ":method" + authorityKey = ":authority" + schemeKey = ":scheme" + pathKey = ":path" + + hostKey = "Host" ) func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { @@ -22,7 +28,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { return nil, errors.New("request cannot be nil") } - reqProto := DefaultRequestHTTPVersion // WARNING: req.Proto is not always accurate + reqProto := DefaultRequestHTTPVersion // WARNING: req.Proto is not always accurate so we force it. protocolHeader := handleRequestProtocolHeader(reqProto, req.Method, *req.URL) headers := convertHeaders(req.Header, req.ContentLength) @@ -40,7 +46,7 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { Headers: append(protocolHeader, headers...), QueryString: convertRequestQueryParams(req.URL), PostData: postData, - HeadersSize: computeRequestHeadersSize(req, headers), + HeadersSize: -1, BodySize: req.ContentLength, }, nil } @@ -48,14 +54,14 @@ func FromHTTPRequest(req *http.Request) (*harfile.Request, error) { func handleRequestProtocolHeader(proto string, method string, url url.URL) []*harfile.NVPair { if proto == "HTTP/2.0" { return []*harfile.NVPair{ - {Name: ":method", Value: method}, - {Name: ":authority", Value: url.Host}, - {Name: ":scheme", Value: url.Scheme}, - {Name: ":path", Value: url.RequestURI()}, + {Name: methodKey, Value: method}, + {Name: authorityKey, Value: url.Host}, + {Name: schemeKey, Value: url.Scheme}, + {Name: pathKey, Value: url.RequestURI()}, } } else { return []*harfile.NVPair{ - {Name: "Host", Value: url.Host}, + {Name: hostKey, Value: url.Host}, } } } @@ -77,19 +83,22 @@ func extractRequestPostData(req *http.Request) (*harfile.PostData, error) { return nil, nil } - buf, err := io.ReadAll(req.Body) + body, err := req.GetBody() + if err != nil { + return nil, err + } + defer body.Close() + + bodyText, err := io.ReadAll(body) if err != nil { return nil, err } - defer req.Body.Close() - req.Body = io.NopCloser(bytes.NewReader(buf)) mimeType := req.Header.Get(ContentTypeKey) postData := &harfile.PostData{MimeType: mimeType} if strings.HasPrefix(mimeType, applicationXWWWFormURLEncoded) { - text := string(buf) - pairs := strings.SplitSeq(text, "&") + pairs := strings.SplitSeq(string(bodyText), "&") for pair := range pairs { nv := strings.SplitN(pair, "=", 2) @@ -139,21 +148,6 @@ func extractRequestPostData(req *http.Request) (*harfile.PostData, error) { return postData, nil } - postData.Text = string(buf) + postData.Text = string(bodyText) return postData, nil } - -func computeRequestHeadersSize(req *http.Request, harHeaders []*harfile.NVPair) int64 { - headersSize := 0 - - requestLine := req.Method + " " + req.URL.RequestURI() + " " + req.Proto + "\r\n" - headersSize += len(requestLine) - - for _, header := range harHeaders { - headerLine := header.Name + ": " + header.Value + "\r\n" - headersSize += len(headerLine) - } - - headersSize += len("\r\n") - return int64(headersSize) -} From e9c0c96b9a1bc4bb687c92d060f25fe6afbb5f86 Mon Sep 17 00:00:00 2001 From: Mathious6 Date: Mon, 25 Aug 2025 21:36:30 +0000 Subject: [PATCH 60/60] refactor: update HTTP protocol handling in tests by replacing hardcoded values with constants and adjusting header assertions --- converter/request_test.go | 42 +++++++++++++++------------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/converter/request_test.go b/converter/request_test.go index 5ca4f62..d32f1be 100644 --- a/converter/request_test.go +++ b/converter/request_test.go @@ -14,10 +14,11 @@ import ( ) const ( - REQ_METHOD = http.MethodPost - REQ_URL = "https://example.com/api?foo=bar" - REQ_URI = "/api?foo=bar" - REQ_PROTOCOL = "HTTP/1.1" + REQ_METHOD = http.MethodPost + REQ_URL = "https://example.com/api?foo=bar" + REQ_URI = "/api?foo=bar" + REQ_PROTOCOL = "HTTP/2.0" + REQ_PROTOCOL_HEADERS_COUNT = 4 REQ_HEADER1_NAME = "Name1" REQ_HEADER1_VALUE = "value1" @@ -75,7 +76,8 @@ func TestConverter_GivenProtocol_WhenConvertingHTTPRequest_ThenProtocolShouldBeC result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, REQ_PROTOCOL, result.HTTPVersion, "HAR protocol <> request protocol") + // assert.Equal(t, REQ_PROTOCOL, result.HTTPVersion, "HAR protocol <> request protocol") + assert.Equal(t, "HTTP/2.0", result.HTTPVersion, "HAR protocol <> request protocol") // WARNING: we force it. } func TestConverter_GivenCookies_WhenConvertingHTTPRequest_ThenCookiesShouldBeCorrect(t *testing.T) { @@ -95,11 +97,11 @@ func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersShouldBeCor result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Len(t, result.Headers, 3, "HAR should contain 3 headers") - assert.Equal(t, REQ_HEADER2_NAME, result.Headers[0].Name, "HAR header name <> request header name") - assert.Equal(t, REQ_HEADER2_VALUE, result.Headers[0].Value, "HAR header value <> request header value") - assert.Equal(t, REQ_HEADER1_NAME, result.Headers[1].Name, "HAR header name <> request header name") - assert.Equal(t, REQ_HEADER1_VALUE, result.Headers[1].Value, "HAR header value <> request header value") + assert.Len(t, result.Headers, REQ_PROTOCOL_HEADERS_COUNT+3, "HAR should contain 3 headers") + assert.Equal(t, REQ_HEADER2_NAME, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+0].Name, "HAR header name <> request header name") + assert.Equal(t, REQ_HEADER2_VALUE, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+0].Value, "HAR header value <> request header value") + assert.Equal(t, REQ_HEADER1_NAME, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+1].Name, "HAR header name <> request header name") + assert.Equal(t, REQ_HEADER1_VALUE, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+1].Value, "HAR header value <> request header value") } func TestConverter_GivenURLWithQueryString_WhenConvertingHTTPRequest_ThenQueryStringShouldBeCorrect(t *testing.T) { @@ -153,8 +155,8 @@ func TestConverter_GivenJSONBody_WhenConvertingHTTPRequest_ThenContentLengthShou result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, converter.ContentLengthKey, result.Headers[2].Name, "HAR content length header name <> request content length header name") - assert.Equal(t, REQ_JSON_CONTENT_LENGTH_VALUE, result.Headers[2].Value, "HAR content length header value <> request content length header value") + assert.Equal(t, converter.ContentLengthKey, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+2].Name, "HAR content length header name <> request content length header name") + assert.Equal(t, REQ_JSON_CONTENT_LENGTH_VALUE, result.Headers[REQ_PROTOCOL_HEADERS_COUNT+2].Value, "HAR content length header value <> request content length header value") } func TestConverter_GivenMultipartBody_WhenConvertingHTTPRequest_ThenPostDataShouldBeCorrect(t *testing.T) { @@ -181,7 +183,8 @@ func TestConverter_GivenHeaders_WhenConvertingHTTPRequest_ThenHeadersSizeShouldB result, err := converter.FromHTTPRequest(req) require.NoError(t, err) - assert.Equal(t, computeHeadersSize(), result.HeadersSize, "HAR header size <> request header size") + // assert.Equal(t, computeHeadersSize(), result.HeadersSize, "HAR header size <> request header size") + assert.Equal(t, int64(-1), result.HeadersSize, "HAR header size <> request header size") // WARNING: we force it. } func TestConverter_GivenBody_WhenConvertingHTTPRequest_ThenBodySizeShouldBeCorrect(t *testing.T) { @@ -218,19 +221,6 @@ func createRequest(t *testing.T, body io.Reader, contentType string) *http.Reque return req } -// computeHeadersSize calculates the total size of HTTP headers in bytes. -// It sums up the lengths of the HTTP request line, individual headers, -// and the terminating double CRLF sequence. -func computeHeadersSize() int64 { - headersSize := len(REQ_METHOD + " " + REQ_URI + " " + REQ_PROTOCOL + "\r\n") - headersSize += len(REQ_HEADER2_NAME + ": " + REQ_HEADER2_VALUE + "\r\n") - headersSize += len(REQ_HEADER1_NAME + ": " + REQ_HEADER1_VALUE + "\r\n") - headersSize += len(converter.CookieKey + ": " + REQ_COOKIE_NAME + "=" + REQ_COOKIE_VALUE + "\r\n") - headersSize += len("\r\n") - - return int64(headersSize) -} - // createMultipartBody constructs a multipart HTTP request body with predefined fields and file content. // It returns the body as a bytes.Buffer and the corresponding Content-Type header value. func createMultipartBody() (body bytes.Buffer, contentType string) {