diff --git a/example/go.mod b/example/go.mod new file mode 100644 index 0000000..4daed0d --- /dev/null +++ b/example/go.mod @@ -0,0 +1,10 @@ +module example.com + +go 1.18 + +require ( + github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 + github.com/snabb/httpreaderat v1.0.1 +) + +require github.com/pkg/errors v0.8.1 // indirect diff --git a/example/go.sum b/example/go.sum new file mode 100644 index 0000000..7a64ab6 --- /dev/null +++ b/example/go.sum @@ -0,0 +1,6 @@ +github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 h1:JIxGEMs4E5Zb6R7z2C5IgecI0mkqS97WAEF31wUbYTM= +github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270/go.mod h1:2XtVRGCw/HthOLxU0Qw6o6jSJrcEoOb2OCCl8gQYvGw= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/snabb/httpreaderat v1.0.1 h1:whlb+vuZmyjqVop8x1EKOg05l2NE4z9lsMMXjmSUCnY= +github.com/snabb/httpreaderat v1.0.1/go.mod h1:lpbGrKDWF37yvRbtRvQsbesS6Ty5c83t8ztannPoMsA= diff --git a/fixtures/file.zip b/fixtures/file.zip new file mode 100644 index 0000000..c283d29 Binary files /dev/null and b/fixtures/file.zip differ diff --git a/fixtures/test1 b/fixtures/test1 new file mode 100644 index 0000000..57e4858 --- /dev/null +++ b/fixtures/test1 @@ -0,0 +1 @@ +some line of text diff --git a/fixtures/test2 b/fixtures/test2 new file mode 100644 index 0000000..94bec47 --- /dev/null +++ b/fixtures/test2 @@ -0,0 +1,5 @@ +Lorem ipsum dolor sit amet. Aut error officiis nam voluptas aliquam aut alias mollitia et omnis voluptatem. Est rerum quia qui voluptatem odio ut voluptatibus Quis et eius quae est similique sint et enim laboriosam ut quia mollitia. Sit expedita autem a explicabo sint ab dicta voluptas aut assumenda maxime. + +Est omnis molestiae non magni laborum vel tempora laborum eum accusantium rerum 33 autem sequi eos omnis perspiciatis sit iusto repellendus. Cum numquam quasi eum beatae eveniet et inventore velit qui vero facilis qui quos odio rem facere odit et aspernatur quia. Eos cumque rerum et sint saepe eos saepe nulla ab rerum quia id necessitatibus pariatur aut laborum sint. + +Est beatae natus et fuga sint vel quod placeat. Eum voluptatem voluptates et atque vero non doloremque asperiores et expedita voluptas. Et explicabo beatae id magni totam et incidunt nesciunt ea mollitia quia et suscipit veniam ut odit consequatur cum aliquam magnam. Ut voluptatibus velit qui tempore voluptas ut Quis velit in labore eligendi et porro soluta ab neque autem in dolorem rerum diff --git a/go.mod b/go.mod index 7d83c3c..1335663 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,11 @@ module github.com/snabb/httpreaderat +go 1.18 + +require github.com/stretchr/testify v1.10.0 + require ( - github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 - github.com/pkg/errors v0.8.1 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1713c7c..713a0b4 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,10 @@ -github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 h1:JIxGEMs4E5Zb6R7z2C5IgecI0mkqS97WAEF31wUbYTM= -github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270/go.mod h1:2XtVRGCw/HthOLxU0Qw6o6jSJrcEoOb2OCCl8gQYvGw= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/httpreaderat.go b/httpreaderat.go index 976d9a8..767d80c 100644 --- a/httpreaderat.go +++ b/httpreaderat.go @@ -9,12 +9,12 @@ package httpreaderat import ( + "errors" "fmt" - "github.com/pkg/errors" "io" "net/http" - "strconv" - "strings" + + "github.com/snabb/httpreaderat/pkg/contentrange" ) // HTTPReaderAt is io.ReaderAt implementation that makes HTTP Range Requests. @@ -93,8 +93,8 @@ func (ra *HTTPReaderAt) ReadAt(p []byte, off int64) (n int, err error) { return ra.readAt(p, off, false) } -func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err error) { - if ra.usebs == true { +func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (int, error) { + if ra.usebs { return ra.bs.ReadAt(p, off) } // fmt.Printf("readat off=%d len=%d\n", off, len(p)) @@ -118,18 +118,18 @@ func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err p = p[:reqLast-reqFirst+1] } - reqRange := fmt.Sprintf("bytes=%d-%d", reqFirst, reqLast) - req.Header.Set("Range", reqRange) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", reqFirst, reqLast)) resp, err := ra.client.Do(req) if err != nil { - return 0, errors.Wrap(err, "http request error") + return 0, fmt.Errorf("http request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - return 0, errors.Errorf("http request error: %s", resp.Status) + return 0, fmt.Errorf("http request error: %s", resp.Status) } + if initialize { ra.meta = getMeta(resp) } else { @@ -138,6 +138,7 @@ func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err return 0, err } } + if resp.StatusCode == http.StatusOK { if ra.bs == nil { return 0, ErrNoRange @@ -160,7 +161,7 @@ func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err } if err != nil { - return 0, err + return int(size), err } return ra.bs.ReadAt(p, off) } @@ -169,23 +170,27 @@ func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err if contentRange == "" { return 0, errors.New("no content-range header in partial response") } - first, last, _, err := parseContentRange(contentRange) + first, last, _, err := contentrange.Parse(contentRange) if err != nil { - return 0, errors.Wrap(err, "http request error") + return 0, fmt.Errorf("http request: %w", err) } if first != reqFirst || last > reqLast { - return 0, errors.Errorf( + return 0, fmt.Errorf( "received different range than requested (req=%d-%d, resp=%d-%d)", reqFirst, reqLast, first, last) } if resp.ContentLength != last-first+1 { return 0, errors.New("content-length mismatch in http response") } - n, err = io.ReadFull(resp.Body, p) - if err == io.ErrUnexpectedEOF { - err = io.EOF + n, err := io.ReadFull(resp.Body, p) + if err != nil { + if errors.Is(err, io.ErrUnexpectedEOF) { + err = io.EOF + } + return 0, err } + if (err == nil || err == io.EOF) && int64(n) != resp.ContentLength { // XXX body size was different from the ContentLength // header? should we do something about it? return error? @@ -196,50 +201,6 @@ func (ra *HTTPReaderAt) readAt(p []byte, off int64, initialize bool) (n int, err return n, err } -var errParse = errors.New("content-range parse error") - -func parseContentRange(str string) (first, last, length int64, err error) { - first, last, length = -1, -1, -1 - - // Content-Range: bytes 42-1233/1234 - // Content-Range: bytes 42-1233/* - // Content-Range: bytes */1234 - // (Maybe I should have used regexp here instead of Splitting... :) - - strs := strings.Split(str, " ") - if len(strs) != 2 || strs[0] != "bytes" { - return -1, -1, -1, errParse - } - strs = strings.Split(strs[1], "/") - if len(strs) != 2 { - return -1, -1, -1, errParse - } - if strs[1] != "*" { - length, err = strconv.ParseInt(strs[1], 10, 64) - if err != nil { - return -1, -1, -1, errParse - } - } - if strs[0] != "*" { - strs = strings.Split(strs[0], "-") - if len(strs) != 2 { - return -1, -1, -1, errParse - } - first, err = strconv.ParseInt(strs[0], 10, 64) - if err != nil { - return -1, -1, -1, errParse - } - last, err = strconv.ParseInt(strs[1], 10, 64) - if err != nil { - return -1, -1, -1, errParse - } - } - if first == -1 && last == -1 && length == -1 { - return -1, -1, -1, errParse - } - return first, last, length, nil -} - func cloneHeader(h http.Header) http.Header { h2 := make(http.Header, len(h)) for k, vv := range h { @@ -288,7 +249,7 @@ func getMeta(resp *http.Response) (meta meta) { case http.StatusPartialContent: contentRange := resp.Header.Get("Content-Range") if contentRange != "" { - _, _, meta.size, _ = parseContentRange(contentRange) + _, _, meta.size, _ = contentrange.Parse(contentRange) } } return meta diff --git a/httpreaderat_test.go b/httpreaderat_test.go new file mode 100644 index 0000000..5b35493 --- /dev/null +++ b/httpreaderat_test.go @@ -0,0 +1,152 @@ +package httpreaderat_test + +import ( + "archive/zip" + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/snabb/httpreaderat" + "github.com/stretchr/testify/suite" +) + +type readerAtFixture struct { + suite.Suite + server *httptest.Server +} + +func (ra *readerAtFixture) AfterTest(suiteName, testName string) { + if ra.server != nil { + ra.server.Close() + } +} + +func (ra *readerAtFixture) reader() (*httpreaderat.HTTPReaderAt, error) { + req, err := http.NewRequest("GET", ra.server.URL+"/file.zip", nil) + ra.Nil(err) + return httpreaderat.New(nil, req, nil) +} + +func TestReaderAtFixture(t *testing.T) { + suite.Run(t, new(readerAtFixture)) +} + +func (ra *readerAtFixture) TestRangeNotSupported() { + ra.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ra.Equal(r.Method, "GET") + ra.Equal(r.URL.String(), "/file.zip") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"content":{"data": [1,2,3]}}`)) + })) + + reader, err := ra.reader() + ra.EqualError(err, "server does not support range requests") + ra.Nil(reader) +} + +func (ra *readerAtFixture) TestNonGetRequestMethod() { + req, err := http.NewRequest("POST", "http://not-valid.url/file.zip", nil) + ra.Nil(err) + reader, err := httpreaderat.New(nil, req, nil) + ra.EqualError(err, "invalid HTTP method") + ra.Nil(reader) +} + +func (ra *readerAtFixture) TestRangeSupportInitial() { + ra.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rnge := r.Header.Get("Range") + ra.Equal(rnge, "bytes=0-0") + w.Header().Set("Content-Range", fmt.Sprintf(" bytes 0-0/%d", 1)) + w.WriteHeader(http.StatusPartialContent) + w.Write([]byte{17}) + })) + + reader, err := ra.reader() + ra.Nil(err) + ra.NotNil(reader) +} + +func (ra *readerAtFixture) TestRangeSupportIntialEmptyResponse() { + ra.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rnge := r.Header.Get("Range") + ra.Equal(rnge, "bytes=0-0") + w.Header().Set("Content-Range", fmt.Sprintf(" bytes 0-0/%d", 1)) + w.WriteHeader(http.StatusPartialContent) + w.Write([]byte{}) + })) + + reader, err := ra.reader() + ra.EqualError(err, "content-length mismatch in http response") + ra.Nil(reader) +} + +func (ra *readerAtFixture) TestRangeSupportIntialTooMuchResponse() { + ra.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rnge := r.Header.Get("Range") + ra.Equal(rnge, "bytes=0-0") + w.Header().Set("Content-Range", fmt.Sprintf(" bytes 0-0/%d", 1)) + w.WriteHeader(http.StatusPartialContent) + w.Write([]byte{17, 18, 19}) // should be 1 + })) + + reader, err := ra.reader() + ra.EqualError(err, "content-length mismatch in http response") + ra.Nil(reader) +} + +func (ra *readerAtFixture) TestFile() { + ra.server = httptest.NewServer(http.FileServer(http.Dir("./fixtures"))) + + reader, err := ra.reader() + ra.Nil(err) + ra.NotNil(reader) + + zfile, err := zip.NewReader(reader, reader.Size()) + ra.NoError(err) + ra.Len(zfile.File, 4) + + txtfile, err := zfile.Open("test2") + ra.NoError(err) + defer txtfile.Close() + + content, err := io.ReadAll(txtfile) + ra.NoError(err) + + ra.Equal(http.DetectContentType(content), "text/plain; charset=utf-8") +} + +func (ra *readerAtFixture) TestParralelFiles() { + ra.server = httptest.NewServer(http.FileServer(http.Dir("./fixtures"))) + + reader, err := ra.reader() + ra.Nil(err) + ra.NotNil(reader) + + zfile, err := zip.NewReader(reader, reader.Size()) + ra.NoError(err) + ra.Len(zfile.File, 4) + + var wg sync.WaitGroup + + testread := func(f *zip.File) { + defer wg.Done() + file, err := f.Open() + ra.NoError(err) + defer file.Close() + + content, err := io.ReadAll(file) + ra.NoError(err) + ra.NotEmpty(content, "name", f.Name) + } + + for _, f := range zfile.File { + wg.Add(1) + go testread(f) + } + + wg.Wait() + +} diff --git a/pkg/contentrange/contentrange.go b/pkg/contentrange/contentrange.go new file mode 100644 index 0000000..309ff04 --- /dev/null +++ b/pkg/contentrange/contentrange.go @@ -0,0 +1,71 @@ +package contentrange + +import ( + "errors" + "fmt" + "strconv" + "strings" + "unicode" +) + +var ( + ErrParse = errors.New("content-range parse error") + ErrUnsupportedUnit = errors.New("unsupported unit") + ErrUnsupportedField = errors.New("unsupported field") +) + +// Parse parse content of a Content-Range header. +func Parse(str string) (first, last, length int64, err error) { + split := func(r rune) bool { + if unicode.IsSpace(r) { + return true + } + + switch r { + case '-', '/': + return true + } + return false + } + + fields := strings.FieldsFunc(str, split) + + if fields[0] != "bytes" { + return -1, -1, -1, ErrUnsupportedUnit + } + + if len(fields) == 4 { + first, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("can't parse first: %w", err) + } + last, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("can't parse first: %w", err) + } + length := int64(-1) + if fields[3] != "*" { + length, err = strconv.ParseInt(fields[3], 10, 64) + if err != nil { + return -1, -1, -1, fmt.Errorf("can't parse length: %w", err) + } + } + + return first, last, length, nil + } + + if len(fields) == 3 { + if fields[1] != "*" { + return -1, -1, -1, ErrUnsupportedField + } + + length, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("can't parse length: %w", err) + } + + return -1, -1, length, nil + } + + return -1, -1, -1, ErrParse +} diff --git a/pkg/contentrange/contentrange_test.go b/pkg/contentrange/contentrange_test.go new file mode 100644 index 0000000..46e3897 --- /dev/null +++ b/pkg/contentrange/contentrange_test.go @@ -0,0 +1,106 @@ +package contentrange_test + +import ( + "log" + "net/http" + "testing" + + "github.com/snabb/httpreaderat/pkg/contentrange" +) + +func TestParse(t *testing.T) { + type args struct { + str string + } + tests := []struct { + name string + args args + wantFirst int64 + wantLast int64 + wantLength int64 + wantErr bool + }{ + { + name: "full", + args: args{ + str: "bytes 42-1233/1234", + }, + wantFirst: 42, + wantLast: 1233, + wantLength: 1234, + wantErr: false, + }, + { + name: "size unknown", + args: args{ + str: "bytes 42-1233/*", + }, + wantFirst: 42, + wantLast: 1233, + wantLength: -1, + wantErr: false, + }, + { + name: "wildcard range", + args: args{ + str: "bytes */1234", + }, + wantFirst: -1, + wantLast: -1, + wantLength: 1234, + wantErr: false, + }, + { + name: "bad unit", + args: args{ + str: "banana 200-1000/67589", + }, + wantFirst: -1, + wantLast: -1, + wantLength: -1, + wantErr: true, + }, + { + name: "bad field", + args: args{ + str: "bytes 0/67589", + }, + wantFirst: -1, + wantLast: -1, + wantLength: -1, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFirst, gotLast, gotLength, err := contentrange.Parse(tt.args.str) + if (err != nil) != tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotFirst != tt.wantFirst { + t.Errorf("Parse() gotFirst = %v, want %v", gotFirst, tt.wantFirst) + } + if gotLast != tt.wantLast { + t.Errorf("Parse() gotLast = %v, want %v", gotLast, tt.wantLast) + } + if gotLength != tt.wantLength { + t.Errorf("Parse() gotLength = %v, want %v", gotLength, tt.wantLength) + } + }) + } +} + +func ExampleParse() { + // fake http response + res := http.Response{} + res.Header.Add("Content-Range", "bytes 42-1233/1234") + + // get header and parse + first, last, length, err := contentrange.Parse(res.Header.Get("Content-Range")) + if err != nil { + log.Panicf("can't parse content-range: %v", err) + } + + log.Printf("%d, %d, %d", first, last, length) +} diff --git a/store.go b/store.go index e29db25..dc2c5e8 100644 --- a/store.go +++ b/store.go @@ -2,7 +2,7 @@ package httpreaderat import ( "bytes" - "github.com/pkg/errors" + "errors" "io" "io/ioutil" "os"