From a9e5d79f4c7f006d93b18231222b270600f8144f Mon Sep 17 00:00:00 2001 From: Patrick Dufour Date: Sun, 27 Oct 2019 10:35:55 -0400 Subject: [PATCH] http and registry work --- pkg/http/ErrNotNegotiable.go | 22 +++++ pkg/http/Ext.go | 36 +++++++++ pkg/http/NegotiateFormat.go | 83 +++++++++++++++++++ pkg/http/NegotiateFormat_test.go | 47 +++++++++++ pkg/http/Respond.go | 45 +++++++++++ pkg/http/RespondWithContent.go | 37 +++++++++ pkg/http/Respond_test.go | 135 +++++++++++++++++++++++++++++++ pkg/http/http.go | 40 +++++++++ pkg/http/http_test.go | 38 +++++++++ pkg/registry/registry.go | 63 +++++++++++++++ pkg/registry/registry_test.go | 70 ++++++++++++++++ 11 files changed, 616 insertions(+) create mode 100644 pkg/http/ErrNotNegotiable.go create mode 100644 pkg/http/Ext.go create mode 100644 pkg/http/NegotiateFormat.go create mode 100644 pkg/http/NegotiateFormat_test.go create mode 100644 pkg/http/Respond.go create mode 100644 pkg/http/RespondWithContent.go create mode 100644 pkg/http/Respond_test.go create mode 100644 pkg/http/http.go create mode 100644 pkg/http/http_test.go create mode 100644 pkg/registry/registry.go create mode 100644 pkg/registry/registry_test.go diff --git a/pkg/http/ErrNotNegotiable.go b/pkg/http/ErrNotNegotiable.go new file mode 100644 index 0000000..72ebc77 --- /dev/null +++ b/pkg/http/ErrNotNegotiable.go @@ -0,0 +1,22 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "fmt" +) + +// ErrNotNegotiable is used when the server cannot negotiate a format given an accept header. +type ErrNotNegotiable struct { + Value string // the name of the unknown format +} + +// Error returns the error formatted as a string. +func (e ErrNotNegotiable) Error() string { + return fmt.Sprintf("could not negotiate format from string %q", e.Value) +} diff --git a/pkg/http/Ext.go b/pkg/http/Ext.go new file mode 100644 index 0000000..1967316 --- /dev/null +++ b/pkg/http/Ext.go @@ -0,0 +1,36 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "net/http" + "path/filepath" + + "github.com/pkg/errors" +) + +var ( + ErrMissingURL = errors.New("missing URL") +) + +// Ext returns the file name extension in the URL path. +// The extension begins after the last period in the file element of the path. +// If no period is in the last element or a period is the last character, then returns a blank string. +func Ext(r *http.Request) (string, error) { + if r.URL == nil { + return "", ErrMissingURL + } + ext := filepath.Ext(r.URL.Path) + if len(ext) == 0 { + return "", nil + } + if ext == "." { + return "", nil + } + return ext[1:], nil +} diff --git a/pkg/http/NegotiateFormat.go b/pkg/http/NegotiateFormat.go new file mode 100644 index 0000000..dafe8d8 --- /dev/null +++ b/pkg/http/NegotiateFormat.go @@ -0,0 +1,83 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "net/http" + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/spatialcurrent/go-simple-serializer/pkg/registry" +) + +var ( + ErrMissingAcceptHeader = errors.New("missing accept header") + ErrMissingRegistry = errors.New("missing file type registry") +) + +// NegotiateFormat negotitates the format for the response based on the incoming request and the given file type registry. +// Returns the matching content type, followed by the format known to GSS, and then an error if any. +func NegotiateFormat(r *http.Request, reg *registry.Registry) (string, string, error) { + + accept := strings.TrimSpace(r.Header.Get(HeaderAccept)) + + if len(accept) == 0 { + return "", "", ErrMissingAcceptHeader + } + + if reg == nil { + return "", "", ErrMissingRegistry + } + + // Parse accept header into map of weights to accepted values + values := map[float64][]string{} + for _, str := range strings.SplitN(accept, ",", -1) { + v := strings.TrimSpace(str) + if strings.Contains(v, ";q=") { + parts := strings.SplitN(v, ";q=", 2) + w, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + return "", "", errors.Wrapf(err, "could not parse quality value for value %q", v) + } + if _, ok := values[w]; !ok { + values[w] = make([]string, 0) + } + values[w] = append(values[w], strings.TrimSpace(parts[0])) + + } else { + if _, ok := values[1.0]; !ok { + values[1.0] = make([]string, 0) + } + values[1.0] = append(values[1.0], v) + } + } + + // Create list of weights + weights := make([]float64, 0, len(values)) + for w := range values { + weights = append(weights, w) + } + + // Sort by weigt in descending order + sort.SliceStable(weights, func(i, j int) bool { + return weights[i] > weights[j] + }) + + // Iterate through accepted values in order of highest weight first + for _, w := range weights { + for _, contentType := range values[w] { + if item, ok := reg.LookupContentType(contentType); ok { + return contentType, item.Format, nil + } + } + } + return "", "", &ErrNotNegotiable{Value: accept} +} diff --git a/pkg/http/NegotiateFormat_test.go b/pkg/http/NegotiateFormat_test.go new file mode 100644 index 0000000..67aca76 --- /dev/null +++ b/pkg/http/NegotiateFormat_test.go @@ -0,0 +1,47 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/spatialcurrent/go-simple-serializer/pkg/serializer" +) + +func TestNegotiateFormatJSON(t *testing.T) { + reg := NewDefaultRegistry() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "application/json") + c, f, err := NegotiateFormat(r, reg) + assert.NoError(t, err) + assert.Equal(t, "application/json", c) + assert.Equal(t, serializer.FormatJSON, f) +} + +func TestNegotiateFormatBSON(t *testing.T) { + reg := NewDefaultRegistry() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "application/ubjson, application/json") + c, f, err := NegotiateFormat(r, reg) + assert.NoError(t, err) + assert.Equal(t, "application/ubjson", c) + assert.Equal(t, serializer.FormatBSON, f) +} + +func TestNegotiateFormatWeight(t *testing.T) { + reg := NewDefaultRegistry() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "text/csv;q=0.8, application/json;q=0.9") + c, f, err := NegotiateFormat(r, reg) + assert.NoError(t, err) + assert.Equal(t, "application/json", c) + assert.Equal(t, serializer.FormatJSON, f) +} diff --git a/pkg/http/Respond.go b/pkg/http/Respond.go new file mode 100644 index 0000000..4d93466 --- /dev/null +++ b/pkg/http/Respond.go @@ -0,0 +1,45 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "net/http" + + "github.com/pkg/errors" + + "github.com/spatialcurrent/go-simple-serializer/pkg/registry" + "github.com/spatialcurrent/go-simple-serializer/pkg/serializer" +) + +// Respond writes the given data to the respond writer, and returns an error if any. +// If filename is not empty, then the "Content-Disposition" header is set to "attachment; filename=". +func Respond(w http.ResponseWriter, r *http.Request, reg *registry.Registry, data interface{}, status int, filename string) error { + + contentType, format, err := NegotiateFormat(r, reg) + if err != nil { + ext, err := Ext(r) + if err != nil || len(ext) == 0 { + return errors.Errorf("could not negotiate format or parse file extension from %#v", r) + } + if item, ok := reg.LookupExtension(ext); ok { + contentType = item.ContentTypes[0] + format = item.Format + } else { + return errors.Errorf("could not negotiate format or parse file extension from %#v", r) + } + } + + s := serializer.New(format) + + body, err := s.Serialize(data) + if err != nil { + return errors.Wrap(err, "error serializing response body") + } + + return RespondWithContent(w, body, contentType, status, filename) +} diff --git a/pkg/http/RespondWithContent.go b/pkg/http/RespondWithContent.go new file mode 100644 index 0000000..5013da8 --- /dev/null +++ b/pkg/http/RespondWithContent.go @@ -0,0 +1,37 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "fmt" + "net/http" + + "github.com/pkg/errors" +) + +// RespondWithContent writes the given content to the response writer, and returns an error if any. +// If filename is not empty, then the "Content-Disposition" header is set to "attachment; filename=". +func RespondWithContent(w http.ResponseWriter, body []byte, contentType string, status int, filename string) error { + + if len(filename) > 0 { + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename)) + } + + w.Header().Set("Content-Type", contentType) + + if status != http.StatusOK { + w.WriteHeader(status) + } + + _, err := w.Write(body) + if err != nil { + return errors.Wrap(err, "error writing response body") + } + + return nil +} diff --git a/pkg/http/Respond_test.go b/pkg/http/Respond_test.go new file mode 100644 index 0000000..e107b7e --- /dev/null +++ b/pkg/http/Respond_test.go @@ -0,0 +1,135 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "net/http" + "net/http/httptest" + "testing" + + "io/ioutil" + + "github.com/stretchr/testify/assert" +) + +func TestRespondAcceptCSV(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "text/csv") + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "foo\nbar\n", string(body)) +} + +func TestRespondAcceptJSON(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "application/json") + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "{\"foo\":\"bar\"}", string(body)) +} + +func TestRespondAcceptYAML(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar", nil) + r.Header.Set("Accept", "application/json;q=0.8, text/yaml;q=0.9") + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "foo: bar\n", string(body)) +} + +func TestRespondExtensionCSV(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar.csv", nil) + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "foo\nbar\n", string(body)) +} + +func TestRespondExtensionJSON(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar.json", nil) + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "{\"foo\":\"bar\"}", string(body)) +} + +func TestRespondExtensionYAML(t *testing.T) { + reg := NewDefaultRegistry() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "https://example.com/foo/bar.yml", nil) + data := map[string]interface{}{"foo": "bar"} + status := http.StatusOK + err := Respond(w, r, reg, data, status, "") + if !assert.NoError(t, err) { + return + } + resp := w.Result() + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + if !assert.NoError(t, err) { + return + } + assert.Equal(t, "foo: bar\n", string(body)) +} diff --git a/pkg/http/http.go b/pkg/http/http.go new file mode 100644 index 0000000..c5fc9a7 --- /dev/null +++ b/pkg/http/http.go @@ -0,0 +1,40 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "github.com/spatialcurrent/go-simple-serializer/pkg/serializer" +) + +const ( + HeaderAccept = "accept" +) + +const ( + NoSkip = 0 // used as SkipLines parameter to indicate no skipping when reading + NoLimit = -1 // used to indicate that there is no limit on reading or writing, depending on context. + NoComment = "" // used to indicate that there is no comment prefix to consider. +) + +var ( + // NoHeader is used to indicate that no defined header is given. + // Derive the header from the input data. + NoHeader = []interface{}{} + // Formats is a list of all the formats supported by GSS + Formats = serializer.Formats +) + +var ( + ContentTypeBSON = "application/ubjson" + ContentTypeCSV = "text/csv" + ContentTypeJSON = "application/json" + ContentTypeTOML = "application/toml" + ContentTypeTSV = "text/tab-separated-values" + ContentTypeYAML = "text/yaml" + ContentTypePlain = "text/plain; charset=utf-8" +) diff --git a/pkg/http/http_test.go b/pkg/http/http_test.go new file mode 100644 index 0000000..071157d --- /dev/null +++ b/pkg/http/http_test.go @@ -0,0 +1,38 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package http + +import ( + "github.com/spatialcurrent/go-simple-serializer/pkg/registry" + "github.com/spatialcurrent/go-simple-serializer/pkg/serializer" +) + +func NewDefaultRegistry() *registry.Registry { + r := registry.New() + r.Add(registry.Item{ + Format: serializer.FormatBSON, + ContentTypes: []string{"application/ubjson"}, + Extensions: []string{}, + }) + r.Add(registry.Item{ + Format: serializer.FormatJSON, + ContentTypes: []string{"application/json", "text/json"}, + Extensions: []string{"json"}, + }) + r.Add(registry.Item{ + Format: serializer.FormatCSV, + ContentTypes: []string{"text/csv"}, + Extensions: []string{"csv"}, + }) + r.Add(registry.Item{ + Format: serializer.FormatYAML, + ContentTypes: []string{"text/yaml"}, + Extensions: []string{"yaml", "yml"}, + }) + return r +} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go new file mode 100644 index 0000000..9e7d6a1 --- /dev/null +++ b/pkg/registry/registry.go @@ -0,0 +1,63 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +// Package registry provides a library for managing a list of known file types. +package registry + +type Item struct { + Format string + ContentTypes []string + Extensions []string +} + +type Registry struct { + items []Item + formats map[string]Item + contentTypes map[string]Item + extensions map[string]Item +} + +func (r Registry) LookupFormat(format string) (Item, bool) { + item, ok := r.formats[format] + return item, ok +} + +func (r Registry) LookupContentType(contentType string) (Item, bool) { + item, ok := r.contentTypes[contentType] + return item, ok +} + +func (r Registry) LookupExtension(extension string) (Item, bool) { + item, ok := r.extensions[extension] + return item, ok +} + +func (r *Registry) Add(item Item) { + r.items = append(r.items, item) + if len(item.Format) > 0 { + r.formats[item.Format] = item + } + if len(item.ContentTypes) > 0 { + for _, key := range item.ContentTypes { + r.contentTypes[key] = item + } + } + if len(item.Extensions) > 0 { + for _, key := range item.Extensions { + r.extensions[key] = item + } + } +} + +func New() *Registry { + return &Registry{ + items: make([]Item, 0), + formats: map[string]Item{}, + contentTypes: map[string]Item{}, + extensions: map[string]Item{}, + } +} diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go new file mode 100644 index 0000000..25f002b --- /dev/null +++ b/pkg/registry/registry_test.go @@ -0,0 +1,70 @@ +// ================================================================= +// +// Copyright (C) 2019 Spatial Current, Inc. - All Rights Reserved +// Released as open source under the MIT License. See LICENSE file. +// +// ================================================================= + +package registry + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRegistryLookupFormatNil(t *testing.T) { + r := New() + item, ok := r.LookupFormat("json") + assert.False(t, ok) + assert.Equal(t, Item{}, item) +} + +func TestRegistryLookupFormatJSON(t *testing.T) { + r := New() + expected := Item{ + Format: "json", + ContentTypes: []string{"application/json", "text/json"}, + Extensions: []string{"json"}, + } + r.Add(expected) + item, ok := r.LookupFormat("json") + assert.True(t, ok) + assert.Equal(t, expected, item) +} + +func TestRegistryLookupFormatCSV(t *testing.T) { + r := New() + r.Add(Item{ + Format: "json", + ContentTypes: []string{"application/json", "text/json"}, + Extensions: []string{"json"}, + }) + expected := Item{ + Format: "csv", + ContentTypes: []string{"text/csv"}, + Extensions: []string{"csv"}, + } + r.Add(expected) + item, ok := r.LookupFormat("csv") + assert.True(t, ok) + assert.Equal(t, expected, item) +} + +func TestRegistryLookupContentTypeCSV(t *testing.T) { + r := New() + r.Add(Item{ + Format: "json", + ContentTypes: []string{"application/json", "text/json"}, + Extensions: []string{"json"}, + }) + expected := Item{ + Format: "csv", + ContentTypes: []string{"text/csv"}, + Extensions: []string{"csv"}, + } + r.Add(expected) + item, ok := r.LookupContentType("text/csv") + assert.True(t, ok) + assert.Equal(t, expected, item) +}