From eca4fdc885461266fc0e539b3d06595de17b0026 Mon Sep 17 00:00:00 2001 From: Roffe Date: Mon, 24 May 2021 00:56:42 +0200 Subject: [PATCH 1/4] add rate & retry opts --- examples/v2/rateAndRetry/main.go | 78 ++++++++++++++++++++++++++++++++ go.mod | 6 ++- go.sum | 4 ++ v2/blizzard.go | 67 +++++++++++++++++++++++++-- v2/blizzard_opts.go | 47 +++++++++++++++++++ 5 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 examples/v2/rateAndRetry/main.go create mode 100644 v2/blizzard_opts.go diff --git a/examples/v2/rateAndRetry/main.go b/examples/v2/rateAndRetry/main.go new file mode 100644 index 0000000..1124993 --- /dev/null +++ b/examples/v2/rateAndRetry/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/FuzzyStatic/blizzard/v2" + "github.com/FuzzyStatic/blizzard/v2/wowsearch" + "github.com/avast/retry-go" +) + +var ( + bnetID string + bnetSECRET string +) + +func init() { + bnetID = os.Getenv("BNET_ID") + bnetSECRET = os.Getenv("BNET_SECRET") + if bnetID == "" || bnetSECRET == "" { + log.Fatal("missing BNET_ID or BNET_SECRET") + } +} +func main() { + blizz := blizzard.NewClient( + bnetID, + bnetSECRET, + blizzard.EU, + blizzard.EnUS, + // This is the client default rate config, we allow 100 requests per second, and a burst of the + blizzard.NewRateOpt(1*time.Second/100, 10), + // This is the client default retry config + blizzard.NewRetryOpt( + retry.Attempts(3), + retry.Delay(100*time.Millisecond), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + switch { + case err.Error() == "429 Too Many Requests": + return true // recoverable error, retry + case err.Error() == "403 Forbidden": + return false + case err.Error() == "404 Not Found": + return false + default: + return false // We cannot retry this away + } + }), + ), + ) + + realmSearch, _, err := blizz.ClassicRealmSearch( + context.TODO(), + wowsearch.Page(1), + wowsearch.PageSize(5), + wowsearch.OrderBy("name.EN_US:asc"), + wowsearch.Field(). + AND("timezone", "Europe/Paris"). + AND("data.locale", "enGB"). + NOT("type.type", "PVP"). + NOT("id", "4756||4757"). + OR("type.type", "NORMAL", "RP"), + ) + if err != nil { + panic(err) + } + + out, err := json.MarshalIndent(realmSearch, "", " ") + if err != nil { + panic(err) + } + + fmt.Println(string(out[:])) +} diff --git a/go.mod b/go.mod index 2682149..fe43530 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module github.com/FuzzyStatic/blizzard go 1.16 -require golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d +require ( + github.com/avast/retry-go v3.0.0+incompatible // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba +) diff --git a/go.sum b/go.sum index e3ba8b0..edb1519 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -9,5 +11,7 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/v2/blizzard.go b/v2/blizzard.go index b19fc40..6dada09 100644 --- a/v2/blizzard.go +++ b/v2/blizzard.go @@ -9,15 +9,23 @@ import ( "io/ioutil" "net/http" "strings" + "time" "github.com/FuzzyStatic/blizzard/v2/wowsearch" + "github.com/avast/retry-go" + "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" + "golang.org/x/time/rate" ) // For testing var c *Client +type ClientOpts interface { + Apply(c *Client) +} + // Client regional API URLs, locale, client ID, client secret type Client struct { httpClient *http.Client @@ -31,6 +39,8 @@ type Client struct { dynamicClassicNamespace, staticClassicNamespace string region Region locale Locale + retryopts []retry.Option + ratelimiter *rate.Limiter } // Region type @@ -86,7 +96,7 @@ const ( ) // NewClient create new Blizzard structure. This structure will be used to acquire your access token and make API calls. -func NewClient(clientID, clientSecret string, region Region, locale Locale) *Client { +func NewClient(clientID, clientSecret string, region Region, locale Locale, opts ...ClientOpts) *Client { var c = Client{ oauth: OAuth{ ClientID: clientID, @@ -102,6 +112,35 @@ func NewClient(clientID, clientSecret string, region Region, locale Locale) *Cli c.SetRegion(region) + for _, opt := range opts { + opt.Apply(&c) + } + + if c.ratelimiter == nil { + c.ratelimiter = rate.NewLimiter(rate.Every(1*time.Second/100), 10) + } + + if len(c.retryopts) == 0 { + c.retryopts = []retry.Option{ + retry.Attempts(3), + retry.Delay(100 * time.Millisecond), + retry.DelayType(retry.BackOffDelay), + retry.MaxJitter(0 * time.Millisecond), + retry.RetryIf(func(err error) bool { + switch { + case err.Error() == "429 Too Many Requests": + return true // recoverable error, retry + case err.Error() == "403 Forbidden": + return false + case err.Error() == "404 Not Found": + return false + default: + return false // We cannot retry this away + } + }), + } + } + return &c } @@ -194,13 +233,20 @@ func buildSearchParams(opts ...wowsearch.Opt) string { return "?" + strings.Join(params, "&") } +func (c *Client) getRetryOpts(ctx context.Context) []retry.Option { + opts := []retry.Option{ + retry.Context(ctx), + } + return append(opts, c.retryopts...) +} + // getStructData processes simple GET request based on pathAndQuery an returns the structured data. func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace string, dat interface{}) (interface{}, *Header, error) { + req, err := http.NewRequestWithContext(ctx, "GET", c.apiHost+pathAndQuery, nil) if err != nil { return dat, nil, err } - req.Header.Set("Accept", "application/json") q := req.URL.Query() @@ -211,10 +257,24 @@ func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace stri req.Header.Set("Battlenet-Namespace", namespace) } - res, err := c.httpClient.Do(req) + var res *http.Response + err = retry.Do( + func() (err error) { + if err := c.ratelimiter.Wait(ctx); err != nil { + return err + } + res, err = c.httpClient.Do(req) + if err != nil && res != nil { + res.Body.Close() + } + return + }, + c.getRetryOpts(ctx)..., + ) if err != nil { return dat, nil, err } + defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) @@ -237,6 +297,7 @@ func (c *Client) getStructData(ctx context.Context, pathAndQuery, namespace stri } return dat, header, nil + } // getStructDataNoNamespace processes simple GET request based on pathAndQuery an returns the structured data. diff --git a/v2/blizzard_opts.go b/v2/blizzard_opts.go new file mode 100644 index 0000000..0295c3f --- /dev/null +++ b/v2/blizzard_opts.go @@ -0,0 +1,47 @@ +package blizzard + +import ( + "time" + + "github.com/avast/retry-go" + "golang.org/x/time/rate" +) + +// NewRateOpt sets the rate of which the client can do requests +// Blizzard allows 36000 per hour or up to 100 per second +// +// Example: +// +// blizzard.NewRateOpt(1*time.Second/100, 10) +func NewRateOpt(rate time.Duration, burst int) ClientOpts { + return &RateOpt{ + rate: rate, + b: burst, + } +} + +type RateOpt struct { + rate time.Duration + b int +} + +func (r *RateOpt) Apply(c *Client) { + c.ratelimiter = rate.NewLimiter( + rate.Every(r.rate), + r.b, + ) +} + +func NewRetryOpt(opts ...retry.Option) ClientOpts { + return &RetryOpt{ + opts: opts, + } +} + +type RetryOpt struct { + opts []retry.Option +} + +func (r *RetryOpt) Apply(c *Client) { + c.retryopts = r.opts +} From 8a854df0eb616d20b374eb389debfa5a237d9c63 Mon Sep 17 00:00:00 2001 From: Roffe Date: Mon, 24 May 2021 00:57:31 +0200 Subject: [PATCH 2/4] better transport settings --- v2/blizzard.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/v2/blizzard.go b/v2/blizzard.go index 6dada09..249a9db 100644 --- a/v2/blizzard.go +++ b/v2/blizzard.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io/ioutil" + "net" "net/http" "strings" "time" @@ -125,7 +126,6 @@ func NewClient(clientID, clientSecret string, region Region, locale Locale, opts retry.Attempts(3), retry.Delay(100 * time.Millisecond), retry.DelayType(retry.BackOffDelay), - retry.MaxJitter(0 * time.Millisecond), retry.RetryIf(func(err error) bool { switch { case err.Error() == "429 Too Many Requests": @@ -183,7 +183,19 @@ func (c *Client) SetRegion(region Region) { } c.cfg.TokenURL = c.oauthHost + "/oauth/token" - c.httpClient = c.cfg.Client(context.Background()) + + defaultTransport := &http.Transport{ + Dial: (&net.Dialer{KeepAlive: 10 * time.Second}).Dial, + MaxIdleConns: 6, + MaxIdleConnsPerHost: 2, + } + + httpClient := &http.Client{ + Timeout: 10 * time.Second, + Transport: defaultTransport, + } + ctx := context.WithValue(context.TODO(), oauth2.HTTPClient, httpClient) + c.httpClient = c.cfg.Client(ctx) } // GetOAuthHost returns the OAuth host of the client From def93240c0e5362ac64227b9d9c25cf34d8aea49 Mon Sep 17 00:00:00 2001 From: Roffe Date: Mon, 24 May 2021 01:07:23 +0200 Subject: [PATCH 3/4] configure max jitter by default --- examples/v2/rateAndRetry/main.go | 21 +++++---------------- v2/blizzard.go | 3 ++- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/examples/v2/rateAndRetry/main.go b/examples/v2/rateAndRetry/main.go index 1124993..5d348a0 100644 --- a/examples/v2/rateAndRetry/main.go +++ b/examples/v2/rateAndRetry/main.go @@ -9,7 +9,6 @@ import ( "time" "github.com/FuzzyStatic/blizzard/v2" - "github.com/FuzzyStatic/blizzard/v2/wowsearch" "github.com/avast/retry-go" ) @@ -38,6 +37,7 @@ func main() { retry.Attempts(3), retry.Delay(100*time.Millisecond), retry.DelayType(retry.BackOffDelay), + retry.MaxJitter(0), retry.RetryIf(func(err error) bool { switch { case err.Error() == "429 Too Many Requests": @@ -53,25 +53,14 @@ func main() { ), ) - realmSearch, _, err := blizz.ClassicRealmSearch( - context.TODO(), - wowsearch.Page(1), - wowsearch.PageSize(5), - wowsearch.OrderBy("name.EN_US:asc"), - wowsearch.Field(). - AND("timezone", "Europe/Paris"). - AND("data.locale", "enGB"). - NOT("type.type", "PVP"). - NOT("id", "4756||4757"). - OR("type.type", "NORMAL", "RP"), - ) + mount, _, err := blizz.WoWMountIndex(context.TODO()) if err != nil { - panic(err) + log.Fatal(err) } - out, err := json.MarshalIndent(realmSearch, "", " ") + out, err := json.MarshalIndent(mount, "", " ") if err != nil { - panic(err) + log.Fatal(err) } fmt.Println(string(out[:])) diff --git a/v2/blizzard.go b/v2/blizzard.go index 249a9db..c7185b7 100644 --- a/v2/blizzard.go +++ b/v2/blizzard.go @@ -126,6 +126,7 @@ func NewClient(clientID, clientSecret string, region Region, locale Locale, opts retry.Attempts(3), retry.Delay(100 * time.Millisecond), retry.DelayType(retry.BackOffDelay), + retry.MaxJitter(0), retry.RetryIf(func(err error) bool { switch { case err.Error() == "429 Too Many Requests": @@ -135,7 +136,7 @@ func NewClient(clientID, clientSecret string, region Region, locale Locale, opts case err.Error() == "404 Not Found": return false default: - return false // We cannot retry this away + return false // unhandled error } }), } From 19946bd6620adb9cc0f619cf85f5536e3c56241c Mon Sep 17 00:00:00 2001 From: Roffe Date: Mon, 24 May 2021 01:09:48 +0200 Subject: [PATCH 4/4] comments --- examples/v2/rateAndRetry/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/v2/rateAndRetry/main.go b/examples/v2/rateAndRetry/main.go index 5d348a0..b469810 100644 --- a/examples/v2/rateAndRetry/main.go +++ b/examples/v2/rateAndRetry/main.go @@ -30,7 +30,7 @@ func main() { bnetSECRET, blizzard.EU, blizzard.EnUS, - // This is the client default rate config, we allow 100 requests per second, and a burst of the + // This is the client default rate config, we allow 100 requests per second, and a burst budget of 10 requests blizzard.NewRateOpt(1*time.Second/100, 10), // This is the client default retry config blizzard.NewRetryOpt(