Experimenting with rate limits

api/api.go

@@ -39,6 +39,8 @@ func NewClient(token string) *Client {
 			r.Header.Set("User-Agent", UserAgent)
 		}
 
+		r.Header.Set("X-RateLimit-Precision", "millisecond")
+
 		return nil
 	}
 

api/rate/majors.go

@@ -0,0 +1,41 @@
+package rate
+
+import (
+	"strconv"
+	"strings"
+)
+
+// TODO: webhook
+var MajorRootPaths = []string{"channels", "guilds"}
+
+func ParseBucketKey(path string) string {
+	path = strings.SplitN(path, "?", 2)[0]
+
+	parts := strings.Split(path, "/")
+	if len(parts) < 1 {
+		return path
+	}
+
+	parts = parts[1:] // [0] is just "" since URL
+
+	var skip int
+
+	for _, part := range MajorRootPaths {
+		if part == parts[0] {
+			skip = 2
+			break
+		}
+	}
+
+	// we need to remove IDs from these
+	for ; skip < len(parts); skip++ {
+		if _, err := strconv.Atoi(parts[skip]); err == nil {
+			// is a number, DELET
+			parts[skip] = ""
+		}
+	}
+
+	// rejoin url
+	path = strings.Join(parts, "/")
+	return "/" + path
+}

api/rate/majors_test.go

@@ -0,0 +1,21 @@
+package rate
+
+import "testing"
+
+func TestBucketKey(t *testing.T) {
+	var tests = [][2]string{
+		{"/guilds/123123/messages", "/guilds/123123/messages"},
+		{"/guilds/123123/", "/guilds/123123/"},
+		{"/channels/123131231", "/channels/123131231"},
+		{"/channels/123123/message/123456", "/channels/123123/message/"},
+		{"/user/123123", "/user/"},
+		{"/user/123123/", "/user//"}, // not sure about this
+	}
+
+	for _, conds := range tests {
+		key := ParseBucketKey(conds[0])
+		if key != conds[1] {
+			t.Fatalf("Expected/got\n%s\n%s", conds[1], key)
+		}
+	}
+}

api/rate/rate.go

@@ -0,0 +1,203 @@
+package rate
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pkg/errors"
+	csync "github.com/sasha-s/go-csync"
+)
+
+// ExtraDelay because Discord is trash. I've seen this in both litcord and
+// discordgo, with dgo claiming from his experiments.
+// RE: Those who want others to fix it for them: release the source code then.
+const ExtraDelay = 250 * time.Millisecond
+
+// This makes me suicidal.
+// https://github.com/bwmarrin/discordgo/blob/master/ratelimit.go
+
+type Limiter struct {
+	// Only 1 per bucket
+	CustomLimits []*CustomRateLimit
+
+	global     *int64 // atomic guarded, unixnano
+	buckets    sync.Map
+	globalRate time.Duration
+}
+
+type CustomRateLimit struct {
+	// This string will match on a Printf format string.
+	// e.g. /guilds/%s/channels/%s/...
+	Contains string
+
+	Reset time.Duration
+}
+
+type bucket struct {
+	lock   csync.Mutex
+	custom *CustomRateLimit
+
+	remaining uint64
+	limit     uint
+
+	reset     time.Time
+	lastReset time.Time // only for custom
+}
+
+func NewLimiter() *Limiter {
+	return &Limiter{
+		global:  new(int64),
+		buckets: sync.Map{},
+		CustomLimits: []*CustomRateLimit{{
+			Contains: "/reactions/",
+			Reset:    200 * time.Millisecond,
+		}},
+	}
+}
+
+func (l *Limiter) getBucket(path string, store bool) *bucket {
+	bc, ok := l.buckets.Load(path)
+	if !ok && !store {
+		return nil
+	}
+
+	if !ok {
+		bc := &bucket{
+			remaining: 1,
+		}
+
+		for _, limit := range l.CustomLimits {
+			if strings.Contains(path, limit.Contains) {
+				bc.custom = limit
+				break
+			}
+		}
+
+		l.buckets.Store(path, bc)
+		return bc
+	}
+
+	return bc.(*bucket)
+}
+
+func (l *Limiter) Acquire(ctx context.Context, path string) error {
+	b := l.getBucket(path, true)
+
+	// Acquire lock with a timeout
+	if err := b.lock.CLock(ctx); err != nil {
+		return err
+	}
+
+	// Time to sleep
+	var sleep time.Duration
+
+	if b.remaining == 0 && b.reset.After(time.Now()) {
+		// out of turns, gotta wait
+		sleep = b.reset.Sub(time.Now())
+	} else {
+		// maybe global rate limit has it
+		now := time.Now()
+		until := time.Unix(0, atomic.LoadInt64(l.global))
+
+		if until.After(now) {
+			sleep = until.Sub(now)
+		}
+	}
+
+	if sleep > 0 {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(sleep):
+		}
+	}
+
+	if b.remaining > 0 {
+		b.remaining--
+	}
+
+	return nil
+}
+
+// Release releases the URL from the locks. This doesn't need a context for
+// timing out, it doesn't block that much.
+func (l *Limiter) Release(path string, headers http.Header) error {
+	b := l.getBucket(path, false)
+	if b == nil {
+		return nil
+	}
+
+	defer b.lock.Unlock()
+
+	// Check custom limiter
+	if b.custom != nil {
+		now := time.Now()
+
+		if now.Sub(b.lastReset) >= b.custom.Reset {
+			b.lastReset = now
+			b.reset = now.Add(b.custom.Reset)
+		}
+
+		return nil
+	}
+
+	var (
+		// boolean
+		global = headers.Get("X-RateLimit-Global")
+
+		// seconds
+		remaining  = headers.Get("X-RateLimit-Remaining")
+		reset      = headers.Get("X-RateLimit-Reset")
+		retryAfter = headers.Get("Retry-After")
+	)
+
+	switch {
+	case retryAfter != "":
+		i, err := strconv.Atoi(retryAfter)
+		if err != nil {
+			return errors.Wrap(err, "Invalid retryAfter "+retryAfter)
+		}
+
+		at := time.Now().Add(time.Duration(i) * time.Millisecond)
+
+		if global != "" { // probably true
+			atomic.StoreInt64(l.global, at.UnixNano())
+		} else {
+			b.reset = at
+		}
+
+	case reset != "":
+		date := headers.Get("Date")
+
+		t, err := http.ParseTime(date)
+		if err != nil {
+			return errors.Wrap(err, "Invalid date "+date)
+		}
+
+		unix, err := strconv.ParseFloat(reset, 64)
+		if err != nil {
+			return errors.Wrap(err, "Invalid reset "+reset)
+		}
+
+		reset := time.Unix(0, int64(unix*float64(time.Second)))
+		delta := reset.Sub(t) + ExtraDelay
+
+		b.reset = time.Now().Add(delta)
+	}
+
+	if remaining != "" {
+		u, err := strconv.ParseUint(remaining, 10, 64)
+		if err != nil {
+			return errors.Wrap(err, "Invalid remaining "+remaining)
+		}
+
+		b.remaining = u
+	}
+
+	return nil
+}

api/rate/rate_test.go

@@ -0,0 +1,83 @@
+package rate
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// https://github.com/bwmarrin/discordgo/blob/master/ratelimit_test.go
+
+func mockRequest(t *testing.T, l *Limiter, path string, headers http.Header) {
+	if err := l.Acquire(context.Background(), path); err != nil {
+		t.Fatal("Failed to acquire lock:", err)
+	}
+
+	/*
+		headers := http.Header{}
+		headers.Set("X-RateLimit-Remaining", "10")
+		headers.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Unix(), 10))
+		headers.Set("Date", time.Now().Format(time.RFC850))
+	*/
+
+	if err := l.Release(path, headers); err != nil {
+		t.Fatal("Failed to release lock:", err)
+	}
+}
+
+// This test takes ~2 seconds to run
+func TestRatelimitReset(t *testing.T) {
+	l := NewLimiter()
+
+	headers := http.Header{}
+	headers.Set("X-RateLimit-Remaining", "0")
+	headers.Set("X-RateLimit-Reset",
+		strconv.FormatInt(time.Now().Add(time.Second*2).Unix(), 10))
+	headers.Set("Date", time.Now().Format(time.RFC850))
+
+	sent := time.Now()
+	mockRequest(t, l, "/guilds/99/channels", headers)
+	mockRequest(t, l, "/guilds/55/channels", headers)
+	mockRequest(t, l, "/guilds/66/channels", headers)
+
+	// call it again
+	mockRequest(t, l, "/guilds/99/channels", headers)
+	mockRequest(t, l, "/guilds/55/channels", headers)
+	mockRequest(t, l, "/guilds/66/channels", headers)
+
+	// We hit the same endpoint 2 times, so we should only be ratelimited 2
+	// second and always less than 4 seconds (unless you're on a stoneage
+	// computer or using swap or something...)
+	if time.Since(sent) >= time.Second && time.Since(sent) < time.Second*4 {
+		t.Log("OK", time.Since(sent))
+	} else {
+		t.Error("Did not ratelimit correctly, got:", time.Since(sent))
+	}
+}
+
+// This test takes ~1 seconds to run
+func TestRatelimitGlobal(t *testing.T) {
+	l := NewLimiter()
+
+	headers := http.Header{}
+	headers.Set("X-RateLimit-Global", "1")
+	// Reset for approx 1 seconds from now
+	headers.Set("Retry-After", "1000")
+
+	sent := time.Now()
+
+	// This should trigger a global ratelimit
+	mockRequest(t, l, "/guilds/99/channels", headers)
+	time.Sleep(time.Millisecond * 100)
+
+	// This shouldn't go through in less than 1 second
+	mockRequest(t, l, "/guilds/55/channels", headers)
+
+	if time.Since(sent) >= time.Second && time.Since(sent) < time.Second*2 {
+		t.Log("OK", time.Since(sent))
+	} else {
+		t.Error("Did not ratelimit correctly, got:", time.Since(sent))
+	}
+}

go.mod

@@ -6,4 +6,7 @@ require (
 	github.com/bwmarrin/discordgo v0.20.2
 	github.com/gorilla/schema v1.1.0
 	github.com/pkg/errors v0.8.1
+	github.com/sasha-s/go-csync v0.0.0-20160729053059-3bc6c8bdb3fa
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 )

go.sum

@@ -6,5 +6,14 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/sasha-s/go-csync v0.0.0-20160729053059-3bc6c8bdb3fa h1:xiD6U6h+QMkAwI195dFwdku2N+enlCy9XwFTnEXaCQo=
+github.com/sasha-s/go-csync v0.0.0-20160729053059-3bc6c8bdb3fa/go.mod h1:KKzWrLiWu6EpzxZBPmPisPgq6oL+do2yLa0C0BTx5fA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

httputil/url.go

@@ -0,0 +1,15 @@
+package httputil
+
+import "fmt"
+
+// URL extends the normal URL and allows for a general string.
+type URL struct {
+	Base string
+	URL  string
+}
+
+func URLf(base string, v ...interface{}) URL {
+	return URL{base, fmt.Sprintf(base, v...)}
+}
+
+func (url URL) String() string { return url.URL }