feat: implement rate limiting and caching layer for MusicBrainz provider
- Add golang.org/x/time/rate dependency for token-bucket rate limiting - Replace custom channel-based rate limiter with rate.NewLimiter(1, 1) - Add context.Context support to doGet for cancellation - Add cached_at column to external_releases via migration 005 - Implement cache hit/miss queries with TTL-based filtering - Add CacheStats type for tracking cached RGIDs - Update ExternalRelease struct with CachedAt field - Add rate limiting tests (1 req/sec enforcement, burst behavior) - Add cache tests (hit, miss, expired, mixed, empty artist) - Update migration count test for new migration
This commit is contained in:
@@ -1,18 +1,16 @@
|
||||
package musicbrainz
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
"naviwatcher/internal/config"
|
||||
)
|
||||
|
||||
func newUnbufferedRateLimiter() *rateLimiter {
|
||||
return newRateLimiter(1000) // high rate to avoid blocking in tests
|
||||
}
|
||||
|
||||
func TestNewClient_ValidConfig(t *testing.T) {
|
||||
cfg := config.MusicBrainzConfig{
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
@@ -48,7 +46,7 @@ func TestNewClientWithLimiter(t *testing.T) {
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
}
|
||||
|
||||
rl := newRateLimiter(1)
|
||||
rl := rate.NewLimiter(rate.Limit(1), 1)
|
||||
client := NewClientWithLimiter(cfg, rl)
|
||||
defer client.Close()
|
||||
|
||||
@@ -78,7 +76,7 @@ func TestDoGet_Success(t *testing.T) {
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
}
|
||||
|
||||
rl := newUnbufferedRateLimiter()
|
||||
rl := rate.NewLimiter(rate.Limit(1000), 1000) // high rate to avoid blocking in tests
|
||||
client := &MusicBrainzClient{
|
||||
httpClient: server.Client(),
|
||||
userAgent: cfg.UserAgent,
|
||||
@@ -87,7 +85,7 @@ func TestDoGet_Success(t *testing.T) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
body, err := client.doGet("/test")
|
||||
body, err := client.doGet(context.Background(), "/test")
|
||||
if err != nil {
|
||||
t.Fatalf("doGet() error = %v", err)
|
||||
}
|
||||
@@ -108,7 +106,7 @@ func TestDoGet_Non200Status(t *testing.T) {
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
}
|
||||
|
||||
rl := newUnbufferedRateLimiter()
|
||||
rl := rate.NewLimiter(rate.Limit(1000), 1000)
|
||||
client := &MusicBrainzClient{
|
||||
httpClient: server.Client(),
|
||||
userAgent: cfg.UserAgent,
|
||||
@@ -117,7 +115,7 @@ func TestDoGet_Non200Status(t *testing.T) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
_, err := client.doGet("/test")
|
||||
_, err := client.doGet(context.Background(), "/test")
|
||||
if err == nil {
|
||||
t.Fatal("doGet() expected error for non-200 status, got nil")
|
||||
}
|
||||
@@ -131,7 +129,7 @@ func TestDoGet_ServerUnreachable(t *testing.T) {
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
}
|
||||
|
||||
rl := newUnbufferedRateLimiter()
|
||||
rl := rate.NewLimiter(rate.Limit(1000), 1000)
|
||||
client := &MusicBrainzClient{
|
||||
httpClient: server.Client(),
|
||||
userAgent: cfg.UserAgent,
|
||||
@@ -140,28 +138,78 @@ func TestDoGet_ServerUnreachable(t *testing.T) {
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
_, err := client.doGet("/test")
|
||||
_, err := client.doGet(context.Background(), "/test")
|
||||
if err == nil {
|
||||
t.Fatal("doGet() expected error for unreachable server, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimiter_BasicBehavior(t *testing.T) {
|
||||
// Test that rate limiter can produce tokens
|
||||
rl := newRateLimiter(1)
|
||||
defer rl.stop()
|
||||
func TestDoGet_ContextCancellation(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`<metadata><test>ok</test></metadata>`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
// Should be able to get a token immediately (bucket was pre-filled)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
rl.wait()
|
||||
close(done)
|
||||
}()
|
||||
cfg := config.MusicBrainzConfig{
|
||||
UserAgent: "naviwatcher/0.1.0 (test@example.com)",
|
||||
}
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// success
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("rateLimiter.wait() blocked on pre-filled bucket")
|
||||
// Use a rate limiter with 0 burst to force blocking on Wait
|
||||
rl := rate.NewLimiter(rate.Limit(0), 0)
|
||||
client := &MusicBrainzClient{
|
||||
httpClient: server.Client(),
|
||||
userAgent: cfg.UserAgent,
|
||||
baseURL: server.URL,
|
||||
rateLimiter: rl,
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel immediately
|
||||
|
||||
_, err := client.doGet(ctx, "/test")
|
||||
if err == nil {
|
||||
t.Fatal("doGet() expected error for cancelled context, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimiter_OnePerSecond(t *testing.T) {
|
||||
// Verify that the rate limiter enforces approximately 1 request per second
|
||||
rl := rate.NewLimiter(rate.Limit(1), 1)
|
||||
|
||||
// First request should be immediate (burst of 1)
|
||||
start := time.Now()
|
||||
if err := rl.Wait(context.Background()); err != nil {
|
||||
t.Fatalf("first Wait() error: %v", err)
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
if elapsed > 100*time.Millisecond {
|
||||
t.Errorf("first Wait() took %v, expected near-instant", elapsed)
|
||||
}
|
||||
|
||||
// Second request should block for approximately 1 second
|
||||
start = time.Now()
|
||||
if err := rl.Wait(context.Background()); err != nil {
|
||||
t.Fatalf("second Wait() error: %v", err)
|
||||
}
|
||||
elapsed = time.Since(start)
|
||||
if elapsed < 800*time.Millisecond {
|
||||
t.Errorf("second Wait() took %v, expected at least ~1s", elapsed)
|
||||
}
|
||||
if elapsed > 2*time.Second {
|
||||
t.Errorf("second Wait() took %v, expected less than 2s", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimiter_BurstBehavior(t *testing.T) {
|
||||
// With burst=1, the first request should be immediate
|
||||
rl := rate.NewLimiter(rate.Limit(1), 1)
|
||||
|
||||
start := time.Now()
|
||||
rl.Wait(context.Background())
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if elapsed > 50*time.Millisecond {
|
||||
t.Errorf("burst Wait() took %v, expected near-instant", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user