feat: implement rate limiting and caching layer for MusicBrainz provider
- Add golang.org/x/time/rate dependency for token-bucket rate limiting - Replace custom channel-based rate limiter with rate.NewLimiter(1, 1) - Add context.Context support to doGet for cancellation - Add cached_at column to external_releases via migration 005 - Implement cache hit/miss queries with TTL-based filtering - Add CacheStats type for tracking cached RGIDs - Update ExternalRelease struct with CachedAt field - Add rate limiting tests (1 req/sec enforcement, burst behavior) - Add cache tests (hit, miss, expired, mixed, empty artist) - Update migration count test for new migration
This commit is contained in:
@@ -1,12 +1,14 @@
|
||||
package musicbrainz
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
"naviwatcher/internal/config"
|
||||
)
|
||||
|
||||
@@ -16,77 +18,26 @@ type MusicBrainzClient struct {
|
||||
httpClient *http.Client
|
||||
userAgent string
|
||||
baseURL string
|
||||
rateLimiter *rateLimiter
|
||||
}
|
||||
|
||||
// rateLimiter wraps a token-bucket rate limiter for API calls.
|
||||
type rateLimiter struct {
|
||||
// tokens is a channel-based semaphore for rate limiting.
|
||||
// It is filled at a fixed interval by a background goroutine.
|
||||
tokens chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// newRateLimiter creates a rate limiter that allows maxCalls per second.
|
||||
// It immediately fills the bucket and starts a refill goroutine.
|
||||
func newRateLimiter(callsPerSecond int) *rateLimiter {
|
||||
rl := &rateLimiter{
|
||||
tokens: make(chan struct{}, callsPerSecond),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
// Fill the bucket initially
|
||||
for i := 0; i < callsPerSecond; i++ {
|
||||
rl.tokens <- struct{}{}
|
||||
}
|
||||
// Refill at the specified interval
|
||||
interval := time.Second / time.Duration(callsPerSecond)
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
select {
|
||||
case rl.tokens <- struct{}{}:
|
||||
default:
|
||||
// bucket full, skip
|
||||
}
|
||||
case <-rl.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return rl
|
||||
}
|
||||
|
||||
// wait blocks until a token is available or the rate limiter is stopped.
|
||||
func (rl *rateLimiter) wait() {
|
||||
<-rl.tokens
|
||||
}
|
||||
|
||||
// stop terminates the refill goroutine.
|
||||
func (rl *rateLimiter) stop() {
|
||||
close(rl.done)
|
||||
rateLimiter *rate.Limiter
|
||||
}
|
||||
|
||||
// NewClient creates a new MusicBrainzClient from the given configuration.
|
||||
// It initializes the HTTP client with a 30-second timeout and sets up
|
||||
// a rate limiter for 1 request per second as required by MusicBrainz policy.
|
||||
func NewClient(cfg config.MusicBrainzConfig) *MusicBrainzClient {
|
||||
rl := newRateLimiter(1) // 1 request per second
|
||||
return &MusicBrainzClient{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
userAgent: cfg.UserAgent,
|
||||
baseURL: "https://musicbrainz.org/ws/2",
|
||||
rateLimiter: rl,
|
||||
rateLimiter: rate.NewLimiter(rate.Limit(1), 1),
|
||||
}
|
||||
}
|
||||
|
||||
// NewClientWithLimiter creates a MusicBrainzClient with a custom rate limiter.
|
||||
// This is primarily used for testing to inject a mock rate limiter.
|
||||
func NewClientWithLimiter(cfg config.MusicBrainzConfig, rl *rateLimiter) *MusicBrainzClient {
|
||||
func NewClientWithLimiter(cfg config.MusicBrainzConfig, rl *rate.Limiter) *MusicBrainzClient {
|
||||
return &MusicBrainzClient{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
@@ -97,17 +48,19 @@ func NewClientWithLimiter(cfg config.MusicBrainzConfig, rl *rateLimiter) *MusicB
|
||||
}
|
||||
}
|
||||
|
||||
// Close cleans up the rate limiter goroutine.
|
||||
func (c *MusicBrainzClient) Close() {
|
||||
c.rateLimiter.stop()
|
||||
}
|
||||
// Close is a no-op for the x/time/rate-based client (the limiter does not
|
||||
// spawn goroutines), but retained for API compatibility.
|
||||
func (c *MusicBrainzClient) Close() {}
|
||||
|
||||
// doGet performs a rate-limited HTTP GET request to the MusicBrainz API.
|
||||
// It sets the proper User-Agent header and returns the response body.
|
||||
func (c *MusicBrainzClient) doGet(path string) ([]byte, error) {
|
||||
c.rateLimiter.wait()
|
||||
// It blocks until the rate limiter allows the request, then sets the proper
|
||||
// User-Agent header and returns the response body.
|
||||
func (c *MusicBrainzClient) doGet(ctx context.Context, path string) ([]byte, error) {
|
||||
if err := c.rateLimiter.Wait(ctx); err != nil {
|
||||
return nil, fmt.Errorf("rate limiter wait: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, c.baseURL+path, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
@@ -152,12 +105,12 @@ type mbArtistCredit struct {
|
||||
// mbReleaseGroup represents the XML structure of a single release-group
|
||||
// in the MusicBrainz release-group list response.
|
||||
type mbReleaseGroup struct {
|
||||
ID string `xml:"id,attr"`
|
||||
Title string `xml:"title"`
|
||||
Type string `xml:"type,attr"`
|
||||
Status string `xml:"status,attr"`
|
||||
ArtistCredit mbArtistCredit `xml:"artist-credit"`
|
||||
ReleaseDate string `xml:"first-release-date"`
|
||||
ID string `xml:"id,attr"`
|
||||
Title string `xml:"title"`
|
||||
Type string `xml:"type,attr"`
|
||||
Status string `xml:"status,attr"`
|
||||
ArtistCredit mbArtistCredit `xml:"artist-credit"`
|
||||
ReleaseDate string `xml:"first-release-date"`
|
||||
}
|
||||
|
||||
// mbReleaseGroupListXML wraps the release-group-list element to properly
|
||||
@@ -169,7 +122,7 @@ type mbReleaseGroupListXML struct {
|
||||
|
||||
// mbReleaseGroupList represents the XML structure of a release-group list response.
|
||||
type mbReleaseGroupList struct {
|
||||
XMLName xml.Name `xml:"metadata"`
|
||||
XMLName xml.Name `xml:"metadata"`
|
||||
ReleaseGroupList mbReleaseGroupListXML `xml:"release-group-list"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user