Mrixs/NaviWatcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement rate limiting and caching layer for MusicBrainz provider

Browse Source 
- Add golang.org/x/time/rate dependency for token-bucket rate limiting
- Replace custom channel-based rate limiter with rate.NewLimiter(1, 1)
- Add context.Context support to doGet for cancellation
- Add cached_at column to external_releases via migration 005
- Implement cache hit/miss queries with TTL-based filtering
- Add CacheStats type for tracking cached RGIDs
- Update ExternalRelease struct with CachedAt field
- Add rate limiting tests (1 req/sec enforcement, burst behavior)
- Add cache tests (hit, miss, expired, mixed, empty artist)
- Update migration count test for new migration
This commit is contained in:
Vladimir Zagainov
2026-05-26 12:15:28 +03:00
parent 674daed93b
commit b0f69d3a4f
10 changed files with 518 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/database/database.go
View File

@@ -115,6 +115,10 @@ func (db *DB) migrate() error {
PRIMARY KEY (rgid, sent_at)
);`,
},
{
name: "005_add_cached_at_to_external_releases",
sql: `ALTER TABLE external_releases ADD COLUMN cached_at DATETIME;`,
},
}
for _, m := range migrations {
@@ -177,12 +181,13 @@ type LocalAlbum struct {
// ExternalRelease represents a row in the external_releases table.
type ExternalRelease struct {
RGID string `json:"rgid"`
ArtistID string `json:"artist_id"`
Title string `json:"title"`
Type string `json:"type"`
ReleaseDate string `json:"release_date"`
IsIgnored bool `json:"is_ignored"`
RGID string `json:"rgid"`
ArtistID string `json:"artist_id"`
Title string `json:"title"`
Type string `json:"type"`
ReleaseDate string `json:"release_date"`
IsIgnored bool `json:"is_ignored"`
CachedAt time.Time `json:"cached_at"`
}
// NotificationSent represents a row in the notifications_sent table.

6
internal/database/database_test.go
View File

@@ -205,8 +205,8 @@ func TestMigrationTracking(t *testing.T) {
t.Fatalf("query migrations count: %v", err)
}
// We have 4 recorded migrations: artist_settings, external_releases, local_albums, notifications_sent.
if count != 4 {
t.Errorf("expected 4 applied migrations, got %d", count)
// We have 5 recorded migrations: artist_settings, external_releases, local_albums, notifications_sent, cached_at column.
if count != 5 {
t.Errorf("expected 5 applied migrations, got %d", count)
}
}

97
internal/database/external_releases.go
View File

@@ -1,28 +1,40 @@
package database
import (
"database/sql"
"fmt"
"time"
_ "github.com/mattn/go-sqlite3"
)
// GetExternalRelease retrieves an external_release row by RGID.
// Returns sql.ErrNoRows if the release is not found.
func GetExternalRelease(db *DB, rgid string) (*ExternalRelease, error) {
var r ExternalRelease
var cachedAt sql.NullTime
err := db.Conn().QueryRow(
"SELECT rgid, artist_id, title, type, release_date, is_ignored FROM external_releases WHERE rgid = ?",
"SELECT rgid, artist_id, title, type, release_date, is_ignored, cached_at FROM external_releases WHERE rgid = ?",
rgid,
).Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored)
).Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored, &cachedAt)
if err != nil {
return nil, err
}
if cachedAt.Valid {
r.CachedAt = cachedAt.Time
}
return &r, nil
}
// SaveExternalRelease inserts or replaces an external_release row.
func SaveExternalRelease(db *DB, release *ExternalRelease) error {
cachedAtStr := ""
if !release.CachedAt.IsZero() {
cachedAtStr = release.CachedAt.Format("2006-01-02 15:04:05")
}
_, err := db.Conn().Exec(
"INSERT OR REPLACE INTO external_releases (rgid, artist_id, title, type, release_date, is_ignored) VALUES (?, ?, ?, ?, ?, ?)",
release.RGID, release.ArtistID, release.Title, release.Type, release.ReleaseDate, release.IsIgnored,
"INSERT OR REPLACE INTO external_releases (rgid, artist_id, title, type, release_date, is_ignored, cached_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
release.RGID, release.ArtistID, release.Title, release.Type, release.ReleaseDate, release.IsIgnored, cachedAtStr,
)
if err != nil {
return fmt.Errorf("save external release: %w", err)
@@ -33,7 +45,7 @@ func SaveExternalRelease(db *DB, release *ExternalRelease) error {
// GetExternalReleasesByArtist returns all external_release rows for a given artist_id.
func GetExternalReleasesByArtist(db *DB, artistID string) ([]ExternalRelease, error) {
rows, err := db.Conn().Query(
"SELECT rgid, artist_id, title, type, release_date, is_ignored FROM external_releases WHERE artist_id = ?",
"SELECT rgid, artist_id, title, type, release_date, is_ignored, cached_at FROM external_releases WHERE artist_id = ?",
artistID,
)
if err != nil {
@@ -44,9 +56,13 @@ func GetExternalReleasesByArtist(db *DB, artistID string) ([]ExternalRelease, er
var results []ExternalRelease
for rows.Next() {
var r ExternalRelease
if err := rows.Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored); err != nil {
var cachedAt sql.NullTime
if err := rows.Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored, &cachedAt); err != nil {
return nil, fmt.Errorf("scan external release: %w", err)
}
if cachedAt.Valid {
r.CachedAt = cachedAt.Time
}
results = append(results, r)
}
if err := rows.Err(); err != nil {
@@ -58,7 +74,7 @@ func GetExternalReleasesByArtist(db *DB, artistID string) ([]ExternalRelease, er
// GetIgnoredReleases returns all external_release rows where is_ignored = 1.
func GetIgnoredReleases(db *DB) ([]ExternalRelease, error) {
rows, err := db.Conn().Query(
"SELECT rgid, artist_id, title, type, release_date, is_ignored FROM external_releases WHERE is_ignored = 1",
"SELECT rgid, artist_id, title, type, release_date, is_ignored, cached_at FROM external_releases WHERE is_ignored = 1",
)
if err != nil {
return nil, fmt.Errorf("query ignored releases: %w", err)
@@ -68,9 +84,13 @@ func GetIgnoredReleases(db *DB) ([]ExternalRelease, error) {
var results []ExternalRelease
for rows.Next() {
var r ExternalRelease
if err := rows.Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored); err != nil {
var cachedAt sql.NullTime
if err := rows.Scan(&r.RGID, &r.ArtistID, &r.Title, &r.Type, &r.ReleaseDate, &r.IsIgnored, &cachedAt); err != nil {
return nil, fmt.Errorf("scan ignored release: %w", err)
}
if cachedAt.Valid {
r.CachedAt = cachedAt.Time
}
results = append(results, r)
}
if err := rows.Err(); err != nil {
@@ -99,3 +119,64 @@ func SetReleaseIgnored(db *DB, rgid string, ignored bool) error {
return nil
}
// DeleteExternalReleasesByArtist removes all external_release rows for a given artist_id.
func DeleteExternalReleasesByArtist(db *DB, artistID string) error {
_, err := db.Conn().Exec(
"DELETE FROM external_releases WHERE artist_id = ?",
artistID,
)
if err != nil {
return fmt.Errorf("delete external releases by artist: %w", err)
}
return nil
}
// CountExternalReleases returns the total number of external_release rows.
func CountExternalReleases(db *DB) (int, error) {
var count int
err := db.Conn().QueryRow("SELECT COUNT(*) FROM external_releases").Scan(&count)
if err != nil {
return 0, fmt.Errorf("count external releases: %w", err)
}
return count, nil
}
// GetExternalReleasesByArtistWithCache returns cached external_release rows for a given artist_id
// that are within the specified TTL.
func GetExternalReleasesByArtistWithCache(db *DB, artistID string, ttl time.Duration) ([]ExternalRelease, error) {
cutoff := time.Now().Add(-ttl)
rows, err := db.Conn().Query(
"SELECT rgid, artist_id, title, type, release_date, is_ignored, cached_at FROM external_releases WHERE artist_id = ? AND cached_at >= ?",
artistID, cutoff.Format("2006-01-02 15:04:05"),
)
if err != nil {
return nil, fmt.Errorf("query cached external releases: %w", err)
}
defer rows.Close()
var results []ExternalRelease
for rows.Next() {
var r ExternalRelease
var releaseDate sql.NullString
var releaseType sql.NullString
var cachedAt sql.NullTime
if err := rows.Scan(&r.RGID, &r.ArtistID, &r.Title, &releaseType, &releaseDate, &r.IsIgnored, &cachedAt); err != nil {
return nil, fmt.Errorf("scan cached external release: %w", err)
}
if releaseType.Valid {
r.Type = releaseType.String
}
if releaseDate.Valid {
r.ReleaseDate = releaseDate.String
}
if cachedAt.Valid {
r.CachedAt = cachedAt.Time
}
results = append(results, r)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate cached external releases: %w", err)
}
return results, nil
}