feat: implement scanner similarity scoring
This commit is contained in:
@@ -62,9 +62,9 @@
|
|||||||
- [x] run tests — must pass before task 3
|
- [x] run tests — must pass before task 3
|
||||||
|
|
||||||
### Task 3: Implement similarity scoring in `internal/scanner`
|
### Task 3: Implement similarity scoring in `internal/scanner`
|
||||||
- [ ] create `internal/scanner/scanner.go` with `Similarity(a, b string) float64` using `normalize.NormalizeString` + `fuzzy.Ratio` (normalized to 0.0–1.0); define `IsMatch(a, b string, threshold float64) bool`
|
- [x] create `internal/scanner/scanner.go` with `Similarity(a, b string) float64` using `normalize.NormalizeString` + `fuzzy.LevenshteinDistance` (normalized to 0.0–1.0); define `IsMatch(a, b string, threshold float64) bool`
|
||||||
- [ ] write tests `internal/scanner/scanner_test.go` (table-driven): exact match → 1.0, `(Remastered)` / year variants still match above 0.85, clearly different titles → below threshold, empty-string handling
|
- [x] write tests `internal/scanner/scanner_test.go` (table-driven): exact match → 1.0, `(Remastered)` / year variants still match above 0.85, clearly different titles → below threshold, empty-string handling
|
||||||
- [ ] run tests — must pass before task 4
|
- [x] run tests — must pass before task 4
|
||||||
|
|
||||||
### Task 4: Implement the diff engine (missing-release detection)
|
### Task 4: Implement the diff engine (missing-release detection)
|
||||||
- [ ] add `type MissingRelease struct { RGID, ArtistID, Title, Type, ReleaseDate string }` in `internal/scanner`
|
- [ ] add `type MissingRelease struct { RGID, ArtistID, Title, Type, ReleaseDate string }` in `internal/scanner`
|
||||||
@@ -99,7 +99,7 @@
|
|||||||
- `bracketRe` = `\[[^\]]*\]` , `parenRe` = `\([^)]*\)`, `yearRe` = `\b(1[0-9]{3}|2[0-9]{3})\b`, `spaceRe` = `\s+`.
|
- `bracketRe` = `\[[^\]]*\]` , `parenRe` = `\([^)]*\)`, `yearRe` = `\b(1[0-9]{3}|2[0-9]{3})\b`, `spaceRe` = `\s+`.
|
||||||
- `NormalizeString`: lowercase → strip brackets/parens → strip years → replace `-`/`_` with space → keep `[a-z0-9 ]` → collapse spaces → trim.
|
- `NormalizeString`: lowercase → strip brackets/parens → strip years → replace `-`/`_` with space → keep `[a-z0-9 ]` → collapse spaces → trim.
|
||||||
- `NormalizeArtistName`: `NormalizeString` then strip leading `the `/`a `/`an ` tokens.
|
- `NormalizeArtistName`: `NormalizeString` then strip leading `the `/`a `/`an ` tokens.
|
||||||
- **Similarity**: `fuzzy.Ratio(normalize(a), normalize(b))` returns an int 0–100; `Similarity` returns `float64(ratio)/100.0`. `IsMatch` returns `Similarity(a,b,threshold) >= threshold`.
|
- **Similarity**: `fuzzy.LevenshteinDistance(normalize(a), normalize(b))` returns an int edit distance; `Similarity` returns `1.0 - float64(dist)/float64(maxLen)` (clamped to [0.0, 1.0]). Empty/whitespace-only inputs normalize to empty and score 0.0 (no false match). `IsMatch` returns `Similarity(a,b) >= threshold`. Note: `fuzzy.Ratio` does not exist in `lithammer/fuzzysearch` v1.1.8 — `LevenshteinDistance` is used instead (contrary to earlier plan assumption).
|
||||||
- **Diff algorithm**: per external release (filtered by `!IsIgnored`), compare normalized title against each local album of the same `ArtistID`; missing if no `IsMatch` at `threshold`.
|
- **Diff algorithm**: per external release (filtered by `!IsIgnored`), compare normalized title against each local album of the same `ArtistID`; missing if no `IsMatch` at `threshold`.
|
||||||
- **Config contract**: `threshold` passed from `config.Scanner.FuzzyThreshold` (default 0.85). Decide: if caller passes `0`, use default — implement explicitly and document.
|
- **Config contract**: `threshold` passed from `config.Scanner.FuzzyThreshold` (default 0.85). Decide: if caller passes `0`, use default — implement explicitly and document.
|
||||||
- **No new DB tables** in this plan (compute-only per user decision).
|
- **No new DB tables** in this plan (compute-only per user decision).
|
||||||
|
|||||||
56
internal/scanner/scanner.go
Normal file
56
internal/scanner/scanner.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
// Package scanner implements the core fuzzy-diff engine of NaviWatcher.
|
||||||
|
//
|
||||||
|
// It compares a user's local albums (from Navidrome) against an artist's
|
||||||
|
// external discography (from MusicBrainz) and reports the releases that are
|
||||||
|
// present externally but have no sufficiently similar local album.
|
||||||
|
package scanner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/lithammer/fuzzysearch/fuzzy"
|
||||||
|
"naviwatcher/internal/normalize"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Similarity returns a normalized similarity score in the range [0.0, 1.0]
|
||||||
|
// between two strings. The strings are normalized first (lowercased,
|
||||||
|
// bracketed/parenthesized content and years stripped, special characters
|
||||||
|
// removed), then compared with a Levenshtein-distance-based ratio.
|
||||||
|
//
|
||||||
|
// A score of 1.0 means the normalized strings are identical; 0.0 means they
|
||||||
|
// share nothing. Empty strings (after normalization) always score 0.0.
|
||||||
|
func Similarity(a, b string) float64 {
|
||||||
|
na := normalize.NormalizeString(a)
|
||||||
|
nb := normalize.NormalizeString(b)
|
||||||
|
|
||||||
|
// Two empty inputs are not considered a match.
|
||||||
|
if na == "" && nb == "" {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
// One empty, one non-empty: no similarity.
|
||||||
|
if na == "" || nb == "" {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
dist := fuzzy.LevenshteinDistance(na, nb)
|
||||||
|
maxLen := len(na)
|
||||||
|
if len(nb) > maxLen {
|
||||||
|
maxLen = len(nb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guard against maxLen == 0 (already handled above, but kept for safety).
|
||||||
|
if maxLen == 0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1.0 - normalized distance → higher is more similar.
|
||||||
|
score := 1.0 - float64(dist)/float64(maxLen)
|
||||||
|
if score < 0.0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
return score
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsMatch reports whether a and b are similar enough to be considered the
|
||||||
|
// same release, given the provided threshold in [0.0, 1.0].
|
||||||
|
func IsMatch(a, b string, threshold float64) bool {
|
||||||
|
return Similarity(a, b) >= threshold
|
||||||
|
}
|
||||||
131
internal/scanner/scanner_test.go
Normal file
131
internal/scanner/scanner_test.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package scanner
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestSimilarity(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
a string
|
||||||
|
b string
|
||||||
|
want float64
|
||||||
|
epsilon float64
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "exact match scores 1.0",
|
||||||
|
a: "The Wall",
|
||||||
|
b: "The Wall",
|
||||||
|
want: 1.0,
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "case-insensitive exact match scores 1.0",
|
||||||
|
a: "The Wall",
|
||||||
|
b: "the wall",
|
||||||
|
want: 1.0,
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "remastered variant stays above threshold",
|
||||||
|
a: "The Wall",
|
||||||
|
b: "The Wall (Remastered)",
|
||||||
|
want: 1.0, // parenthesized content is stripped during normalization
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "year-suffixed variant stays above threshold",
|
||||||
|
a: "Abbey Road",
|
||||||
|
b: "Abbey Road (2019 Remix)",
|
||||||
|
// After normalization both collapse to "abbey road" → identical.
|
||||||
|
want: 1.0,
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "clearly different titles score below 0.85",
|
||||||
|
a: "The Wall",
|
||||||
|
b: "Completely Different Album",
|
||||||
|
want: 0.0,
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "substring-ish title scores moderately",
|
||||||
|
a: "Dark Side of the Moon",
|
||||||
|
b: "Dark Side of the Moon Part II",
|
||||||
|
want: 0.0, // non-empty; value asserted only as below threshold
|
||||||
|
epsilon: 1e-9,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := Similarity(tt.a, tt.b)
|
||||||
|
switch {
|
||||||
|
case tt.name == "clearly different titles score below 0.85" ||
|
||||||
|
tt.name == "substring-ish title scores moderately":
|
||||||
|
if got >= 0.85 {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want < 0.85", tt.a, tt.b, got)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if diff := got - tt.want; diff > tt.epsilon || diff < -tt.epsilon {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want %v (+/- %v)", tt.a, tt.b, got, tt.want, tt.epsilon)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSimilarity_EmptyStrings(t *testing.T) {
|
||||||
|
// Both empty → no match (0.0).
|
||||||
|
if got := Similarity("", ""); got != 0.0 {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want 0.0", "", "", got)
|
||||||
|
}
|
||||||
|
// One empty, one non-empty → no similarity.
|
||||||
|
if got := Similarity("The Wall", ""); got != 0.0 {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want 0.0", "The Wall", "", got)
|
||||||
|
}
|
||||||
|
if got := Similarity("", "The Wall"); got != 0.0 {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want 0.0", "", "The Wall", got)
|
||||||
|
}
|
||||||
|
// Whitespace-only inputs normalize to empty → no match.
|
||||||
|
if got := Similarity(" ", "The Wall"); got != 0.0 {
|
||||||
|
t.Errorf("Similarity(%q, %q) = %v, want 0.0", " ", "The Wall", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsMatch(t *testing.T) {
|
||||||
|
const threshold = 0.85
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
a string
|
||||||
|
b string
|
||||||
|
expected bool
|
||||||
|
}{
|
||||||
|
{name: "exact match is a match", a: "The Wall", b: "The Wall", expected: true},
|
||||||
|
{name: "remastered variant is a match", a: "The Wall", b: "The Wall (Remastered)", expected: true},
|
||||||
|
{name: "year variant is a match", a: "Abbey Road", b: "Abbey Road (2019)", expected: true},
|
||||||
|
{name: "clearly different is not a match", a: "The Wall", b: "Random Noise", expected: false},
|
||||||
|
{name: "empty vs non-empty is not a match", a: "", b: "The Wall", expected: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := IsMatch(tt.a, tt.b, threshold); got != tt.expected {
|
||||||
|
t.Errorf("IsMatch(%q, %q, %v) = %v, want %v", tt.a, tt.b, threshold, got, tt.expected)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsMatch_ThresholdBoundary(t *testing.T) {
|
||||||
|
// A moderately different title should be a match at a low threshold but
|
||||||
|
// not at a high one, confirming the boundary is inclusive (>=).
|
||||||
|
a, b := "The Wall", "The Wall Live"
|
||||||
|
low := IsMatch(a, b, 0.5)
|
||||||
|
high := IsMatch(a, b, 0.99)
|
||||||
|
if !low {
|
||||||
|
t.Errorf("IsMatch(%q, %q, 0.5) = false, want true", a, b)
|
||||||
|
}
|
||||||
|
if high {
|
||||||
|
t.Errorf("IsMatch(%q, %q, 0.99) = true, want false", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user