diff --git a/cmd/naviwatcher/fuzzy_smoke_test.go b/cmd/naviwatcher/fuzzy_smoke_test.go new file mode 100644 index 0000000..d108168 --- /dev/null +++ b/cmd/naviwatcher/fuzzy_smoke_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" + + "github.com/lithammer/fuzzysearch/fuzzy" +) + +// TestFuzzySmoke verifies the fuzzysearch dependency is importable and that +// its ranking API behaves as the scanner engine will expect. +// +// Note: this library does NOT expose a `fuzzy.Ratio` (0-100) function as the +// plan's Technical Details assumed. The relevant signal here is RankMatch, +// which returns 0 for an exact match, a small positive distance for near +// matches, and -1 when source is not a subsequence of target. Task 3 will +// convert this into a normalized 0.0-1.0 similarity score. +func TestFuzzySmoke(t *testing.T) { + // Exact match scores 0 (distance). + if got := fuzzy.RankMatch("the wall", "the wall"); got != 0 { + t.Errorf("expected RankMatch of identical strings to be 0, got %d", got) + } + + // Similar strings score closer to 0 than dissimilar ones, and a real + // subsequence match returns a non-negative distance. + similar := fuzzy.RankMatch("the wall", "the wall remastered") + dissimilar := fuzzy.RankMatch("the wall", "completely different album") + + if similar < 0 { + t.Errorf("expected similar to be a valid match (>=0), got %d", similar) + } + if dissimilar >= 0 { + t.Errorf("expected dissimilar to be a non-match (-1), got %d", dissimilar) + } + if dissimilar != -1 { + t.Errorf("expected dissimilar to be -1 (no subsequence match), got %d", dissimilar) + } + + // A near match (valid, >=0) is preferable to a total miss (-1). + if similar < 0 { + t.Errorf("expected similar to be a valid match (>=0), got %d", similar) + } + if dissimilar != -1 { + t.Errorf("expected dissimilar to be a non-match (-1), got %d", dissimilar) + } +} diff --git a/docs/plans/2026-07-19-scanner-engine-fuzzy-diff.md b/docs/plans/2026-07-19-scanner-engine-fuzzy-diff.md new file mode 100644 index 0000000..167bd07 --- /dev/null +++ b/docs/plans/2026-07-19-scanner-engine-fuzzy-diff.md @@ -0,0 +1,118 @@ +# Scanner Engine: Fuzzy Diff (local vs external releases) + +## Overview +- Implement the **Scanner Engine** — the missing core of NaviWatcher. It compares a user's local albums (from Navidrome, stored in `local_albums`) against an artist's external discography (from MusicBrainz, stored in `external_releases`) and returns the list of **missing releases** (external releases with no sufficiently similar local album). +- Problem solved: without this, `main.run()` is empty and the service cannot fulfil its stated purpose (find missing albums and notify). This plan delivers only the computation core; persistence, notifier, and Web UI are explicitly out of scope. +- Integrates with existing data layer: reads `database.LocalAlbum` and `database.ExternalRelease`, reuses normalization logic, and consumes `config.Scanner.FuzzyThreshold` (default 0.85). + +## Context (from discovery) +- Files/components involved: + - `internal/database/local_albums.go` — `LocalAlbum{ID, ArtistID, Title}`, accessors `GetLocalAlbumsByArtist`, `GetAllLocalAlbums`, `GetLocalAlbums` (to confirm names during impl). + - `internal/database/external_releases.go` — `ExternalRelease{RGID, ArtistID, Title, Type, ReleaseDate, IsIgnored, CachedAt}`, accessors `GetExternalReleasesByArtist`, `GetIgnoredReleases`. + - `internal/database/database.go:176-191` — struct definitions. + - `internal/config/config.go:50-54` — `ScannerConfig{FuzzyThreshold, IgnoreBootlegs, IncludeCompilations}`. + - `internal/musicbrainz/api.go:132-165` — existing `NormalizeString` / `NormalizeArtistName` (regexes precompiled at init). + - `cmd/naviwatcher/main.go` — `App` struct, `NewApp`, empty `run()`. + - `go.mod` — **no fuzzy library present**; `lithammer/fuzzysearch` must be added. +- Related patterns found: + - MusicBrainz provider uses `ctx.Err()` checks before/within loops, `fmt.Errorf("...: %w", err)` wrapping, `db.Begin()`/`defer tx.Rollback()`/`tx.Commit()`, and table-driven white-box tests with `newTestDB(t, ":memory:")` + `seedArtist` fixtures. + - Existing `NormalizeString` already covers: lowercase, strip `[...]`/`(...)`, strip years `(1|2)xxx`, strip non-alphanumerics, collapse spaces. Bracket stripping removes keywords like Deluxe/Anniversary/Expanded regardless of a keyword list. +- Dependencies identified: + - New dep: `github.com/lithammer/fuzzysearch` (specified in Specification.md §2). + - New package: `internal/normalize` (extracted from `musicbrainz.NormalizeString`). + - New package: `internal/scanner` (the engine). + +## Development Approach +- **Testing approach**: TDD — write/extend tests alongside every task's code. +- Complete each task fully (code + tests passing) before moving to the next. +- **CRITICAL: every task MUST include new/updated tests** for code changes in that task (success + error/edge cases). +- **CRITICAL: all tests must pass before starting next task** — no exceptions. +- Update this plan file when scope changes; mark items `[x]` immediately on completion. +- Reuse existing test helpers (`newTestDB`, `seedArtist`, table-driven style) and the same error-wrapping/context conventions. + +## Testing Strategy +- **Unit tests** (required per task): normalization, similarity scoring, and the scanner diff are all pure functions — ideal for table-driven tests with no DB. Scanner diff against DB uses `:memory:` SQLite + `seedArtist` fixtures, mirroring `musicbrainz/sync_test.go`. +- No UI/e2e in this plan (Web UI is out of scope). + +## Progress Tracking +- Mark completed items with `[x]` immediately when done. +- Add newly discovered tasks with ➕ prefix. +- Document blockers with ⚠️ prefix. +- Keep plan in sync with actual work done. + +## What Goes Where +- **Implementation Steps** (`[ ]`): all code + test tasks below. +- **Post-Completion** (no checkboxes): manual/integration verification notes. + +## Implementation Steps + +### Task 1: Add fuzzysearch dependency +- [x] run `go get github.com/lithammer/fuzzysearch@latest` and confirm it appears in `go.mod`/`go.sum` +- [x] run `go mod tidy` and verify the build still compiles (`go build ./...`) +- [x] write a trivial smoke test (or rely on Task 3's first test) confirming `fuzzy.Ratio` is importable and returns expected ordering +- [x] run tests — must pass before task 2 + +> Note: the library exposes `fuzzy.RankMatch` (subsequence-ranked Levenshtein distance: 0=exact, -1=no match) rather than a `fuzzy.Ratio` 0-100 function assumed in the plan. Task 3 will normalize this into a 0.0-1.0 similarity score. + +### Task 2: Extract normalization into `internal/normalize` +- [ ] create `internal/normalize/normalize.go` with `NormalizeString(s string) string` and `NormalizeArtistName(s string) string`, moving the regexes + logic from `internal/musicbrainz/api.go:132-165` +- [ ] refactor `internal/musicbrainz/api.go` to call `normalize.NormalizeString` / `normalize.NormalizeArtistName` instead of its local copies (remove duplicated regexes/functions) +- [ ] write tests `internal/normalize/normalize_test.go` (table-driven): lowercase, bracket/paren strip, year strip `(20xx)`, special-char strip, space collapse, `NormalizeArtistName` prefix strip (`the `/`a `/`an `) +- [ ] update existing `internal/musicbrainz/api_test.go` if it referenced the moved functions, ensuring it still passes +- [ ] run tests — must pass before task 3 + +### Task 3: Implement similarity scoring in `internal/scanner` +- [ ] create `internal/scanner/scanner.go` with `Similarity(a, b string) float64` using `normalize.NormalizeString` + `fuzzy.Ratio` (normalized to 0.0–1.0); define `IsMatch(a, b string, threshold float64) bool` +- [ ] write tests `internal/scanner/scanner_test.go` (table-driven): exact match → 1.0, `(Remastered)` / year variants still match above 0.85, clearly different titles → below threshold, empty-string handling +- [ ] run tests — must pass before task 4 + +### Task 4: Implement the diff engine (missing-release detection) +- [ ] add `type MissingRelease struct { RGID, ArtistID, Title, Type, ReleaseDate string }` in `internal/scanner` +- [ ] implement `FindMissingReleases(local []database.LocalAlbum, external []database.ExternalRelease, threshold float64) []MissingRelease`: + - skip external releases where `IsIgnored == true` + - for each external release, check if any local album (same ArtistID) is a match via `IsMatch`; if none matches, it is missing + - respect context cancellation if signature uses `ctx` (decide in impl; pure slice version preferred for testability) +- [ ] write tests `internal/scanner/scanner_test.go` (table-driven, using in-memory DB fixtures or hand-built slices): no local albums → all external are missing; exact title present → not missing; fuzzy title present (e.g. `The Wall` vs `The Wall (Remastered)`) → not missing; ignored external → never reported; different ArtistID → not matched across artists; threshold boundary (0.85) behaviour +- [ ] run tests — must pass before task 5 + +### Task 5: Wire a DB-backed scanner entrypoint + `main.go` hook (compute-only) +- [ ] add `func ScanArtist(ctx context.Context, db *database.DB, artistID string, threshold float64) ([]MissingRelease, error)` that loads local + external by artist via `database.GetLocalAlbumsByArtist` / `database.GetExternalReleasesByArtist` and calls `FindMissingReleases` +- [ ] add `func ScanAll(ctx context.Context, db *database.DB, threshold float64) ([]MissingRelease, error)` iterating monitored artists (reuse `database.GetAllArtistSettings`) with `ctx.Err()` checks between artists +- [ ] write tests `internal/scanner/scan_test.go` using `newTestDB(t)` + `seedArtist` + seeded `local_albums`/`external_releases` rows; assert missing set matches expectations; test ctx-cancellation returns early +- [ ] extend `cmd/naviwatcher/main.go` `App` struct + `NewApp` to construct the scanner (or keep stateless) and add a compute-only call in `run()` (e.g. log count of missing releases for monitored artists) without starting notifier/web — keep `run()` non-blocking / goroutine-safe per spec concurrency note +- [ ] write/extend `cmd/naviwatcher/main_test.go` if `App`/wiring changed +- [ ] run full test suite (`go test ./...`) and `go vet ./...` — must pass before final task + +### Task 6: Verify acceptance criteria +- [ ] verify `FindMissingReleases`/`ScanArtist`/`ScanAll` meet spec: normalization + 0.85 fuzzy threshold, ignore `IsIgnored`, per-artist scoping +- [ ] verify `config.Scanner.FuzzyThreshold` default 0.85 is used when threshold arg is zero (or document the chosen contract) +- [ ] run full test suite (unit) — all green +- [ ] run `go vet ./...` and `gofmt -l ./internal ./cmd` — zero issues +- [ ] verify test coverage of `internal/scanner` and `internal/normalize` (target 80%+) + +### Task 7: Update documentation +- [ ] update `README.md` to note the Scanner Engine is implemented (compute-only; notifier/web pending) +- [ ] add a short note in `CLAUDE.md` or a plan-completion comment if new package conventions (e.g. `internal/normalize` is the shared normalization home) were established + +## Technical Details +- **Normalization** (`internal/normalize`): port regexes from `musicbrainz/api.go`: + - `bracketRe` = `\[[^\]]*\]` , `parenRe` = `\([^)]*\)`, `yearRe` = `\b(1[0-9]{3}|2[0-9]{3})\b`, `spaceRe` = `\s+`. + - `NormalizeString`: lowercase → strip brackets/parens → strip years → replace `-`/`_` with space → keep `[a-z0-9 ]` → collapse spaces → trim. + - `NormalizeArtistName`: `NormalizeString` then strip leading `the `/`a `/`an ` tokens. +- **Similarity**: `fuzzy.Ratio(normalize(a), normalize(b))` returns an int 0–100; `Similarity` returns `float64(ratio)/100.0`. `IsMatch` returns `Similarity(a,b,threshold) >= threshold`. +- **Diff algorithm**: per external release (filtered by `!IsIgnored`), compare normalized title against each local album of the same `ArtistID`; missing if no `IsMatch` at `threshold`. +- **Config contract**: `threshold` passed from `config.Scanner.FuzzyThreshold` (default 0.85). Decide: if caller passes `0`, use default — implement explicitly and document. +- **No new DB tables** in this plan (compute-only per user decision). + +## Post-Completion +*Informational only — no checkboxes.* + +**Manual verification** (optional, requires live Navidrome + MusicBrainz cache): +- Run the binary with a real `config.yaml`, observe `run()` log line reporting missing-release counts for monitored artists. +- Confirm no false positives for `(Remastered)` / year-suffixed local titles. + +**Follow-up (out of scope, future plans)**: +- Persist `MissingRelease` into a new `missing_releases` table for notifier/Web UI. +- Implement Notifier (Telegram bot + cron) consuming scanner output. +- Implement Web UI (dashboard / artist / archive) with basic-auth and `//go:embed` templates. +- Replace the compute-only `run()` hook with full goroutine orchestration (scanner + notifier + web). diff --git a/go.mod b/go.mod index 1847b97..f6cb38b 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) -require golang.org/x/time v0.15.0 +require ( + github.com/lithammer/fuzzysearch v1.1.8 + golang.org/x/time v0.15.0 +) + +require golang.org/x/text v0.9.0 // indirect diff --git a/go.sum b/go.sum index 0630cee..b7a8762 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,43 @@ github.com/delucks/go-subsonic v0.0.0-20240806025900-2a743ec36238 h1:uejyepOdHISrJTw7P84Y7yEC0FMyv1q3KNDRxWsviKw= github.com/delucks/go-subsonic v0.0.0-20240806025900-2a743ec36238/go.mod h1:vnbEuj6Z20PLcHB4rrLQAOXGMjtULfMGhRVSFPcSdUo= +github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= +github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=