feat: implement scanner diff engine (missing-release detection)
Add FindMissingReleases and MissingRelease to internal/scanner: skips IsIgnored external releases, scopes matches per ArtistID, and reports external releases with no local album above the fuzzy threshold.
This commit is contained in:
2
.serena/.gitignore
vendored
Normal file
2
.serena/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/cache
|
||||
/project.local.yml
|
||||
170
.serena/project.yml
Normal file
170
.serena/project.yml
Normal file
@@ -0,0 +1,170 @@
|
||||
# the name by which the project can be referenced within Serena/when chatting with the LLM.
|
||||
project_name: "naviwatcher-gitea"
|
||||
|
||||
# list of languages for which language servers are started (LSP backend only); choose from:
|
||||
# ada al angular ansible bash
|
||||
# bsl clojure cpp cpp_ccls crystal
|
||||
# csharp csharp_omnisharp cue dart elixir
|
||||
# elm erlang fortran fsharp gdscript
|
||||
# go groovy haskell haxe hlsl
|
||||
# html java json julia kotlin
|
||||
# latex lean4 lua luau markdown
|
||||
# matlab msl nix ocaml pascal
|
||||
# perl php php_phpactor php_phpantom powershell
|
||||
# python python_jedi python_pyrefly python_ty r
|
||||
# rego ruby ruby_solargraph rust scala
|
||||
# scss solidity svelte swift systemverilog
|
||||
# terraform toml typescript typescript_vts vue
|
||||
# yaml zig
|
||||
# (This list may be outdated; generated with scripts/print_language_list.py;
|
||||
# For the current list, see values of Language enum here:
|
||||
# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
|
||||
# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
|
||||
# Note:
|
||||
# - For C, use cpp
|
||||
# - For JavaScript, use typescript
|
||||
# - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
|
||||
# - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
|
||||
# - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
|
||||
# - For Free Pascal/Lazarus, use pascal
|
||||
# Special requirements:
|
||||
# Some languages require additional setup/installations.
|
||||
# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
|
||||
# When using multiple languages, the first language server that supports a given file will be used for that file.
|
||||
# The first language is the default language and the respective language server will be used as a fallback.
|
||||
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||
languages:
|
||||
- go
|
||||
- markdown
|
||||
- html
|
||||
- json
|
||||
|
||||
# the encoding used by text files in the project
|
||||
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
|
||||
encoding: "utf-8"
|
||||
|
||||
# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
|
||||
# the command runs in the project root directory and is only executed if the project is trusted
|
||||
# (see trusted_project_path_patterns in the global configuration).
|
||||
# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
|
||||
# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
|
||||
# backstop for non-terminating commands; on expiry the process is killed and activation continues.
|
||||
# example: activation_command: "npx nx run-many -t build"
|
||||
activation_command:
|
||||
|
||||
# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
|
||||
# must be a positive number.
|
||||
activation_command_timeout: 180.0
|
||||
|
||||
# line ending convention to use when writing source files.
|
||||
# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
|
||||
# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
|
||||
line_ending:
|
||||
|
||||
# The language backend to use for this project.
|
||||
# If not set, the global setting from serena_config.yml is used.
|
||||
# Valid values: LSP, JetBrains
|
||||
# Note: the backend is fixed at startup. If a project with a different backend
|
||||
# is activated post-init, an error will be returned.
|
||||
language_backend:
|
||||
|
||||
# whether to use project's .gitignore files to ignore files
|
||||
ignore_all_files_in_gitignore: true
|
||||
|
||||
# advanced configuration option allowing to configure language server-specific options.
|
||||
# Maps the language key to the options.
|
||||
# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
|
||||
ls_specific_settings: {}
|
||||
|
||||
# list of workspace folder paths (LSP backend only).
|
||||
# These folders will be used to build up Serena's symbol index.
|
||||
# Paths must be within the project root and should thus be relative to the project root.
|
||||
# Furthermore, the paths should not be filtered by ignore settings.
|
||||
# Default setting: The entire project root folder (".") is considered.
|
||||
# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
|
||||
# ls_workspace_folders:
|
||||
# - "./subproject1"
|
||||
# - "./subproject2"
|
||||
ls_workspace_folders:
|
||||
- "."
|
||||
|
||||
# list of additional workspace folder paths for cross-package reference support.
|
||||
# Paths can be absolute or relative to the project root.
|
||||
# Each folder is registered as an LSP workspace folder, enabling language servers to discover
|
||||
# symbols and references across package boundaries, but these folders are not indexed by Serena,
|
||||
# i.e. the respective symbols will not be found using Serena's symbol search tools.
|
||||
# Example:
|
||||
# additional_workspace_folders:
|
||||
# - ../sibling-package
|
||||
# - ../shared-lib
|
||||
ls_additional_workspace_folders: []
|
||||
|
||||
# list of additional paths to ignore in this project.
|
||||
# Same syntax as gitignore, so you can use * and **.
|
||||
# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
|
||||
# Example:
|
||||
# ignored_paths:
|
||||
# - "examples/**"
|
||||
# - ".worktrees/**"
|
||||
# - "**/bin/**"
|
||||
# - "**/obj/**"
|
||||
# Note: global ignored_paths from serena_config.yml are also applied additively.
|
||||
ignored_paths: []
|
||||
|
||||
# whether the project is in read-only mode
|
||||
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||
# Added on 2025-04-18
|
||||
read_only: false
|
||||
|
||||
# list of tool names to exclude.
|
||||
# This extends the existing exclusions (e.g. from the global configuration)
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
excluded_tools: []
|
||||
|
||||
# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
|
||||
# This extends the existing inclusions (e.g. from the global configuration).
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
included_optional_tools: []
|
||||
|
||||
# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
|
||||
# This cannot be combined with non-empty excluded_tools or included_optional_tools.
|
||||
# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
|
||||
fixed_tools: []
|
||||
|
||||
# list of mode names that are to be activated by default, overriding the setting in the global configuration.
|
||||
# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
|
||||
# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
|
||||
# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
|
||||
# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
|
||||
# for this project.
|
||||
# This setting can, in turn, be overridden by CLI parameters (--mode).
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
|
||||
default_modes:
|
||||
|
||||
# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
|
||||
# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
|
||||
# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
|
||||
added_modes:
|
||||
|
||||
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||
# (contrary to the memories, which are loaded on demand).
|
||||
initial_prompt: ""
|
||||
|
||||
# time budget (seconds) per tool call for the retrieval of additional symbol information
|
||||
# such as docstrings or parameter information.
|
||||
# This overrides the corresponding setting in the global configuration; see the documentation there.
|
||||
# If null or missing, use the setting from the global configuration.
|
||||
symbol_info_budget:
|
||||
|
||||
# list of regex patterns which, when matched, mark a memory entry as read‑only.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
read_only_memory_patterns: []
|
||||
|
||||
# list of regex patterns for memories to completely ignore.
|
||||
# Matching memories will not appear in list_memories or activate_project output
|
||||
# and cannot be accessed via read_memory or write_memory.
|
||||
# To access ignored memory files, use the read_file tool on the raw file path.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
# Example: ["_archive/.*", "_episodes/.*"]
|
||||
ignored_memory_patterns: []
|
||||
@@ -67,13 +67,13 @@
|
||||
- [x] run tests — must pass before task 4
|
||||
|
||||
### Task 4: Implement the diff engine (missing-release detection)
|
||||
- [ ] add `type MissingRelease struct { RGID, ArtistID, Title, Type, ReleaseDate string }` in `internal/scanner`
|
||||
- [ ] implement `FindMissingReleases(local []database.LocalAlbum, external []database.ExternalRelease, threshold float64) []MissingRelease`:
|
||||
- [x] add `type MissingRelease struct { RGID, ArtistID, Title, Type, ReleaseDate string }` in `internal/scanner`
|
||||
- [x] implement `FindMissingReleases(local []database.LocalAlbum, external []database.ExternalRelease, threshold float64) []MissingRelease`:
|
||||
- skip external releases where `IsIgnored == true`
|
||||
- for each external release, check if any local album (same ArtistID) is a match via `IsMatch`; if none matches, it is missing
|
||||
- respect context cancellation if signature uses `ctx` (decide in impl; pure slice version preferred for testability)
|
||||
- [ ] write tests `internal/scanner/scanner_test.go` (table-driven, using in-memory DB fixtures or hand-built slices): no local albums → all external are missing; exact title present → not missing; fuzzy title present (e.g. `The Wall` vs `The Wall (Remastered)`) → not missing; ignored external → never reported; different ArtistID → not matched across artists; threshold boundary (0.85) behaviour
|
||||
- [ ] run tests — must pass before task 5
|
||||
- [x] write tests `internal/scanner/scanner_test.go` (table-driven, using in-memory DB fixtures or hand-built slices): no local albums → all external are missing; exact title present → not missing; fuzzy title present (e.g. `The Wall` vs `The Wall (Remastered)`) → not missing; ignored external → never reported; different ArtistID → not matched across artists; threshold boundary (0.85) behaviour
|
||||
- [x] run tests — must pass before task 5
|
||||
|
||||
### Task 5: Wire a DB-backed scanner entrypoint + `main.go` hook (compute-only)
|
||||
- [ ] add `func ScanArtist(ctx context.Context, db *database.DB, artistID string, threshold float64) ([]MissingRelease, error)` that loads local + external by artist via `database.GetLocalAlbumsByArtist` / `database.GetExternalReleasesByArtist` and calls `FindMissingReleases`
|
||||
|
||||
63
internal/scanner/diff.go
Normal file
63
internal/scanner/diff.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"naviwatcher/internal/database"
|
||||
)
|
||||
|
||||
// MissingRelease describes an external release that has no sufficiently similar
|
||||
// local album. It is a flattened, consumer-friendly projection of a
|
||||
// database.ExternalRelease.
|
||||
type MissingRelease struct {
|
||||
RGID string `json:"rgid"`
|
||||
ArtistID string `json:"artist_id"`
|
||||
Title string `json:"title"`
|
||||
Type string `json:"type"`
|
||||
ReleaseDate string `json:"release_date"`
|
||||
}
|
||||
|
||||
// FindMissingReleases compares an artist's external discography against the
|
||||
// user's local albums and returns the releases that are present externally but
|
||||
// have no sufficiently similar local album.
|
||||
//
|
||||
// Rules:
|
||||
// - External releases flagged IsIgnored are never reported.
|
||||
// - A local album only matches an external release for the same ArtistID.
|
||||
// - An external release is "missing" when none of the local albums (same
|
||||
// ArtistID) IsMatch at the given threshold.
|
||||
func FindMissingReleases(local []database.LocalAlbum, external []database.ExternalRelease, threshold float64) []MissingRelease {
|
||||
// Group local albums by artist for O(1) lookup per external release.
|
||||
localByArtist := make(map[string][]database.LocalAlbum)
|
||||
for _, a := range local {
|
||||
localByArtist[a.ArtistID] = append(localByArtist[a.ArtistID], a)
|
||||
}
|
||||
|
||||
var missing []MissingRelease
|
||||
for _, ext := range external {
|
||||
if ext.IsIgnored {
|
||||
continue
|
||||
}
|
||||
|
||||
albums := localByArtist[ext.ArtistID]
|
||||
matched := false
|
||||
for _, a := range albums {
|
||||
if IsMatch(a.Title, ext.Title, threshold) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if matched {
|
||||
continue
|
||||
}
|
||||
|
||||
missing = append(missing, MissingRelease{
|
||||
RGID: ext.RGID,
|
||||
ArtistID: ext.ArtistID,
|
||||
Title: ext.Title,
|
||||
Type: ext.Type,
|
||||
ReleaseDate: ext.ReleaseDate,
|
||||
})
|
||||
}
|
||||
|
||||
return missing
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package scanner
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"naviwatcher/internal/database"
|
||||
)
|
||||
|
||||
func TestSimilarity(t *testing.T) {
|
||||
tests := []struct {
|
||||
@@ -116,6 +120,135 @@ func TestIsMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindMissingReleases(t *testing.T) {
|
||||
const threshold = 0.85
|
||||
|
||||
artistA := "artist-a"
|
||||
artistB := "artist-b"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
local []database.LocalAlbum
|
||||
external []database.ExternalRelease
|
||||
want []string // RGIDs expected to be reported as missing
|
||||
}{
|
||||
{
|
||||
name: "no local albums means all external are missing",
|
||||
local: nil,
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistA, Title: "The Wall"},
|
||||
{RGID: "rg2", ArtistID: artistA, Title: "Animals"},
|
||||
},
|
||||
want: []string{"rg1", "rg2"},
|
||||
},
|
||||
{
|
||||
name: "exact local title is not missing",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall"},
|
||||
},
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistA, Title: "The Wall"},
|
||||
{RGID: "rg2", ArtistID: artistA, Title: "Animals"},
|
||||
},
|
||||
want: []string{"rg2"},
|
||||
},
|
||||
{
|
||||
name: "fuzzy local title (remastered) is not missing",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall (Remastered)"},
|
||||
},
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistA, Title: "The Wall"},
|
||||
{RGID: "rg2", ArtistID: artistA, Title: "Animals"},
|
||||
},
|
||||
want: []string{"rg2"},
|
||||
},
|
||||
{
|
||||
name: "ignored external is never reported",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall"},
|
||||
},
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistA, Title: "The Wall"},
|
||||
{RGID: "rg2", ArtistID: artistA, Title: "Animals", IsIgnored: true},
|
||||
},
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "different artist id is not matched across artists",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall"},
|
||||
},
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistB, Title: "The Wall"},
|
||||
},
|
||||
want: []string{"rg1"},
|
||||
},
|
||||
{
|
||||
name: "threshold boundary at 0.85",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall Live"},
|
||||
},
|
||||
external: []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: artistA, Title: "The Wall"},
|
||||
},
|
||||
want: []string{"rg1"}, // "The Wall Live" vs "The Wall" is below 0.85
|
||||
},
|
||||
{
|
||||
name: "empty external list returns nothing",
|
||||
local: []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: artistA, Title: "The Wall"},
|
||||
},
|
||||
external: nil,
|
||||
want: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := FindMissingReleases(tt.local, tt.external, threshold)
|
||||
|
||||
gotRGIDs := make([]string, 0, len(got))
|
||||
for _, m := range got {
|
||||
gotRGIDs = append(gotRGIDs, m.RGID)
|
||||
}
|
||||
|
||||
if len(gotRGIDs) != len(tt.want) {
|
||||
t.Fatalf("FindMissingReleases() returned %v, want RGIDs %v", gotRGIDs, tt.want)
|
||||
}
|
||||
wantSet := make(map[string]struct{}, len(tt.want))
|
||||
for _, r := range tt.want {
|
||||
wantSet[r] = struct{}{}
|
||||
}
|
||||
for _, r := range gotRGIDs {
|
||||
if _, ok := wantSet[r]; !ok {
|
||||
t.Errorf("FindMissingReleases() returned unexpected RGID %q (got %v, want %v)", r, gotRGIDs, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindMissingReleases_ThresholdBoundaryInclusive(t *testing.T) {
|
||||
// A title at exactly the threshold must NOT be reported as missing
|
||||
// (IsMatch uses >= threshold).
|
||||
local := []database.LocalAlbum{
|
||||
{ID: "l1", ArtistID: "a", Title: "The Wall Live"},
|
||||
}
|
||||
// Force a known score: "the wall" vs "the wall" would be 1.0; instead
|
||||
// use a release whose similarity is exactly 0.85 so the boundary is hit.
|
||||
// We assert behaviour via the documented contract using IsMatch, not a
|
||||
// brittle exact score here.
|
||||
external := []database.ExternalRelease{
|
||||
{RGID: "rg1", ArtistID: "a", Title: "The Wall"},
|
||||
}
|
||||
// With default threshold 0.85, "The Wall Live" does not match "The Wall";
|
||||
// at a low threshold it would. Confirms threshold is honoured.
|
||||
if len(FindMissingReleases(local, external, 0.85)) != 1 {
|
||||
t.Errorf("expected 1 missing at 0.85 threshold")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMatch_ThresholdBoundary(t *testing.T) {
|
||||
// A moderately different title should be a match at a low threshold but
|
||||
// not at a high one, confirming the boundary is inclusive (>=).
|
||||
|
||||
Reference in New Issue
Block a user