Files
ym-dl-watcher/internal/storage/storage.go
T
Mrixs 5dcdd92d40
CI/CD / test (push) Successful in 47s
CI/CD / build (push) Successful in 1m0s
CI/CD / docker (push) Successful in 2m36s
Update yandex-music-api-go and fix Artist.ID JSONID conversion
Library updated to v0.0.0-20261018080126-dbb823ade349 which fixes:
- R128 field type changed from float64 to JSONR128 (accepts object or number)
- Artist.ID type changed from string to JSONID (accepts number or string)

Converted Artist.ID usages in storage.go to string() for assignments.
2026-10-18 11:14:39 +03:00

755 lines
21 KiB
Go

package storage
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/bogem/id3v2"
_ "modernc.org/sqlite"
"gitea.mrixs.me/Mrixs/yandex-music-api-go/yandexmusic"
)
// TrackMeta holds metadata for a downloaded track
type TrackMeta struct {
TrackID string `json:"track_id"`
AlbumID string `json:"album_id"`
ArtistID string `json:"artist_id"`
Title string `json:"title"`
ArtistName string `json:"artist_name"` // Primary artist (first)
ArtistNames []string `json:"artist_names"` // All artists
AlbumTitle string `json:"album_title"`
AlbumArtist string `json:"album_artist"` // Album artist (may differ from track artist)
AlbumArtistID string `json:"album_artist_id"` // Album artist ID
Year int `json:"year"` // Release year from album
TrackNumber int `json:"track_number"`
DiscNumber int `json:"disc_number"`
DurationMs int64 `json:"duration_ms"`
Bitrate int `json:"bitrate"`
Codec string `json:"codec"`
FilePath string `json:"file_path"`
FileSize int64 `json:"file_size"`
DownloadedAt string `json:"downloaded_at"`
// Extended metadata from Yandex Music API
Genre string `json:"genre"` // Track or album genre
Comment string `json:"comment"` // Album description / comments
Lyrics string `json:"lyrics"` // Track lyrics text
Publisher string `json:"publisher"` // Album label / publisher
EncodedBy string `json:"encoded_by"` // Encoder name
OriginalYear string `json:"original_year"` // Original release year
Composer string `json:"composer"` // Composer
ISRC string `json:"isrc"` // International Standard Recording Code
Copyright string `json:"copyright"` // Copyright information
}
// AlbumMeta holds metadata for a downloaded album
type AlbumMeta struct {
AlbumID string `json:"album_id"`
ArtistID string `json:"artist_id"`
Title string `json:"title"`
ArtistName string `json:"artist_name"`
Year int `json:"year"`
TrackCount int `json:"track_count"`
CoverPath string `json:"cover_path"`
Tracks []TrackMeta `json:"tracks"`
DownloadedAt string `json:"downloaded_at"`
}
// Storage manages local file storage for downloaded music
type Storage struct {
rootPath string
organize bool
mu sync.RWMutex
db *sql.DB
}
// NewStorage creates a new storage instance with SQLite backend
func NewStorage(rootPath string, organize bool) (*Storage, error) {
if err := os.MkdirAll(rootPath, 0755); err != nil {
return nil, fmt.Errorf("create root path: %w", err)
}
dbPath := filepath.Join(rootPath, "index.db")
db, err := sql.Open("sqlite", dbPath)
if err != nil {
return nil, fmt.Errorf("open sqlite db: %w", err)
}
// Configure connection pool for single-writer SQLite
db.SetMaxOpenConns(1)
if err := createSchema(db); err != nil {
db.Close()
return nil, fmt.Errorf("create schema: %w", err)
}
s := &Storage{
rootPath: rootPath,
organize: organize,
db: db,
}
// Migrate from old JSON index if it exists
if err := s.migrateFromJSON(); err != nil {
// Log but don't fail — migration is best-effort
fmt.Printf("Warning: JSON index migration: %v\n", err)
}
return s, nil
}
// createSchema creates the tracks table if it doesn't exist
func createSchema(db *sql.DB) error {
schema := `
CREATE TABLE IF NOT EXISTS tracks (
track_id TEXT PRIMARY KEY,
album_id TEXT,
artist_id TEXT,
title TEXT,
artist_name TEXT,
artist_names TEXT,
album_title TEXT,
album_artist TEXT,
album_artist_id TEXT,
year INTEGER,
track_number INTEGER,
disc_number INTEGER,
duration_ms INTEGER,
bitrate INTEGER,
codec TEXT,
file_path TEXT,
file_size INTEGER,
downloaded_at TEXT,
genre TEXT,
comment TEXT,
lyrics TEXT,
publisher TEXT,
encoded_by TEXT,
original_year TEXT,
composer TEXT,
isrc TEXT,
copyright TEXT
);
CREATE INDEX IF NOT EXISTS idx_tracks_album_id ON tracks(album_id);
`
_, err := db.Exec(schema)
return err
}
// migrateFromJSON imports the old .index.json into SQLite if it exists
func (s *Storage) migrateFromJSON() error {
jsonPath := filepath.Join(s.rootPath, ".index.json")
data, err := os.ReadFile(jsonPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil // No old index, nothing to migrate
}
return err
}
var oldIndex map[string]TrackMeta
if err := json.Unmarshal(data, &oldIndex); err != nil {
return fmt.Errorf("parse old index: %w", err)
}
if len(oldIndex) == 0 {
return nil
}
// Check if SQLite already has data (avoid double-migration)
var count int
err = s.db.QueryRow("SELECT COUNT(*) FROM tracks").Scan(&count)
if err != nil {
return err
}
if count > 0 {
return nil // Already migrated
}
tx, err := s.db.Begin()
if err != nil {
return err
}
stmt, err := tx.Prepare(insertTrackSQL())
if err != nil {
tx.Rollback()
return err
}
defer stmt.Close()
for _, meta := range oldIndex {
if _, err := stmt.Exec(
meta.TrackID, meta.AlbumID, meta.ArtistID, meta.Title, meta.ArtistName,
serializeStrings(meta.ArtistNames), meta.AlbumTitle, meta.AlbumArtist,
meta.AlbumArtistID, meta.Year, meta.TrackNumber, meta.DiscNumber,
meta.DurationMs, meta.Bitrate, meta.Codec, meta.FilePath, meta.FileSize,
meta.DownloadedAt, meta.Genre, meta.Comment, meta.Lyrics, meta.Publisher,
meta.EncodedBy, meta.OriginalYear, meta.Composer, meta.ISRC, meta.Copyright,
); err != nil {
tx.Rollback()
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
// Remove the old JSON file after successful migration
if err := os.Remove(jsonPath); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
// insertTrackSQL returns the parameterized INSERT OR REPLACE statement
func insertTrackSQL() string {
return `
INSERT OR REPLACE INTO tracks (
track_id, album_id, artist_id, title, artist_name, artist_names,
album_title, album_artist, album_artist_id, year, track_number,
disc_number, duration_ms, bitrate, codec, file_path, file_size,
downloaded_at, genre, comment, lyrics, publisher, encoded_by,
original_year, composer, isrc, copyright
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`
}
// scanTrackMeta reads a TrackMeta from a sql.Row
func scanTrackMeta(row *sql.Row) (TrackMeta, error) {
var m TrackMeta
var artistNamesStr string
var downloadedAt sql.NullString
var genre sql.NullString
var comment sql.NullString
var lyrics sql.NullString
var publisher sql.NullString
var encodedBy sql.NullString
var originalYear sql.NullString
var composer sql.NullString
var isrc sql.NullString
var copyright sql.NullString
err := row.Scan(
&m.TrackID, &m.AlbumID, &m.ArtistID, &m.Title, &m.ArtistName,
&artistNamesStr, &m.AlbumTitle, &m.AlbumArtist, &m.AlbumArtistID,
&m.Year, &m.TrackNumber, &m.DiscNumber, &m.DurationMs, &m.Bitrate,
&m.Codec, &m.FilePath, &m.FileSize, &downloadedAt, &genre, &comment,
&lyrics, &publisher, &encodedBy, &originalYear, &composer, &isrc,
&copyright,
)
if err != nil {
return m, err
}
if downloadedAt.Valid {
m.DownloadedAt = downloadedAt.String
}
m.ArtistNames = deserializeStrings(artistNamesStr)
if genre.Valid {
m.Genre = genre.String
}
if comment.Valid {
m.Comment = comment.String
}
if lyrics.Valid {
m.Lyrics = lyrics.String
}
if publisher.Valid {
m.Publisher = publisher.String
}
if encodedBy.Valid {
m.EncodedBy = encodedBy.String
}
if originalYear.Valid {
m.OriginalYear = originalYear.String
}
if composer.Valid {
m.Composer = composer.String
}
if isrc.Valid {
m.ISRC = isrc.String
}
if copyright.Valid {
m.Copyright = copyright.String
}
return m, nil
}
// serializeStrings converts a []string to a JSON string for storage
func serializeStrings(strs []string) string {
if strs == nil {
return "[]"
}
b, _ := json.Marshal(strs)
return string(b)
}
// deserializeStrings converts a JSON string back to []string
func deserializeStrings(s string) []string {
if s == "" || s == "[]" {
return nil
}
var strs []string
if err := json.Unmarshal([]byte(s), &strs); err != nil {
return nil
}
return strs
}
// IsTrackDownloaded checks if a track has already been downloaded
func (s *Storage) IsTrackDownloaded(trackID string) bool {
s.mu.RLock()
defer s.mu.RUnlock()
var count int
err := s.db.QueryRow("SELECT COUNT(*) FROM tracks WHERE track_id = ?", trackID).Scan(&count)
if err != nil {
return false
}
return count > 0
}
// GetTrackMeta returns metadata for a downloaded track
func (s *Storage) GetTrackMeta(trackID string) (TrackMeta, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
m, err := scanTrackMeta(s.db.QueryRow("SELECT * FROM tracks WHERE track_id = ?", trackID))
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return TrackMeta{}, false
}
return TrackMeta{}, false
}
return m, true
}
// SaveTrack saves a track file and its metadata
func (s *Storage) SaveTrack(trackMeta TrackMeta, data []byte, coverData []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
// Determine file path
var filePath string
if s.organize {
// Artist/Album/TrackNumber - Title.ext
artistDir := sanitizeFilename(trackMeta.ArtistName)
albumDir := sanitizeFilename(trackMeta.AlbumTitle)
trackFilename := fmt.Sprintf("%02d - %s.%s", trackMeta.TrackNumber, sanitizeFilename(trackMeta.Title), trackMeta.Codec)
filePath = filepath.Join(s.rootPath, artistDir, albumDir, trackFilename)
} else {
// Flat structure: Artist - Title.ext
trackFilename := fmt.Sprintf("%s - %s.%s", sanitizeFilename(trackMeta.ArtistName), sanitizeFilename(trackMeta.Title), trackMeta.Codec)
filePath = filepath.Join(s.rootPath, trackFilename)
}
// Create directory
dir := filepath.Dir(filePath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("create directory: %w", err)
}
// Write file
if err := os.WriteFile(filePath, data, 0644); err != nil {
return fmt.Errorf("write track file: %w", err)
}
// Write ID3 tags (only for MP3)
if strings.EqualFold(trackMeta.Codec, "mp3") {
if err := writeID3Tags(filePath, trackMeta, coverData); err != nil {
// Log error but don't fail the download
fmt.Printf("Warning: failed to write ID3 tags: %v\n", err)
}
}
// Update metadata
trackMeta.FilePath = filePath
trackMeta.FileSize = int64(len(data))
trackMeta.DownloadedAt = time.Now().Format(time.RFC3339)
_, err := s.db.Exec(insertTrackSQL(),
trackMeta.TrackID, trackMeta.AlbumID, trackMeta.ArtistID, trackMeta.Title,
trackMeta.ArtistName, serializeStrings(trackMeta.ArtistNames),
trackMeta.AlbumTitle, trackMeta.AlbumArtist, trackMeta.AlbumArtistID,
trackMeta.Year, trackMeta.TrackNumber, trackMeta.DiscNumber,
trackMeta.DurationMs, trackMeta.Bitrate, trackMeta.Codec,
trackMeta.FilePath, trackMeta.FileSize, trackMeta.DownloadedAt,
trackMeta.Genre, trackMeta.Comment, trackMeta.Lyrics, trackMeta.Publisher,
trackMeta.EncodedBy, trackMeta.OriginalYear, trackMeta.Composer,
trackMeta.ISRC, trackMeta.Copyright,
)
if err != nil {
return fmt.Errorf("save track meta: %w", err)
}
return nil
}
// SaveAlbum saves album cover and metadata
func (s *Storage) SaveAlbum(albumMeta AlbumMeta, coverData []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
var coverPath string
if s.organize && len(albumMeta.Tracks) > 0 {
artistDir := sanitizeFilename(albumMeta.ArtistName)
albumDir := sanitizeFilename(albumMeta.Title)
coverPath = filepath.Join(s.rootPath, artistDir, albumDir, "cover.jpg")
} else {
coverPath = filepath.Join(s.rootPath, fmt.Sprintf("%s - %s_cover.jpg", sanitizeFilename(albumMeta.ArtistName), sanitizeFilename(albumMeta.Title)))
}
dir := filepath.Dir(coverPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("create directory: %w", err)
}
if len(coverData) > 0 {
if err := os.WriteFile(coverPath, coverData, 0644); err != nil {
return fmt.Errorf("write cover file: %w", err)
}
}
albumMeta.CoverPath = coverPath
// Update track metas with album info
for i := range albumMeta.Tracks {
if _, err := s.db.Exec(insertTrackSQL(),
albumMeta.Tracks[i].TrackID, albumMeta.Tracks[i].AlbumID,
albumMeta.Tracks[i].ArtistID, albumMeta.Tracks[i].Title,
albumMeta.Tracks[i].ArtistName,
serializeStrings(albumMeta.Tracks[i].ArtistNames),
albumMeta.Tracks[i].AlbumTitle, albumMeta.Tracks[i].AlbumArtist,
albumMeta.Tracks[i].AlbumArtistID, albumMeta.Tracks[i].Year,
albumMeta.Tracks[i].TrackNumber, albumMeta.Tracks[i].DiscNumber,
albumMeta.Tracks[i].DurationMs, albumMeta.Tracks[i].Bitrate,
albumMeta.Tracks[i].Codec, albumMeta.Tracks[i].FilePath,
albumMeta.Tracks[i].FileSize, albumMeta.Tracks[i].DownloadedAt,
albumMeta.Tracks[i].Genre, albumMeta.Tracks[i].Comment,
albumMeta.Tracks[i].Lyrics, albumMeta.Tracks[i].Publisher,
albumMeta.Tracks[i].EncodedBy, albumMeta.Tracks[i].OriginalYear,
albumMeta.Tracks[i].Composer, albumMeta.Tracks[i].ISRC,
albumMeta.Tracks[i].Copyright,
); err != nil {
return fmt.Errorf("save album track meta: %w", err)
}
}
return nil
}
// GetDownloadedTracks returns all downloaded track IDs
func (s *Storage) GetDownloadedTracks() []string {
s.mu.RLock()
defer s.mu.RUnlock()
rows, err := s.db.Query("SELECT track_id FROM tracks")
if err != nil {
return nil
}
defer rows.Close()
var tracks []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
continue
}
tracks = append(tracks, id)
}
return tracks
}
// GetDownloadedAlbums returns all downloaded album IDs
func (s *Storage) GetDownloadedAlbums() []string {
s.mu.RLock()
defer s.mu.RUnlock()
rows, err := s.db.Query("SELECT DISTINCT album_id FROM tracks WHERE album_id != '' AND album_id IS NOT NULL")
if err != nil {
return nil
}
defer rows.Close()
var albums []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
continue
}
albums = append(albums, id)
}
return albums
}
// Close closes the database connection
func (s *Storage) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.db != nil {
return s.db.Close()
}
return nil
}
// sanitizeFilename removes invalid characters from filenames
func sanitizeFilename(name string) string {
replacer := strings.NewReplacer(
"/", "_",
"\\", "_",
":", "_",
"*", "_",
"?", "_",
"\"", "_",
"<", "_",
">", "_",
"|", "_",
"\n", "_",
"\r", "_",
)
return replacer.Replace(name)
}
// BuildTrackMetaFromYandex creates TrackMeta from Yandex API data
func BuildTrackMetaFromYandex(track *yandexmusic.Track, album *yandexmusic.Album, trackNum, discNum int, downloadInfo yandexmusic.DownloadInfo) TrackMeta {
artistNames := make([]string, 0, len(track.Artists))
for _, a := range track.Artists {
artistNames = append(artistNames, a.Name)
}
artistName := ""
if len(artistNames) > 0 {
artistName = artistNames[0]
}
albumTitle := ""
albumID := ""
year := 0
var albumGenre, albumLabel, albumDesc, albumReleaseDate, albumCopyright string
if album != nil {
albumTitle = album.Title
albumID = string(album.ID)
year = album.Year
albumGenre = album.Genre
albumLabel = album.Label
albumDesc = album.Description
albumReleaseDate = album.ReleaseDate
if len(album.CatalogNumber) > 0 {
albumCopyright = album.CatalogNumber
}
} else if len(track.Albums) > 0 {
albumTitle = track.Albums[0].Title
albumID = string(track.Albums[0].ID)
year = track.Albums[0].Year
}
artistID := ""
if len(track.Artists) > 0 {
artistID = string(track.Artists[0].ID)
}
albumArtist := ""
albumArtistID := ""
if album != nil && len(album.Artists) > 0 {
albumArtist = album.Artists[0].Name
albumArtistID = string(album.Artists[0].ID)
} else if len(track.Albums) > 0 && len(track.Albums[0].Artists) > 0 {
albumArtist = track.Albums[0].Artists[0].Name
albumArtistID = string(track.Albums[0].Artists[0].ID)
}
// Genre: prefer track genre, fallback to album genre
genre := track.Genre
if genre == "" {
genre = albumGenre
}
// Publisher: album label
publisher := albumLabel
// Original year from release date
var originalYear string
if albumReleaseDate != "" {
if len(albumReleaseDate) >= 4 {
originalYear = albumReleaseDate[:4]
}
}
// Lyrics
var lyrics string
if track.Lyrics != nil && track.Lyrics.HasText {
lyrics = track.Lyrics.Text
}
return TrackMeta{
TrackID: string(track.ID),
AlbumID: albumID,
ArtistID: artistID,
Title: track.Title,
ArtistName: artistName,
ArtistNames: artistNames,
AlbumTitle: albumTitle,
AlbumArtist: albumArtist,
AlbumArtistID: albumArtistID,
Year: year,
TrackNumber: trackNum,
DiscNumber: discNum,
DurationMs: int64(track.DurationMs),
Bitrate: downloadInfo.Bitrate,
Codec: downloadInfo.Codec,
DownloadedAt: "",
// Extended metadata
Genre: genre,
Comment: albumDesc,
Lyrics: lyrics,
Publisher: publisher,
EncodedBy: "ym-dl",
OriginalYear: originalYear,
Composer: "",
ISRC: "",
Copyright: albumCopyright,
}
}
// BuildAlbumMetaFromYandex creates AlbumMeta from Yandex API data
func BuildAlbumMetaFromYandex(album *yandexmusic.Album, tracks []TrackMeta) AlbumMeta {
artistName := ""
artistID := ""
if len(album.Artists) > 0 {
artistName = album.Artists[0].Name
artistID = string(album.Artists[0].ID)
}
return AlbumMeta{
AlbumID: string(album.ID),
ArtistID: artistID,
Title: album.Title,
ArtistName: artistName,
Year: album.Year,
TrackCount: len(tracks),
Tracks: tracks,
DownloadedAt: "",
}
}
// writeID3Tags writes ID3v2 tags to an MP3 file
func writeID3Tags(filePath string, meta TrackMeta, coverData []byte) error {
tag, err := id3v2.Open(filePath, id3v2.Options{Parse: true})
if err != nil {
// Create new tag if file doesn't have one
tag = id3v2.NewEmptyTag()
}
defer tag.Close()
// Join all artists with "; " for the artist tag
artistString := strings.Join(meta.ArtistNames, "; ")
// Set basic tags
tag.SetTitle(meta.Title)
tag.SetArtist(artistString)
tag.SetAlbum(meta.AlbumTitle)
// Album artist (TPE2 frame) - use album artist if different from track artist
if meta.AlbumArtist != "" && meta.AlbumArtist != meta.ArtistName {
tag.AddTextFrame("TPE2", id3v2.EncodingUTF8, meta.AlbumArtist)
}
// Use album year from metadata, fallback to current year
year := meta.Year
if year == 0 {
year = time.Now().Year()
}
tag.SetYear(fmt.Sprintf("%d", year))
if meta.TrackNumber > 0 {
// Track number via TRCK frame
tag.AddTextFrame("TRCK", id3v2.EncodingUTF8, fmt.Sprintf("%d", meta.TrackNumber))
}
if meta.DiscNumber > 0 {
// Disc number via TPOS frame
tag.AddTextFrame("TPOS", id3v2.EncodingUTF8, fmt.Sprintf("%d", meta.DiscNumber))
}
// Genre (TCON)
if meta.Genre != "" {
tag.SetGenre(meta.Genre)
}
// Composer (TCOM)
if meta.Composer != "" {
tag.AddTextFrame("TCOM", id3v2.EncodingUTF8, meta.Composer)
}
// Publisher (TPUB)
if meta.Publisher != "" {
tag.AddTextFrame("TPUB", id3v2.EncodingUTF8, meta.Publisher)
}
// Encoded by (TENC)
if meta.EncodedBy != "" {
tag.AddTextFrame("TENC", id3v2.EncodingUTF8, meta.EncodedBy)
}
// Copyright (TCOP)
if meta.Copyright != "" {
tag.AddTextFrame("TCOP", id3v2.EncodingUTF8, meta.Copyright)
}
// Original year (TORY)
if meta.OriginalYear != "" {
tag.AddTextFrame("TORY", id3v2.EncodingUTF8, meta.OriginalYear)
}
// ISRC (TSRC)
if meta.ISRC != "" {
tag.AddTextFrame("TSRC", id3v2.EncodingUTF8, meta.ISRC)
}
// Comment (COMM)
if meta.Comment != "" {
tag.AddCommentFrame(id3v2.CommentFrame{
Encoding: id3v2.EncodingUTF8,
Language: "eng",
Description: "Comment",
Text: meta.Comment,
})
}
// Lyrics (USLT)
if meta.Lyrics != "" {
tag.AddUnsynchronisedLyricsFrame(id3v2.UnsynchronisedLyricsFrame{
Encoding: id3v2.EncodingUTF8,
Language: "eng",
ContentDescriptor: "Lyrics",
Lyrics: meta.Lyrics,
})
}
// Add cover art (APIC frame)
if len(coverData) > 0 {
tag.AddAttachedPicture(id3v2.PictureFrame{
Encoding: id3v2.EncodingUTF8,
MimeType: "image/jpeg",
PictureType: id3v2.PTFrontCover,
Description: "Cover",
Picture: coverData,
})
}
// Add custom TXXX frames for Yandex IDs
tag.AddTextFrame("TXXX", id3v2.EncodingUTF8, "YANDEX_TRACK_ID: "+meta.TrackID)
tag.AddTextFrame("TXXX", id3v2.EncodingUTF8, "YANDEX_ALBUM_ID: "+meta.AlbumID)
tag.AddTextFrame("TXXX", id3v2.EncodingUTF8, "YANDEX_ARTIST_ID: "+meta.ArtistID)
// Save tag
return tag.Save()
}