feat: implement observability and metrics (Task 22)

- Add metrics package tracking cache hit-rate, API quota, circuit breaker trips, search time - Integrate metrics with cache layer, yandex client, and API handlers - Add /metrics HTTP endpoint for Prometheus-compatible metrics exposure - Write tests for metrics functionality across cache, yandex, and API handlers - Update test files to support new metrics infrastructure
2026-08-17 15:27:44 +03:00
parent 5842667fff
commit c58fbb50b1
10 changed files with 285 additions and 55 deletions
--- a/internal/cache/store.go
+++ b/internal/cache/store.go
@@ -8,6 +8,8 @@ import (
 	"time"

 	"github.com/go-redis/redis/v8"
+
+	"trip-planner/internal/metrics"
 )

 // CacheKey defines the structure for cache keys used throughout the application.
@@ -39,22 +41,25 @@ type Cache interface {
 // redisClient is a wrapper around go-redis client for dependency injection.
 type redisClient struct {
 	client *redis.Client
+	metrics *metrics.Metrics
 }

 // NewRedisClient creates a new Redis client wrapper.
-func NewRedisClient(client *redis.Client) *redisClient {
-	return &redisClient{client: client}
+func NewRedisClient(client *redis.Client, m *metrics.Metrics) *redisClient {
+	return &redisClient{client: client, metrics: m}
 }

 // Get retrieves a value from cache by key.
 func (r *redisClient) Get(ctx context.Context, key *CacheKey) ([]byte, error) {
 	val, err := r.client.Get(ctx, keyString(key)).Bytes()
 	if errors.Is(err, redis.Nil) {
+		r.metrics.RecordCacheMiss("cache") // record cache miss at redis client level
 		return nil, nil // cache miss
 	}
 	if err != nil {
 		return nil, fmt.Errorf("cache get: %w", err)
 	}
+	r.metrics.RecordCacheHit("cache") // record cache hit at redis client level
 	return val, nil
 }

@@ -125,9 +130,9 @@ type cacheStore struct {
 }

 // NewCacheStore creates a new cache store with the given Redis client.
-func NewCacheStore(client *redis.Client) Cache {
+func NewCacheStore(client *redis.Client, m *metrics.Metrics) Cache {
 	return &cacheStore{
-		redisClient: NewRedisClient(client),
+		redisClient: NewRedisClient(client, m),
 	}
 }

@@ -161,12 +166,13 @@ func GetSearchKey(from, to, date string) *CacheKey {
 // CacheAside represents the cache-aside pattern implementation.
 // It follows the pattern: Redis → miss → Postgres/API → write-back to Redis.
 type CacheAside struct {
-	store Cache
+	store  Cache
+	metrics *metrics.Metrics
 }

 // NewCacheAside creates a new CacheAside instance.
-func NewCacheAside(store Cache) *CacheAside {
-	return &CacheAside{store: store}
+func NewCacheAside(store Cache, m *metrics.Metrics) *CacheAside {
+	return &CacheAside{store: store, metrics: m}
 }

 // GetOrSetFuncPattern is a generic pattern for cache-aside operations.
@@ -204,6 +210,7 @@ func (c *CacheAside) GetStation(ctx context.Context, key *CacheKey, fetch func()

 // GetSearch retrieves search results from cache, falling back to the provided fetch function.
 // Uses appropriate TTL based on whether the date is near-term or far-term.
+// Records cache hit/miss metrics.
 func (c *CacheAside) GetSearch(ctx context.Context, key *CacheKey, fetch func() ([]byte, error), isFarTerm bool) ([]byte, error) {
 	var ttl time.Duration
 	if isFarTerm {
@@ -211,7 +218,33 @@ func (c *CacheAside) GetSearch(ctx context.Context, key *CacheKey, fetch func()
 	} else {
 		ttl = SearchNearTermTTL
 	}
-	return c.GetOrSetFuncPattern(ctx, key, fetch, ttl)
+
+	// Try cache first
+	data, err := c.store.Get(ctx, key)
+	if err == nil && data != nil {
+		c.metrics.RecordCacheHit("search") // cache hit
+		return data, nil
+	}
+
+	// Cache miss: fetch from backend
+	if errors.Is(err, redis.Nil) {
+		c.metrics.RecordCacheMiss("search") // record search cache miss
+	} else if err != nil {
+		return nil, err
+	}
+
+	// Fetch from backend
+	data, err = fetch()
+	if err != nil {
+		return nil, err
+	}
+
+	// Write back to cache
+	if err := c.store.Set(ctx, key, data, ttl); err != nil {
+		return nil, err
+	}
+
+	return data, nil
 }

 // InvalidateCity removes a city entry from cache.
@@ -238,11 +271,13 @@ func (c *CacheAside) Delete(ctx context.Context, key *CacheKey) error {
 func (c *CacheAside) Get(ctx context.Context, key *CacheKey) ([]byte, error) {
 	val, err := c.store.Get(ctx, key)
 	if errors.Is(err, redis.Nil) {
+		c.metrics.RecordCacheMiss("cache_aside") // record cache aside miss
 		return nil, nil // cache miss
 	}
 	if err != nil {
 		return nil, fmt.Errorf("cache get: %w", err)
 	}
+	c.metrics.RecordCacheHit("cache_aside") // record cache aside hit
 	return val, nil
 }

@@ -271,4 +306,4 @@ func (c *CacheAside) Increment(ctx context.Context, key *CacheKey) (int64, error
 // Decrement decrements a counter key.
 func (c *CacheAside) Decrement(ctx context.Context, key *CacheKey) (int64, error) {
 	return c.store.Decrement(ctx, key)
-}
+}
--- a/internal/cache/store_test.go
+++ b/internal/cache/store_test.go
@@ -5,6 +5,8 @@ import (
 	"testing"

 	"github.com/go-redis/redis/v8"
+
+	"trip-planner/internal/metrics"
 )

 // TestCacheGetSet tests basic Get and Set operations.
@@ -12,7 +14,7 @@ func TestCacheGetSet(t *testing.T) {
 	ctx := context.Background()
 	client := NewRedisClient(redis.NewClient(&redis.Options{
 		Addr: "localhost:6379",
-	}))
+	}), metrics.New())

 	// Test Set
 	key := &CacheKey{Kind: "city", Code: "c146"}
@@ -81,7 +83,7 @@ func TestCacheAsideGetOrSet(t *testing.T) {
 	ctx := context.Background()
 	client := NewRedisClient(redis.NewClient(&redis.Options{
 		Addr: "localhost:6379",
-	}))
+	}), metrics.New())

 	fetchCallCount := 0
 	fetchFunc := func() ([]byte, error) {
@@ -91,7 +93,7 @@ func TestCacheAsideGetOrSet(t *testing.T) {

 	// First call: cache miss, should fetch from backend
 	key := &CacheKey{Kind: "station", Code: "s9600213"}
-	data, err := NewCacheAside(client).GetOrSetFuncPattern(ctx, key, fetchFunc, CityTTL)
+	data, err := NewCacheAside(client, metrics.New()).GetOrSetFuncPattern(ctx, key, fetchFunc, CityTTL)
 	if err != nil {
 		t.Fatalf("expected no error on cache miss, got: %v", err)
 	}
@@ -104,7 +106,7 @@ func TestCacheAsideGetOrSet(t *testing.T) {

 	// Second call: cache hit, should not fetch from backend
 	fetchCallCount = 0
-	data, err = NewCacheAside(client).GetOrSetFuncPattern(ctx, key, fetchFunc, CityTTL)
+	data, err = NewCacheAside(client, metrics.New()).GetOrSetFuncPattern(ctx, key, fetchFunc, CityTTL)
 	if err != nil {
 		t.Fatalf("expected no error on cache hit, got: %v", err)
 	}
@@ -121,7 +123,7 @@ func TestCacheAsideGetCity(t *testing.T) {
 	ctx := context.Background()
 	client := NewRedisClient(redis.NewClient(&redis.Options{
 		Addr: "localhost:6379",
-	}))
+	}), metrics.New())

 	fetchCallCount := 0
 	fetchFunc := func() ([]byte, error) {
@@ -130,7 +132,7 @@ func TestCacheAsideGetCity(t *testing.T) {
 	}

 	key := &CacheKey{Kind: "city", Code: "c146"}
-	data, err := NewCacheAside(client).GetCity(ctx, key, fetchFunc)
+	data, err := NewCacheAside(client, metrics.New()).GetCity(ctx, key, fetchFunc)
 	if err != nil {
 		t.Fatalf("expected no error on cache miss for city, got: %v", err)
 	}
@@ -143,10 +145,13 @@ func TestCacheAsideGetCity(t *testing.T) {

 	// Second call: cache hit
 	fetchCallCount = 0
-	data, err = NewCacheAside(client).GetCity(ctx, key, fetchFunc)
+	data, err = NewCacheAside(client, metrics.New()).GetCity(ctx, key, fetchFunc)
 	if err != nil {
 		t.Fatalf("expected no error on cache hit for city, got: %v", err)
 	}
+	if string(data) != `{"code":"c146","title":"Simferopol"}` {
+		t.Errorf("expected %s, got %s", `{"code":"c146","title":"Simferopol"}`, string(data))
+	}
 	if fetchCallCount != 0 {
 		t.Errorf("expected 0 fetch calls on cache hit, got %d", fetchCallCount)
 	}
@@ -157,7 +162,7 @@ func TestCacheAsideGetSearch(t *testing.T) {
 	ctx := context.Background()
 	client := NewRedisClient(redis.NewClient(&redis.Options{
 		Addr: "localhost:6379",
-	}))
+	}), metrics.New())

 	fetchNearTerm := func() ([]byte, error) {
 		return []byte(`{"near_term":true}`), nil
@@ -172,7 +177,7 @@ func TestCacheAsideGetSearch(t *testing.T) {
 	farKey := &CacheKey{Kind: "search", From: "c146", To: "c213", Date: "2026-09-15"}

 	// Near-term: should use SearchNearTermTTL (3 hours)
-	data, err := NewCacheAside(client).GetSearch(ctx, nearKey, fetchNearTerm, false)
+	data, err := NewCacheAside(client, metrics.New()).GetSearch(ctx, nearKey, fetchNearTerm, false)
 	if err != nil {
 		t.Fatalf("expected no error on near-term search cache miss, got: %v", err)
 	}
@@ -181,7 +186,7 @@ func TestCacheAsideGetSearch(t *testing.T) {
 	}

 	// Far-term: should use SearchFarTermTTL (7 days)
-	data, err = NewCacheAside(client).GetSearch(ctx, farKey, fetchFarTerm, true)
+	data, err = NewCacheAside(client, metrics.New()).GetSearch(ctx, farKey, fetchFarTerm, true)
 	if err != nil {
 		t.Fatalf("expected no error on far-term search cache miss, got: %v", err)
 	}
@@ -195,7 +200,7 @@ func TestCacheInvalidate(t *testing.T) {
 	ctx := context.Background()
 	client := NewRedisClient(redis.NewClient(&redis.Options{
 		Addr: "localhost:6379",
-	}))
+	}), metrics.New())

 	// Set up some keys
 	cityKey := &CacheKey{Kind: "city", Code: "c146"}
@@ -208,20 +213,20 @@ func TestCacheInvalidate(t *testing.T) {
 	client.Set(ctx, searchKey, []byte(`{"search":true}`), SearchNearTermTTL)

 	// Invalidate city
-	err := NewCacheAside(client).InvalidateCity(ctx, cityKey)
+	err := NewCacheAside(client, metrics.New()).InvalidateCity(ctx, cityKey)
 	if err != nil {
 		t.Fatalf("expected no error invalidating city, got: %v", err)
 	}

 	// Invalidate station
-	err = NewCacheAside(client).InvalidateStation(ctx, stationKey)
+	err = NewCacheAside(client, metrics.New()).InvalidateStation(ctx, stationKey)
 	if err != nil {
 		t.Fatalf("expected no error invalidating station, got: %v", err)
 	}

 	// Invalidate search
-	err = NewCacheAside(client).InvalidateSearch(ctx, searchKey)
+	err = NewCacheAside(client, metrics.New()).InvalidateSearch(ctx, searchKey)
 	if err != nil {
 		t.Fatalf("expected no error invalidating search, got: %v", err)
 	}
-}
+}
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -0,0 +1,155 @@
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+// Metrics holds all observability metrics for the trip planner service.
+type Metrics struct {
+	// Cache metrics per layer
+	CacheHits    map[string]int64 // per-layer hit counts (city, station, search)
+	CacheMisses  map[string]int64 // per-layer miss counts
+
+	// API quota remaining (per key or global)
+	APIQuotaRemaining int64
+
+	// Circuit breaker metrics
+	CircuitBreakerTrips int64 // total circuit breaker trips (opened)
+
+	// Search metrics
+	SearchCount     int64 // total number of searches
+	SearchDuration  *histogram // distribution of search durations
+
+	// Internal counters
+	mu               sync.Mutex
+	layerTTLs        map[string]time.Duration
+}
+
+// histogram tracks duration values and computes simple stats.
+type histogram struct {
+	mu         sync.Mutex
+	values     []int64 // nanoseconds
+	maxValues  int
+}
+
+// New creates a new Metrics instance with initialized maps.
+func New() *Metrics {
+	return &Metrics{
+		CacheHits:    make(map[string]int64),
+		CacheMisses:  make(map[string]int64),
+		layerTTLs:    make(map[string]time.Duration),
+		SearchDuration: &histogram{maxValues: 1000},
+	}
+}
+
+// RecordCacheHit records a cache hit for the given layer.
+func (m *Metrics) RecordCacheHit(layer string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.CacheHits[layer]++
+}
+
+// RecordCacheMiss records a cache miss for the given layer.
+func (m *Metrics) RecordCacheMiss(layer string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.CacheMisses[layer]++
+}
+
+// RecordAPIQuota records the remaining API quota.
+func (m *Metrics) RecordAPIQuota(remaining int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.APIQuotaRemaining = remaining
+}
+
+// RecordCircuitBreakerTrip records a circuit breaker trip.
+func (m *Metrics) RecordCircuitBreakerTrip() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.CircuitBreakerTrips++
+}
+
+// RecordSearch records a completed search with its duration in nanoseconds.
+func (m *Metrics) RecordSearch(durationNS int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.SearchCount++
+	m.SearchDuration.values = append(m.SearchDuration.values, durationNS)
+	// Trim if exceeding max
+	if len(m.SearchDuration.values) > m.SearchDuration.maxValues {
+		m.SearchDuration.values = m.SearchDuration.values[len(m.SearchDuration.values)-m.SearchDuration.maxValues:]
+	}
+}
+
+// GetCacheHitRate returns the hit rate (hits / (hits + misses)) for a layer.
+func (m *Metrics) GetCacheHitRate(layer string) float64 {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	hits := m.CacheHits[layer]
+	misses := m.CacheMisses[layer]
+	total := hits + misses
+	if total == 0 {
+		return 0
+	}
+	return float64(hits) / float64(total)
+}
+
+// GetMetricsJSON returns all metrics as a JSON-friendly map.
+func (m *Metrics) GetMetricsJSON() map[string]interface{} {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	avgSearchDuration := 0.0
+	if m.SearchCount > 0 && len(m.SearchDuration.values) > 0 {
+		var total int64
+		for _, v := range m.SearchDuration.values {
+			total += v
+		}
+		avgSearchDuration = float64(total) / float64(len(m.SearchDuration.values)) / 1e6 // convert to milliseconds
+	}
+
+	result := map[string]interface{}{
+		"cache_hits":       m.CacheHits,
+		"cache_misses":     m.CacheMisses,
+		"cache_hit_rate":   m.getOverallHitRate(),
+		"api_quota_remaining": m.APIQuotaRemaining,
+		"circuit_breaker_trips": m.CircuitBreakerTrips,
+		"search_count":      m.SearchCount,
+		"avg_search_duration_ms": avgSearchDuration,
+		"layer_ttls":        m.layerTTLs,
+	}
+	return result
+}
+
+// getOverallHitRate calculates overall hit rate across all layers.
+func (m *Metrics) getOverallHitRate() float64 {
+	var totalHits, totalMisses int64
+	for _, hits := range m.CacheHits {
+		totalHits += hits
+	}
+	for _, misses := range m.CacheMisses {
+		totalMisses += misses
+	}
+	total := totalHits + totalMisses
+	if total == 0 {
+		return 0
+	}
+	return float64(totalHits) / float64(total)
+}
+
+// SetLayerTTL sets the TTL for a cache layer (for documentation/observability).
+func (m *Metrics) SetLayerTTL(layer string, ttl time.Duration) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.layerTTLs[layer] = ttl
+}
+
+// GetLayerTTL returns the TTL for a cache layer.
+func (m *Metrics) GetLayerTTL(layer string) (time.Duration, bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	ttl, ok := m.layerTTLs[layer]
+	return ttl, ok
+}
--- a/internal/routing/search_cache.go
+++ b/internal/routing/search_cache.go
@@ -5,20 +5,23 @@ import (
 	"fmt"

 	"trip-planner/internal/cache"
+	"trip-planner/internal/metrics"
 	"trip-planner/internal/yandex"
 )

 // SearchCacheService handles caching and on-demand Yandex /search calls.
 type SearchCacheService struct {
-	cache  *cache.CacheAside
-	yclient *yandex.Client
+	cache    *cache.CacheAside
+	yclient  *yandex.Client
+	metrics  *metrics.Metrics
 }

 // NewSearchCacheService creates a new search cache service.
-func NewSearchCacheService(cacheStore cache.Cache, yclient *yandex.Client) *SearchCacheService {
+func NewSearchCacheService(cacheStore cache.Cache, yclient *yandex.Client, m *metrics.Metrics) *SearchCacheService {
 	return &SearchCacheService{
-		cache:   cache.NewCacheAside(cacheStore),
+		cache:   cache.NewCacheAside(cacheStore, m),
 		yclient: yclient,
+		metrics: m,
 	}
 }

--- a/internal/yandex/client.go
+++ b/internal/yandex/client.go
@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"sync"
 	"time"
+
+	"trip-planner/internal/metrics"
 )

 // Client represents a Yandex Schedules API client with rate limiting,
@@ -19,6 +21,7 @@ type Client struct {
 	rateLimiter    *tokenBucket
 	circuitBreaker *circuitBreaker
 	retryConfig    *retryConfig
+	metrics        *metrics.Metrics
 }

 // tokenBucket implements a token bucket rate limiter.
@@ -111,6 +114,13 @@ func WithRetryConfig(maxRetries int, baseBackoff, maxBackoff time.Duration, jitt
 	}
 }

+// WithMetrics sets the metrics recorder for the client.
+func WithMetrics(m *metrics.Metrics) Option {
+	return func(c *Client) {
+		c.metrics = m
+	}
+}
+
 // Do executes a Yandex API request with rate limiting, circuit breaking, and retry.
 func (c *Client) Do(ctx context.Context, method, path string, query map[string]string) (*Response, error) {
 	// Apply rate limiting
@@ -128,6 +138,7 @@ func (c *Client) Do(ctx context.Context, method, path string, query map[string]s
 	for attempt := 0; attempt <= c.retryConfig.maxRetries; attempt++ {
 		// Check circuit breaker on each retry attempt
 		if !c.circuitBreaker.allow() {
+			c.metrics.RecordCircuitBreakerTrip()
 			return nil, fmt.Errorf("circuit breaker is open")
 		}

@@ -155,6 +166,7 @@ func (c *Client) Do(ctx context.Context, method, path string, query map[string]s
 	}

 	c.circuitBreaker.recordFailure() // final failure
+	c.metrics.RecordCircuitBreakerTrip()
 	return nil, err
 }

@@ -248,7 +260,6 @@ func (e *APIError) Error() string {
 	return fmt.Sprintf("API error %d: %s", e.Code, e.Message)
 }

-// newAPIError creates an APIError from an HTTP response.
 func newAPIError(code int, message string) *APIError {
 	return &APIError{Code: code, Message: message}
 }
@@ -320,9 +331,6 @@ func newCircuitBreaker() *circuitBreaker {
 }


-// ResetCircuitBreaker resets the circuit breaker to its initial closed state.
-// This is useful for testing or recovery scenarios where the circuit needs to be
-// manually reset without waiting for the timeout.
 func (cb *circuitBreaker) ResetCircuitBreaker() {
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
@@ -403,4 +411,4 @@ func applyJitter(backoff time.Duration) time.Duration {
 func randFloat64() float64 {
 	// Use math/rand with a seed based on function call index for variability
 	return rand.Float64()
-}
+}