Add cache for debug files

Author: marcsanmi

cmd/symbolization/main.go

@@ -21,7 +21,7 @@ func main() {
 	// Alternatively, use a local debug info file:
 	//client := &localDebuginfodClient{debugFilePath: "/path/to/your/debug/file"}
 
-	s := symbolizer.NewSymbolizer(client)
+	s := symbolizer.NewSymbolizer(client, nil)
 	ctx := context.Background()
 
 	_, err := client.FetchDebuginfo(buildID)

pkg/experiment/query_backend/backend.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
-
 	"github.com/go-kit/log"
 	"github.com/grafana/dskit/grpcclient"
 	"github.com/grafana/dskit/services"
+	"github.com/grafana/pyroscope/pkg/objstore/client"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
@@ -21,13 +21,14 @@ import (
 type Config struct {
 	Address          string            `yaml:"address"`
 	GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate between the query-frontends and the query-schedulers."`
-	DebuginfodURL    string            `yaml:"debuginfod_url"`
+	Symbolizer       symbolizer.Config `yaml:"symbolizer"`
+	DebugStorage     client.Config     `yaml:"debug_storage"`
 }
 
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.StringVar(&cfg.Address, "query-backend.address", "localhost:9095", "")
-	f.StringVar(&cfg.DebuginfodURL, "query-backend.debuginfod-url", "https://debuginfod.elfutils.org", "URL of the debuginfod server")
 	cfg.GRPCClientConfig.RegisterFlagsWithPrefix("query-backend.grpc-client-config", f)
+	cfg.Symbolizer.RegisterFlagsWithPrefix("query-backend.symbolizer", f)
 }
 
 func (cfg *Config) Validate() error {
@@ -63,10 +64,12 @@ func New(
 	blockReader QueryHandler,
 ) (*QueryBackend, error) {
 	var sym *symbolizer.Symbolizer
-	if config.DebuginfodURL != "" {
-		sym = symbolizer.NewSymbolizer(
-			symbolizer.NewDebuginfodClient(config.DebuginfodURL),
-		)
+	if config.Symbolizer.DebuginfodURL != "" {
+		var err error
+		sym, err = symbolizer.NewFromConfig(context.Background(), config.Symbolizer)
+		if err != nil {
+			return nil, fmt.Errorf("create symbolizer: %w", err)
+		}
 	}
 
 	q := QueryBackend{

pkg/experiment/symbolizer/cache.go

@@ -0,0 +1,91 @@
+package symbolizer
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/grafana/pyroscope/pkg/objstore"
+)
+
+// CacheConfig holds configuration for the debug info cache
+type CacheConfig struct {
+	Enabled bool          `yaml:"enabled"`
+	MaxAge  time.Duration `yaml:"max_age"`
+}
+
+func NewObjstoreCache(bucket objstore.Bucket, maxAge time.Duration) *ObjstoreCache {
+	return &ObjstoreCache{
+		bucket: bucket,
+		maxAge: maxAge,
+	}
+}
+
+// DebugInfoCache handles caching of debug info files
+type DebugInfoCache interface {
+	Get(ctx context.Context, buildID string) (io.ReadCloser, error)
+	Put(ctx context.Context, buildID string, reader io.Reader) error
+}
+
+// ObjstoreCache implements DebugInfoCache using S3 storage
+type ObjstoreCache struct {
+	bucket objstore.Bucket
+	maxAge time.Duration
+}
+
+func (c *ObjstoreCache) Get(ctx context.Context, buildID string) (io.ReadCloser, error) {
+	// First check if object exists to avoid unnecessary operations
+	reader, err := c.bucket.Get(ctx, buildID)
+	if err != nil {
+		if c.bucket.IsObjNotFoundErr(err) {
+			return nil, err
+		}
+		return nil, fmt.Errorf("get from cache: %w", err)
+	}
+
+	// Get attributes - this should use the same HEAD request that Get used
+	attrs, err := c.bucket.Attributes(ctx, buildID)
+	if err != nil {
+		reader.Close()
+		return nil, fmt.Errorf("get cache attributes: %w", err)
+	}
+
+	// Check if expired
+	if time.Since(attrs.LastModified) > c.maxAge {
+		reader.Close()
+		// Async deletion to not block the request
+		go func() {
+			delCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			_ = c.bucket.Delete(delCtx, buildID)
+		}()
+		return nil, fmt.Errorf("cached object expired")
+	}
+
+	return reader, nil
+}
+
+func (c *ObjstoreCache) Put(ctx context.Context, buildID string, reader io.Reader) error {
+	if err := c.bucket.Upload(ctx, buildID, reader); err != nil {
+		return fmt.Errorf("upload to cache: %w", err)
+	}
+	return nil
+}
+
+// NullCache implements DebugInfoCache but performs no caching
+type NullCache struct{}
+
+func NewNullCache() DebugInfoCache {
+	return &NullCache{}
+}
+
+func (n *NullCache) Get(ctx context.Context, buildID string) (io.ReadCloser, error) {
+	// Always return cache miss
+	return nil, fmt.Errorf("cache miss")
+}
+
+func (n *NullCache) Put(ctx context.Context, buildID string, reader io.Reader) error {
+	// Do nothing
+	return nil
+}

pkg/experiment/symbolizer/symbolizer.go

@@ -4,7 +4,13 @@ import (
 	"context"
 	"debug/dwarf"
 	"debug/elf"
+	"flag"
 	"fmt"
+	"io"
+	"os"
+	"time"
+
+	objstoreclient "github.com/grafana/pyroscope/pkg/objstore/client"
 )
 
 // DwarfResolver implements the liner interface
@@ -37,44 +43,106 @@ func (d *DwarfResolver) Close() error {
 	return d.file.Close()
 }
 
+type Config struct {
+	DebuginfodURL string                `yaml:"debuginfod_url"`
+	Cache         CacheConfig           `yaml:"cache"`
+	Storage       objstoreclient.Config `yaml:"storage"`
+}
+
 type Symbolizer struct {
 	client DebuginfodClient
+	cache  DebugInfoCache
 }
 
-func NewSymbolizer(client DebuginfodClient) *Symbolizer {
+func NewSymbolizer(client DebuginfodClient, cache DebugInfoCache) *Symbolizer {
+	if cache == nil {
+		cache = NewNullCache()
+	}
 	return &Symbolizer{
 		client: client,
+		cache:  cache,
+	}
+}
+
+func NewFromConfig(ctx context.Context, cfg Config) (*Symbolizer, error) {
+	client := NewDebuginfodClient(cfg.DebuginfodURL)
+
+	// Default to no caching
+	var cache = NewNullCache()
+
+	if cfg.Cache.Enabled {
+		if cfg.Storage.Backend == "" {
+			return nil, fmt.Errorf("storage configuration required when cache is enabled")
+		}
+		bucket, err := objstoreclient.NewBucket(ctx, cfg.Storage, "debuginfo")
+		if err != nil {
+			return nil, fmt.Errorf("create debug info storage: %w", err)
+		}
+		cache = NewObjstoreCache(bucket, cfg.Cache.MaxAge)
 	}
+
+	return &Symbolizer{
+		client: client,
+		cache:  cache,
+	}, nil
 }
 
 func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error {
-	// Fetch debug info file
-	debugFilePath, err := s.client.FetchDebuginfo(req.BuildID)
+	debugReader, err := s.cache.Get(ctx, req.BuildID)
+	if err == nil {
+		defer debugReader.Close()
+		return s.symbolizeFromReader(ctx, debugReader, req)
+	}
+
+	// Cache miss - fetch from debuginfod
+	filepath, err := s.client.FetchDebuginfo(req.BuildID)
 	if err != nil {
 		return fmt.Errorf("fetch debuginfo: %w", err)
 	}
 
-	// Open ELF file
-	f, err := elf.Open(debugFilePath)
+	// Open for symbolization
+	f, err := os.Open(filepath)
 	if err != nil {
-		return fmt.Errorf("open ELF file: %w", err)
+		return fmt.Errorf("open debug file: %w", err)
 	}
 	defer f.Close()
 
+	// Cache it for future use
+	if _, err := f.Seek(0, 0); err != nil {
+		return fmt.Errorf("seek file: %w", err)
+	}
+	if err := s.cache.Put(ctx, req.BuildID, f); err != nil {
+		// TODO: Log it but don't fail?
+	}
+
+	// Seek back to start for symbolization
+	if _, err := f.Seek(0, 0); err != nil {
+		return fmt.Errorf("seek file: %w", err)
+	}
+
+	return s.symbolizeFromReader(ctx, f, req)
+}
+
+func (s *Symbolizer) symbolizeFromReader(ctx context.Context, r io.ReadCloser, req Request) error {
+	elfFile, err := elf.NewFile(io.NewSectionReader(r.(io.ReaderAt), 0, 1<<63-1))
+	if err != nil {
+		return fmt.Errorf("create ELF file from reader: %w", err)
+	}
+	defer elfFile.Close()
+
 	// Get executable info for address normalization
-	ei, err := ExecutableInfoFromELF(f)
+	ei, err := ExecutableInfoFromELF(elfFile)
 	if err != nil {
 		return fmt.Errorf("executable info from ELF: %w", err)
 	}
 
 	// Create liner
-	liner, err := NewDwarfResolver(f)
+	liner, err := NewDwarfResolver(elfFile)
 	if err != nil {
 		return fmt.Errorf("create liner: %w", err)
 	}
 	//defer liner.Close()
 
-	// Process each mapping's locations
 	for _, mapping := range req.Mappings {
 		for _, loc := range mapping.Locations {
 			addr, err := MapRuntimeAddress(loc.Address, ei, Mapping{
@@ -92,48 +160,17 @@ func (s *Symbolizer) Symbolize(ctx context.Context, req Request) error {
 				continue // Skip errors for individual addresses
 			}
 
-			// Update the location directly
 			loc.Lines = lines
 		}
 	}
 
 	return nil
 }
 
-func (s *Symbolizer) SymbolizeAll(ctx context.Context, buildID string) error {
-	// Reuse the existing debuginfo file
-	debugFilePath, err := s.client.FetchDebuginfo(buildID)
-	if err != nil {
-		return fmt.Errorf("fetch debuginfo: %w", err)
-	}
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.DebuginfodURL, prefix+".debuginfod-url", "https://debuginfod.elfutils.org", "URL of the debuginfod server")
 
-	f, err := elf.Open(debugFilePath)
-	if err != nil {
-		return fmt.Errorf("open ELF file: %w", err)
-	}
-	defer f.Close()
-
-	debugData, err := f.DWARF()
-	if err != nil {
-		return fmt.Errorf("get DWARF data: %w", err)
-	}
-
-	debugInfo := NewDWARFInfo(debugData)
-	allSymbols := debugInfo.SymbolizeAllAddresses()
-
-	fmt.Println("\nSymbolizing all addresses in DWARF file:")
-	fmt.Println("----------------------------------------")
-
-	for addr, lines := range allSymbols {
-		fmt.Printf("\nAddress: 0x%x\n", addr)
-		for _, line := range lines {
-			fmt.Printf("  Function: %s\n", line.Function.Name)
-			fmt.Printf("  File: %s\n", line.Function.Filename)
-			fmt.Printf("  Line: %d\n", line.Line)
-			fmt.Printf("  StartLine: %d\n", line.Function.StartLine)
-			fmt.Println("----------------------------------------")
-		}
-	}
-
-	return nil
+	cachePrefix := prefix + ".cache"
+	f.BoolVar(&cfg.Cache.Enabled, cachePrefix+".enabled", false, "Enable debug info caching")
+	f.DurationVar(&cfg.Cache.MaxAge, cachePrefix+".max-age", 7*24*time.Hour, "Maximum age of cached debug info")
 }

pkg/phlaredb/symdb/resolver_tree.go

@@ -2,6 +2,7 @@ package symdb
 
 import (
 	"context"
+	"fmt"
 	"sync"
 
 	pprof "github.com/google/pprof/profile"
@@ -23,8 +24,9 @@ func buildTree(
 ) (*model.Tree, error) {
 	// Try debuginfod symbolization first
 	if symbols != nil && symbols.Symbolizer != nil {
+		//nolint:staticcheck
 		if err := symbolizeLocations(ctx, symbols); err != nil {
-			// TODO: Log error but continue? partial symbolization is better than none
+			// TODO: Log/process error but continue? partial symbolization is better than none
 		}
 	}
 
@@ -250,6 +252,8 @@ func minValue(nodes []Node, maxNodes int64) int64 {
 }
 
 func symbolizeLocations(ctx context.Context, symbols *Symbols) error {
+	var errs []error
+
 	type locToSymbolize struct {
 		idx     int32
 		loc     *schemav1.InMemoryLocation
@@ -259,11 +263,12 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error {
 
 	// Find all locations needing symbolization
 	for i, loc := range symbols.Locations {
+		locCopy := loc
 		if mapping := &symbols.Mappings[loc.MappingId]; symbols.needsDebuginfodSymbolization(&loc, mapping) {
 			buildIDStr := symbols.Strings[mapping.BuildId]
 			locsByBuildId[buildIDStr] = append(locsByBuildId[buildIDStr], locToSymbolize{
 				idx:     int32(i),
-				loc:     &loc,
+				loc:     &locCopy,
 				mapping: mapping,
 			})
 		}
@@ -290,7 +295,7 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error {
 		}
 
 		if err := symbols.Symbolizer.Symbolize(ctx, req); err != nil {
-			// TODO: log/process errors but continue with other build IDs
+			errs = append(errs, fmt.Errorf("symbolize build ID %s: %w", buildID, err))
 			continue
 		}
 
@@ -328,5 +333,10 @@ func symbolizeLocations(ctx context.Context, symbols *Symbols) error {
 			}
 		}
 	}
+
+	if len(errs) > 0 {
+		return fmt.Errorf("symbolization errors: %v", errs)
+	}
+
 	return nil
 }