Skip to content

Prototype(symbolization): Add symbolization in Pyroscope read path #3799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 31 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
268d7f1
feat(symbolization): Add DWARF symbolization POC with debuginfod support
marcsanmi Dec 20, 2024
2024309
fix lint errors
marcsanmi Dec 20, 2024
ad040af
Add symbolization inside the read path
marcsanmi Jan 16, 2025
deb91ca
Add cache for debug files
marcsanmi Jan 17, 2025
8754fc9
Adding symbolizer instrumentation
marcsanmi Jan 19, 2025
9c26ec5
chore: move symbolizatoin into query frontend & add intermediate repr…
marcsanmi Feb 10, 2025
7918c09
fix lint and unstaged-changes check
marcsanmi Feb 19, 2025
362cabd
Update symbolization to symbolize from pprof & improve deubginfod client
marcsanmi Feb 20, 2025
f0a9e7b
Add new flags help
marcsanmi Feb 20, 2025
a76010f
Add multi-layered caching with LRU for symbols, Ristretto for debugin…
marcsanmi Mar 3, 2025
f4c869e
fix testdata symbolized json files since we don't rely on unsymbolize…
marcsanmi Mar 3, 2025
3130e2a
fix another set of tests related to otlp conversion and help options
marcsanmi Mar 3, 2025
2a4b163
Address review several comments
marcsanmi Mar 5, 2025
98ec7f9
place symbolizer flags registration under v2 umbrella
marcsanmi Mar 5, 2025
91a4bb2
Add __needs_symbolization__ metadata label & made debuginfo store non…
marcsanmi Mar 17, 2025
a054f58
Fix flags after rebase
marcsanmi Mar 17, 2025
c38dac7
update metadata labels to work with needs_symbolization & fix maxNodes
marcsanmi Mar 20, 2025
8628f33
Address review and several fixes:
marcsanmi Mar 22, 2025
c650907
Made needsSymbolization thread-safe
marcsanmi Mar 22, 2025
af13cfb
Update .gitattributes w/ normalize
marcsanmi Mar 22, 2025
fa1ff76
ensure valid existing symbolization is preserved
marcsanmi Mar 24, 2025
73a48aa
update symbols check by only checking lines length
marcsanmi Mar 26, 2025
95f8a6f
refactor: added TreeFromBackendProfileSampleType for better performan…
marcsanmi Mar 27, 2025
0008486
refactor: move symbols logic from segment to head & change needsSymbo…
marcsanmi Mar 27, 2025
ca335b8
feat: symbolize only otel-related profiles
marcsanmi Mar 30, 2025
762a596
TODO: review this commit, it has logs and otel only smbolization disa…
marcsanmi Apr 2, 2025
13c2659
Change lidia write to accept interface
marcsanmi Apr 16, 2025
3522cd0
Use new Lidia format insitead of dwarf
marcsanmi Apr 16, 2025
a6bd5c9
Add Lidia binary layout support
marcsanmi Apr 22, 2025
61d2b41
Remove unnecessary debuginfod client raw files cache & address severa…
marcsanmi Apr 22, 2025
1ca0b6b
feat: add symbolizer per tenant overrides (#4136)
marcsanmi Apr 29, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .mockery.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ packages:
github.com/grafana/pyroscope/pkg/frontend/read_path/query_frontend:
interfaces:
QueryBackend:
Symbolizer:
github.com/grafana/pyroscope/pkg/ingester/otlp:
interfaces:
PushService:
Expand Down Expand Up @@ -60,3 +61,7 @@ packages:
interfaces:
Exporter:
Ruler:
github.com/grafana/pyroscope/pkg/experiment/symbolizer:
interfaces:
DebuginfodClient:
DebugInfoStore:
135 changes: 135 additions & 0 deletions cmd/symbolization/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package main

import (
"context"
"fmt"
"io"
"log"
"os"

log2 "github.com/go-kit/log"

googlev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1"
"github.com/grafana/pyroscope/pkg/experiment/symbolizer"
)

const (
// buildID = "c047672cae7964324658491e7dee26748ae5d2f8"
//buildID = "2fa2055ef20fabc972d5751147e093275514b142"
buildID = "6d64b17fbac799e68da7ebd9985ddf9b5cb375e6"
invalidName = "<invalid>"
)

func main() {
client := &localDebuginfodClient{debugFilePath: "/usr/lib/debug/.build-id/6d/64b17fbac799e68da7ebd9985ddf9b5cb375e6.debug"}
logger := log2.NewLogfmtLogger(os.Stdout)
s, err := symbolizer.NewProfileSymbolizer(logger, client, nil, symbolizer.NewMetrics(nil), 10, 10)

if err != nil {
log.Fatalf("Failed to create debuginfod client: %v", err)
}

ctx := context.Background()
_, err = client.FetchDebuginfo(ctx, buildID)
if err != nil {
log.Fatalf("Failed to fetch debug info: %v", err)
}

profile := &googlev1.Profile{
Mapping: []*googlev1.Mapping{{
MemoryStart: 0x28000,
MemoryLimit: 0x1b0000,
FileOffset: 0x28000,
}},
Location: []*googlev1.Location{
{
MappingId: 1,
Address: 0x2a28a,
},
{
MappingId: 1,
Address: 0x124dec,
},
{
MappingId: 1,
Address: 0x2a1c9,
},
},
StringTable: []string{"", buildID},
}

if err := s.SymbolizePprof(ctx, profile); err != nil {
log.Fatalf("Failed to symbolize: %v", err)
}

printResults(profile)
}

func printResults(p *googlev1.Profile) {
fmt.Println("Symbolization Results:")

fmt.Println("\nFunction Table:")
for i, fn := range p.Function {
name := invalidName
if fn.Name >= 0 && int(fn.Name) < len(p.StringTable) {
name = p.StringTable[fn.Name]
}

filename := invalidName
if fn.Filename >= 0 && int(fn.Filename) < len(p.StringTable) {
filename = p.StringTable[fn.Filename]
}

fmt.Printf(" Function[%d]: ID=%d, Name=%s, File=%s, StartLine=%d\n",
i, fn.Id, name, filename, fn.StartLine)
}

fmt.Println("\nLocations:")
for _, loc := range p.Location {
fmt.Printf("\nAddress: 0x%x\n", loc.Address)
if len(loc.Line) == 0 {
fmt.Println(" No symbolization information found")
continue
}

for i, line := range loc.Line {
fmt.Printf(" Line %d (FunctionID=%d):\n", i+1, line.FunctionId)

var fn *googlev1.Function
for _, function := range p.Function {
if function.Id == line.FunctionId {
fn = function
break
}
}

if fn != nil {
name := invalidName
if fn.Name >= 0 && int(fn.Name) < len(p.StringTable) {
name = p.StringTable[fn.Name]
}

filename := invalidName
if fn.Filename >= 0 && int(fn.Filename) < len(p.StringTable) {
filename = p.StringTable[fn.Filename]
}

fmt.Printf(" Function: %s\n", name)
fmt.Printf(" File: %s\n", filename)
fmt.Printf(" Line: %d\n", line.Line)
fmt.Printf(" StartLine: %d\n", fn.StartLine)
} else {
fmt.Printf(" ERROR: Cannot find function with ID %d\n", line.FunctionId)
}
}
}
fmt.Println("\nSymbolization completed successfully.")
}

type localDebuginfodClient struct {
debugFilePath string
}

func (c *localDebuginfodClient) FetchDebuginfo(ctx context.Context, buildID string) (io.ReadCloser, error) {
return os.Open(c.debugFilePath)
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/cespare/xxhash/v2 v2.3.0
github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381
github.com/dennwc/varint v1.0.0
github.com/dgraph-io/ristretto/v2 v2.2.0
github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c
github.com/dolthub/swiss v0.2.1
github.com/drone/envsubst v1.0.3
Expand All @@ -35,6 +36,7 @@ require (
github.com/grafana/pyroscope-go/godeltaprof v0.1.8
github.com/grafana/pyroscope-go/x/k6 v0.0.0-20241003203156-a917cea171d3
github.com/grafana/pyroscope/api v0.4.0
github.com/grafana/pyroscope/lidia v0.0.0-20250416154336-a5c33510d5ff
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE=
github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM=
github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38=
github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c h1:cHaw4wmusVzAZLEPWOCCGCfu6UvFXx9UboCHQCnjvxY=
github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c/go.mod h1:MlkUQveSLEDbIgq2r1e++tSf0zfzU9mQpa9Qkczl+9Y=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
Expand Down Expand Up @@ -375,6 +379,8 @@ github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKt
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grafana/pyroscope-go/x/k6 v0.0.0-20241003203156-a917cea171d3 h1:GtwQDlBz8aJHMy2Ko28UDRGgGzi7v4Vf20+ZyXaGy7M=
github.com/grafana/pyroscope-go/x/k6 v0.0.0-20241003203156-a917cea171d3/go.mod h1:nfbW6/4ke3ywlqLb+Zgr9t1z9Zv3m+2ImUp+vbkzHpc=
github.com/grafana/pyroscope/lidia v0.0.0-20250416154336-a5c33510d5ff h1:9dI8HuAIA5L/EsPXkk3YqA3Pp5i7ECvQkWMwUepLoCY=
github.com/grafana/pyroscope/lidia v0.0.0-20250416154336-a5c33510d5ff/go.mod h1:3f/0/CzhTk1moGlVOp2nJQKCAL+4M5GTnkfDd5hqDIk=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
Expand Down
24 changes: 24 additions & 0 deletions go.work.sum
Original file line number Diff line number Diff line change
Expand Up @@ -896,15 +896,23 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3
github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw=
github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg=
github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4=
github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs=
github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI=
github.com/dave/jennifer v1.6.0 h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI=
github.com/dave/jennifer v1.6.0/go.mod h1:AxTG893FiZKqxy3FP1kL80VMshSMuz2G+EgvszgGRnk=
github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o=
github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE=
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d h1:hUWoLdw5kvo2xCsqlsIBMvWUc1QCSsCYD2J2+Fg6YoU=
github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d/go.mod h1:C7Es+DLenIpPc9J6IYw4jrK0h7S9bKj4DNl8+KxGEXU=
github.com/digitalocean/godo v1.104.1/go.mod h1:VAI/L5YDzMuPRU01lEEUSQ/sp5Z//1HnnFv/RBTEdbg=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
Expand Down Expand Up @@ -936,9 +944,12 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90 h1:WXb3TSNmHp2vHoCro
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8 h1:a9ENSRDFBUPkJ5lCgVZh26+ZbGyoVJG7yb5SSzF5H54=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew=
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA=
github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI=
github.com/go-fonts/stix v0.2.2 h1:v9krocr13J1llaOHLEol1eaHsv8S43UuFX/1bFgEJJ4=
github.com/go-fonts/stix v0.2.2/go.mod h1:SUxggC9dxd/Q+rb5PkJuvfvTbOPtNc2Qaua00fIp9iU=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0=
Expand Down Expand Up @@ -1032,6 +1043,8 @@ github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pS
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg=
github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM=
github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc=
github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk=
github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8=
Expand Down Expand Up @@ -1098,6 +1111,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab h1:HqW4xhhynfjrtEiiSGcQUd6vrK23iMam1FO8rI7mwig=
github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/ionos-cloud/sdk-go/v6 v6.1.9/go.mod h1:EzEgRIDxBELvfoa/uBN0kOQaqovLjUWEB7iW4/Q+t4k=
Expand Down Expand Up @@ -1288,6 +1303,8 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww=
github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245 h1:K1Xf3bKttbF+koVGaX5xngRIZ5bVjbmPnaxE/dR08uY=
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
Expand Down Expand Up @@ -1323,6 +1340,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
Expand Down Expand Up @@ -1439,9 +1458,13 @@ go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw
go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
go.starlark.net v0.0.0-20231101134539-556fd59b42f6 h1:+eC0F/k4aBLC4szgOcjd7bDTEnpxADJyWJE0yowgM3E=
go.starlark.net v0.0.0-20231101134539-556fd59b42f6/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
Expand Down Expand Up @@ -1526,6 +1549,7 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
Expand Down
19 changes: 19 additions & 0 deletions lidia/binary_layout.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package lidia

// BinaryLayoutInfo represents the layout information of the binary
type BinaryLayoutInfo struct {
Type uint16 // e_type from ELF header
ProgramHeaders []ProgramHeaderInfo
}

// ProgramHeaderInfo represents a program header from the ELF file
type ProgramHeaderInfo struct {
Type uint32
Flags uint32
Offset uint64
VirtualAddr uint64
PhysAddr uint64
FileSize uint64
MemSize uint64
Align uint64
}
48 changes: 45 additions & 3 deletions lidia/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package lidia

import (
"encoding/binary"
"fmt"
"sort"
)

Expand Down Expand Up @@ -104,9 +105,10 @@ func (s *sortByVADepth) Swap(i, j int) {

// rangeCollector
type rangeCollector struct {
sb *stringBuilder
rb *rangesBuilder
lb *lineBuilder
sb *stringBuilder
rb *rangesBuilder
lb *lineBuilder
blb *binaryLayoutBuilder

opt options
}
Expand Down Expand Up @@ -135,3 +137,43 @@ func (rc *rangeCollector) VisitRange(r *Range) {
}
rc.rb.add(r.VA, e)
}

type binaryLayoutBuilder struct {
buf []byte
}

func newBinaryLayoutBuilder() *binaryLayoutBuilder {
return &binaryLayoutBuilder{
buf: make([]byte, 0, 256),
}
}

func (blb *binaryLayoutBuilder) write(layout *BinaryLayoutInfo) error {
if layout == nil {
return fmt.Errorf("nil binary layout")
}
Comment on lines +152 to +154
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this possible?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not as of now.


// Reset buffer
blb.buf = blb.buf[:0]

// Write ELF type
blb.buf = binary.LittleEndian.AppendUint16(blb.buf, layout.Type)

// Write number of program headers
count := uint32(len(layout.ProgramHeaders))
blb.buf = binary.LittleEndian.AppendUint32(blb.buf, count)

// Write each program header
for _, ph := range layout.ProgramHeaders {
blb.buf = binary.LittleEndian.AppendUint32(blb.buf, ph.Type)
blb.buf = binary.LittleEndian.AppendUint32(blb.buf, ph.Flags)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.Offset)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.VirtualAddr)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.PhysAddr)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.FileSize)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.MemSize)
blb.buf = binary.LittleEndian.AppendUint64(blb.buf, ph.Align)
}

return nil
}
5 changes: 4 additions & 1 deletion lidia/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,14 @@ const (
version uint32 = 1

// Size of the file header in bytes
headerSize = 0x80
headerSize = 0x98

// Number of fields in a line table entry
lineTableFieldsCount = 2

// Size of each line table entry (2 uint16s or 2 uint32s)
lineTableFieldsSize = 4

Comment on lines 13 to +21
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just in case: does such change invalidate existing objects?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll remove the constant to properly calculate this dynamically as:

entrySize := 4
if hdr.lineTablesHeader.fieldSize == 4 {
    entrySize = 8
}
hdr.binaryLayoutHeader.offset = hdr.lineTablesHeader.offset + uint64(len(rc.lb.entries)*entrySize)

But answer your question, no, this should only affect the offset calculation when writing a new file, ensuring that the binary layout section starts at the correct position. It has no impact on how we read existing files.

Copy link
Contributor Author

@marcsanmi marcsanmi Apr 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

N/A anymore since we removed MapRuntimeAddress so no binaryLayout is needed.

// Number of fields in a range entry
fieldsCount = 8

Expand Down
Loading