Skip to content

Commit 5c90de4

Browse files
committed
reporter: Add support for v2 schema
1 parent 3bfb906 commit 5c90de4

File tree

9 files changed

+1530
-50
lines changed

9 files changed

+1530
-50
lines changed

flags/flags.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,8 @@ type FlagsRemoteStore struct {
345345

346346
ClientCert string `help:"Client certificate for mTLS"`
347347
ClientKey string `help:"Client key for mTLS"`
348+
349+
UseV2Schema bool `default:"false" help:"Use v2 Arrow schema with inline stacktraces and ListView deduplication (experimental)."`
348350
}
349351

350352
// FlagsDebuginfo contains flags to configure debuginfo.

go.mod

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ module github.com/parca-dev/parca-agent
33
go 1.25.0
44

55
require (
6-
buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2
7-
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1
6+
buf.build/gen/go/parca-dev/parca/grpc/go v1.6.1-20260225102827-5fda07223114.1
7+
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20260225102827-5fda07223114.1
88
buf.build/gen/go/prometheus/prometheus/protocolbuffers/go v1.36.6-20250320161912-af2aab87b1b3.1
99
github.com/KimMachineGun/automemlimit v0.7.3
1010
github.com/alecthomas/kong v1.12.1
1111
github.com/alecthomas/kong-yaml v0.2.0
12-
github.com/apache/arrow/go/v16 v16.1.0
12+
github.com/apache/arrow/go/v18 v18.0.0-20241007013041-ab95a4d25142
1313
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
1414
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
1515
github.com/containerd/containerd v1.7.29
@@ -64,6 +64,8 @@ require (
6464
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
6565
github.com/Microsoft/go-winio v0.6.2 // indirect
6666
github.com/Microsoft/hcsshim v0.12.9 // indirect
67+
github.com/andybalholm/brotli v1.1.0 // indirect
68+
github.com/apache/thrift v0.20.0 // indirect
6769
github.com/beorn7/perks v1.0.1 // indirect
6870
github.com/blang/semver/v4 v4.0.0 // indirect
6971
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -111,11 +113,14 @@ require (
111113
github.com/josharian/native v1.1.0 // indirect
112114
github.com/jpillora/backoff v1.0.0 // indirect
113115
github.com/json-iterator/go v1.1.12 // indirect
116+
github.com/klauspost/asmfmt v1.3.2 // indirect
114117
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
115118
github.com/mailru/easyjson v0.9.0 // indirect
116119
github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d // indirect
117120
github.com/mdlayher/netlink v1.7.2 // indirect
118121
github.com/mdlayher/socket v0.4.1 // indirect
122+
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
123+
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
119124
github.com/minio/sha256-simd v1.0.1 // indirect
120125
github.com/moby/docker-image-spec v1.3.1 // indirect
121126
github.com/moby/locker v1.0.1 // indirect

go.sum

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
buf.build/gen/go/gogo/protobuf/protocolbuffers/go v1.36.6-20240617172848-e1dbca2775a7.1 h1:DHj/fDjM+Ij3KR1IpFs6WdNcCtD4+Th3tEWyx3Xgs14=
22
buf.build/gen/go/gogo/protobuf/protocolbuffers/go v1.36.6-20240617172848-e1dbca2775a7.1/go.mod h1:iCb72C37pWGhjKDeq9IbcMqVJAnXXHs5tEjiePfouhk=
3-
buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2 h1:AiTEEAQ8AHE5udDizcA2r3GNjxxlk4f2Buot/hA2oYM=
4-
buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2/go.mod h1:U8BtFPtz71GSALR7K7ALn39RnvrKsD++lJzMI5Gf4fs=
5-
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1 h1:gXHJuGlWoXUkH9O9Qxw9skNAhIDgNgJx0tYSbI8fjIo=
6-
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1/go.mod h1:58eXMQL4tavOGzyXIveWU4f3yTFGmLIHxM9uk11DOIo=
3+
buf.build/gen/go/parca-dev/parca/grpc/go v1.6.1-20260225102827-5fda07223114.1 h1:vxHTP5quJBa7/yMhL5KHfci9LxgB970bBJPByU9K9EQ=
4+
buf.build/gen/go/parca-dev/parca/grpc/go v1.6.1-20260225102827-5fda07223114.1/go.mod h1:Gm9/4TuxSC1DmFMqCBuLOWQ2nbRPjt7rSEQiTIksYBY=
5+
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20260225102827-5fda07223114.1 h1:lEamNY09GPFoJ3avHHtvnCR+mjx2sZx418nTVrgjC9Q=
6+
buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.11-20260225102827-5fda07223114.1/go.mod h1:TR9iiFuhuMGsEil3U6KZnYmvZ1W2SY6uw5HPP+4mJU4=
77
buf.build/gen/go/prometheus/prometheus/protocolbuffers/go v1.36.6-20250320161912-af2aab87b1b3.1 h1:EuFqAB/kfs/jh9aUGcvBjcxtU89wnXwsuQfcwGX1rhE=
88
buf.build/gen/go/prometheus/prometheus/protocolbuffers/go v1.36.6-20250320161912-af2aab87b1b3.1/go.mod h1:ea/VK8bRnfyOuhQRzIk5hGigCqbZdzI8SHNLC3IyABU=
99
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -18,6 +18,8 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod
1818
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
1919
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
2020
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
21+
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
22+
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
2123
github.com/KimMachineGun/automemlimit v0.7.3 h1:oPgMp0bsWez+4fvgSa11Rd9nUDrd8RLtDjBoT3ro+/A=
2224
github.com/KimMachineGun/automemlimit v0.7.3/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
2325
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
@@ -32,8 +34,12 @@ github.com/alecthomas/kong-yaml v0.2.0 h1:iiVVqVttmOsHKawlaW/TljPsjaEv1O4ODx6dlo
3234
github.com/alecthomas/kong-yaml v0.2.0/go.mod h1:vMvOIy+wpB49MCZ0TA3KMts38Mu9YfRP03Q1StN69/g=
3335
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
3436
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
35-
github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI=
36-
github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA=
37+
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
38+
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
39+
github.com/apache/arrow/go/v18 v18.0.0-20241007013041-ab95a4d25142 h1:6EtsUpu9/vLtVl6oVpFiZe9GRax7STd2bG55VNwsRdI=
40+
github.com/apache/arrow/go/v18 v18.0.0-20241007013041-ab95a4d25142/go.mod h1:GjCnS5QddrJzyqrdYqCUvwlND7SfAw4WH/722M2U2NM=
41+
github.com/apache/thrift v0.20.0 h1:631+KvYbsBZxmuJjYwhezVsrfc/TbqtZV4QcxOX1fOI=
42+
github.com/apache/thrift v0.20.0/go.mod h1:hOk1BQqcp2OLzGsyVXdfMk7YFlMxK3aoEVhjD06QhB8=
3743
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs=
3844
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
3945
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -219,6 +225,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
219225
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
220226
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
221227
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
228+
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
229+
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
222230
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
223231
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
224232
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
@@ -246,6 +254,10 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/
246254
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
247255
github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
248256
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
257+
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
258+
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
259+
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
260+
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
249261
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
250262
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
251263
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=

main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
telemetrygrpc "buf.build/gen/go/parca-dev/parca/grpc/go/parca/telemetry/v1alpha1/telemetryv1alpha1grpc"
2525
telemetrypb "buf.build/gen/go/parca-dev/parca/protocolbuffers/go/parca/telemetry/v1alpha1"
2626
_ "github.com/KimMachineGun/automemlimit"
27-
"github.com/apache/arrow/go/v16/arrow/memory"
27+
"github.com/apache/arrow/go/v18/arrow/memory"
2828
"github.com/armon/circbuf"
2929
"github.com/common-nighthawk/go-figure"
3030
"github.com/felixge/fgprof"
@@ -424,6 +424,7 @@ func mainWithExitCode() flags.ExitCode {
424424
f.Metadata.DisableCPULabel,
425425
f.Metadata.DisableThreadIDLabel,
426426
f.Metadata.DisableThreadCommLabel,
427+
f.RemoteStore.UseV2Schema,
427428
)
428429
if err != nil {
429430
return flags.Failure("Failed to start reporting: %v", err)

reporter/arrow.go

Lines changed: 74 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,70 @@ import (
55
"slices"
66
"unsafe"
77

8-
"github.com/apache/arrow/go/v16/arrow"
9-
"github.com/apache/arrow/go/v16/arrow/array"
10-
"github.com/apache/arrow/go/v16/arrow/memory"
8+
"github.com/apache/arrow/go/v18/arrow"
9+
"github.com/apache/arrow/go/v18/arrow/array"
10+
"github.com/apache/arrow/go/v18/arrow/memory"
1111
"golang.org/x/exp/maps"
1212
)
1313

14+
func stringRunEndBuilder(arr array.Builder) *StringRunEndBuilder {
15+
ree := arr.(*array.RunEndEncodedBuilder)
16+
sb := ree.ValueBuilder().(*array.StringBuilder)
17+
return &StringRunEndBuilder{
18+
ree: ree,
19+
sb: sb,
20+
}
21+
}
22+
23+
type StringRunEndBuilder struct {
24+
ree *array.RunEndEncodedBuilder
25+
sb *array.StringBuilder
26+
}
27+
28+
func (b *StringRunEndBuilder) Release() {
29+
b.ree.Release()
30+
}
31+
32+
func (b *StringRunEndBuilder) NewArray() arrow.Array {
33+
return b.ree.NewArray()
34+
}
35+
36+
func (b *StringRunEndBuilder) Len() int {
37+
return b.ree.Len()
38+
}
39+
40+
func (b *StringRunEndBuilder) EnsureLength(l int) {
41+
for b.ree.Len() < l {
42+
b.AppendNull()
43+
}
44+
}
45+
46+
func (b *StringRunEndBuilder) AppendNull() {
47+
b.ree.AppendNull()
48+
}
49+
50+
func (b *StringRunEndBuilder) AppendString(v string) {
51+
if b.sb.Len() > 0 &&
52+
!b.sb.IsNull(b.sb.Len()-1) &&
53+
v == b.sb.Value(int(b.sb.Len()-1)) {
54+
b.ree.ContinueRun(1)
55+
return
56+
}
57+
b.ree.Append(1)
58+
b.sb.Append(v)
59+
}
60+
61+
func (b *StringRunEndBuilder) AppendStringN(v string, n uint64) {
62+
if b.sb.Len() > 0 &&
63+
!b.sb.IsNull(b.sb.Len()-1) &&
64+
v == b.sb.Value(int(b.sb.Len()-1)) {
65+
b.ree.ContinueRun(n)
66+
return
67+
}
68+
b.ree.Append(n)
69+
b.sb.Append(v)
70+
}
71+
1472
func binaryDictionaryRunEndBuilder(arr array.Builder) *BinaryDictionaryRunEndBuilder {
1573
ree := arr.(*array.RunEndEncodedBuilder)
1674
bd := ree.ValueBuilder().(*array.BinaryDictionaryBuilder)
@@ -105,6 +163,10 @@ func (b *Uint64RunEndBuilder) NewArray() arrow.Array {
105163
return b.ree.NewArray()
106164
}
107165

166+
func (b *Uint64RunEndBuilder) Append(v uint64) {
167+
b.AppendN(v, 1)
168+
}
169+
108170
func (b *Uint64RunEndBuilder) AppendN(v uint64, n uint64) {
109171
if b.ub.Len() > 0 && v == b.ub.Value(b.ub.Len()-1) {
110172
b.ree.ContinueRun(n)
@@ -157,11 +219,11 @@ type LocationsWriter struct {
157219
MappingBuildID *BinaryDictionaryRunEndBuilder
158220
Lines *array.ListBuilder
159221
Line *array.StructBuilder
160-
LineNumber *array.Int64Builder
222+
LineNumber *array.Uint64Builder
161223
FunctionName *array.BinaryDictionaryBuilder
162224
FunctionSystemName *array.BinaryDictionaryBuilder
163225
FunctionFilename *BinaryDictionaryRunEndBuilder
164-
FunctionStartLine *array.Int64Builder
226+
FunctionStartLine *array.Uint64Builder
165227
}
166228

167229
func (w *LocationsWriter) NewRecord(stacktraceIDs *array.Binary) arrow.Record {
@@ -208,7 +270,7 @@ type SampleWriter struct {
208270
PeriodUnit *BinaryDictionaryRunEndBuilder
209271
Temporality *BinaryDictionaryRunEndBuilder
210272
Period *Int64RunEndBuilder
211-
Duration *Int64RunEndBuilder
273+
Duration *Uint64RunEndBuilder
212274
Timestamp *Int64RunEndBuilder
213275
}
214276

@@ -305,7 +367,7 @@ var (
305367
Name: "lines",
306368
Type: arrow.ListOf(arrow.StructOf([]arrow.Field{{
307369
Name: "line",
308-
Type: arrow.PrimitiveTypes.Int64,
370+
Type: arrow.PrimitiveTypes.Uint64,
309371
}, {
310372
Name: "function_name",
311373
Type: &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint32, ValueType: arrow.BinaryTypes.Binary},
@@ -320,7 +382,7 @@ var (
320382
),
321383
}, {
322384
Name: "function_start_line",
323-
Type: arrow.PrimitiveTypes.Int64,
385+
Type: arrow.PrimitiveTypes.Uint64,
324386
}}...)),
325387
}}...)),
326388
}
@@ -393,7 +455,7 @@ var (
393455

394456
DurationField = arrow.Field{
395457
Name: "duration",
396-
Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64),
458+
Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Uint64),
397459
}
398460

399461
TimestampField = arrow.Field{
@@ -493,11 +555,11 @@ func NewLocationsWriter(mem memory.Allocator) *LocationsWriter {
493555

494556
lines := locations.FieldBuilder(7).(*array.ListBuilder)
495557
line := lines.ValueBuilder().(*array.StructBuilder)
496-
lineNumber := line.FieldBuilder(0).(*array.Int64Builder)
558+
lineNumber := line.FieldBuilder(0).(*array.Uint64Builder)
497559
functionName := line.FieldBuilder(1).(*array.BinaryDictionaryBuilder)
498560
functionSystemName := line.FieldBuilder(2).(*array.BinaryDictionaryBuilder)
499561
functionFilename := binaryDictionaryRunEndBuilder(line.FieldBuilder(3))
500-
functionStartLine := line.FieldBuilder(4).(*array.Int64Builder)
562+
functionStartLine := line.FieldBuilder(4).(*array.Uint64Builder)
501563

502564
return &LocationsWriter{
503565
IsComplete: isComplete,
@@ -530,7 +592,7 @@ func NewSampleWriter(mem memory.Allocator) *SampleWriter {
530592
periodUnit := binaryDictionaryRunEndBuilder(array.NewBuilder(mem, PeriodUnitField.Type))
531593
temporality := binaryDictionaryRunEndBuilder(array.NewBuilder(mem, TemporalityField.Type))
532594
period := int64RunEndBuilder(array.NewBuilder(mem, PeriodField.Type))
533-
duration := int64RunEndBuilder(array.NewBuilder(mem, DurationField.Type))
595+
duration := uint64RunEndBuilder(array.NewBuilder(mem, DurationField.Type))
534596
timestamp := int64RunEndBuilder(array.NewBuilder(mem, TimestampField.Type))
535597

536598
return &SampleWriter{

0 commit comments

Comments
 (0)