Skip to content

Commit 75e2b72

Browse files
bwplotkabwplotka
committed
om2: add om2 complex type format for PoC
This change is for demo purposes, exploring the benefits (and downsides) for the complex type format for OM2 captured in prometheus/docs#2679. This assumes Prometheus stores NS and NHCB (and NH) going forward (for best case efficiency), but is expected to work for classic mode too with little overhead (benchmarks will tell us). Part of the PromCon talk we do with @krajorama Signed-off-by: bwplotka <[email protected]>
1 parent fe4e684 commit 75e2b72

File tree

14 files changed

+2010
-26
lines changed

14 files changed

+2010
-26
lines changed

‎internal/tools/go.mod‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,5 +111,9 @@ require (
111111
google.golang.org/grpc v1.73.0 // indirect
112112
google.golang.org/protobuf v1.36.6 // indirect
113113
gopkg.in/yaml.v3 v3.0.1 // indirect
114+
modernc.org/fileutil v1.2.0 // indirect
115+
modernc.org/golex v1.1.0 // indirect
116+
modernc.org/lex v1.1.1 // indirect
117+
modernc.org/lexer v1.0.5 // indirect
114118
pluginrpc.com/pluginrpc v0.5.0 // indirect
115119
)

‎internal/tools/go.sum‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
186186
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
187187
github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q=
188188
github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
189+
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
189190
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
190191
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
191192
github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
@@ -262,6 +263,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
262263
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
263264
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
264265
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
266+
golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
265267
golang.org/x/exp v0.0.0-20250228200357-dead58393ab7 h1:aWwlzYV971S4BXRS9AmqwDLAD85ouC6X+pocatKY58c=
266268
golang.org/x/exp v0.0.0-20250228200357-dead58393ab7/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
267269
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -324,5 +326,16 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
324326
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
325327
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
326328
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
329+
modernc.org/fileutil v1.1.2/go.mod h1:HdjlliqRHrMAI4nVOvvpYVzVgvRSK7WnoCiG0GUWJNo=
330+
modernc.org/fileutil v1.2.0 h1:c7fsfzHf9WfUFXvv/RY9sStAr+VAKXYGKiAhBQQNoT4=
331+
modernc.org/fileutil v1.2.0/go.mod h1:0rLMFc17WSz6Bm/GtHeme7TOX8pNRhFN2NkfBlOZhrQ=
332+
modernc.org/golex v1.1.0 h1:dmSaksHMd+y6NkBsRsCShNPRaSNCNH+abrVm5/gZic8=
333+
modernc.org/golex v1.1.0/go.mod h1:2pVlfqApurXhR1m0N+WDYu6Twnc4QuvO4+U8HnwoiRA=
334+
modernc.org/lex v1.1.1 h1:prSCNTLw1R4rn7M/RzwsuMtAuOytfyR3cnyM07P+Pas=
335+
modernc.org/lex v1.1.1/go.mod h1:6r8o8DLJkAnOsQaGi8fMoi+Vt6LTbDaCrkUK729D8xM=
336+
modernc.org/lexer v1.0.4/go.mod h1:tOajb8S4sdfOYitzCgXDFmbVJ/LE0v1fNJ7annTw36U=
337+
modernc.org/lexer v1.0.5 h1:NiKuv6LaU6+D2zra31y6FewnAU8LfrtSwHckwdnDSCg=
338+
modernc.org/lexer v1.0.5/go.mod h1:8npHn3u/NxCEtlC/tRSY77x5+WB3HvHMzMVElQ76ayI=
339+
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
327340
pluginrpc.com/pluginrpc v0.5.0 h1:tOQj2D35hOmvHyPu8e7ohW2/QvAnEtKscy2IJYWQ2yo=
328341
pluginrpc.com/pluginrpc v0.5.0/go.mod h1:UNWZ941hcVAoOZUn8YZsMmOZBzbUjQa3XMns8RQLp9o=

‎model/textparse/README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Making changes to textparse lexers
2-
In the rare case that you need to update the textparse lexers, edit promlex.l or openmetricslex.l and then run the following command:
3-
`golex -o=promlex.l.go promlex.l`
42

5-
Note that you need golex installed:
6-
`go get -u modernc.org/golex`
3+
Run from the repo root:
4+
5+
```bash
6+
bash ./scripts/gentextlex.sh
7+
```

‎model/textparse/benchmark_test.go‎

Lines changed: 60 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@ import (
2323
"strings"
2424
"testing"
2525

26+
"github.com/google/go-cmp/cmp"
2627
"github.com/prometheus/common/expfmt"
2728
"github.com/prometheus/common/model"
2829
"github.com/stretchr/testify/require"
2930

3031
"github.com/prometheus/prometheus/model/exemplar"
3132
"github.com/prometheus/prometheus/model/labels"
33+
"github.com/prometheus/prometheus/util/testutil"
3234
)
3335

3436
// BenchmarkParse... set of benchmarks analyze efficiency of parsing various
@@ -138,32 +140,81 @@ func BenchmarkParseOpenMetricsNHCB(b *testing.B) {
138140
}
139141
}
140142

141-
func benchParse(b *testing.B, data []byte, parser string) {
142-
type newParser func([]byte, *labels.SymbolTable) Parser
143+
// BenchmarkParseOpenMetricsNHCB_OM1vs2 is for demo of the benefit for the complex
144+
// type format for OM2, assuming Prometheus stores NS and NHCB (and NH) going forward.
145+
// Format draft: https://github.com/prometheus/docs/pull/2679
146+
/*
147+
export bench=out && go test ./model/textparse/... \
148+
-run '^$' -bench '^BenchmarkParseOpenMetricsNHCB_OM1vs2' \
149+
-benchtime 2s -count 6 -cpu 2 -benchmem -timeout 999m \
150+
| tee ${bench}.txt
151+
*/
152+
func BenchmarkParseOpenMetricsNHCB_OM1vs2(b *testing.B) {
153+
parseCases := []struct {
154+
parser string
155+
data []byte
156+
}{
157+
{
158+
parser: "omtext_with_nhcb", // Measure NHCB over OM parser.
159+
data: readTestdataFile(b, "1histogram.om.txt"),
160+
},
161+
{
162+
parser: "om2text_with_nhcb", // https://github.com/prometheus/docs/pull/2679 with NHCB output.
163+
data: readTestdataFile(b, "1histogram.om2.txt"),
164+
},
165+
}
166+
167+
// Before we go, test parsing works as expected.
168+
gotA := testParse(b, newParser(b, parseCases[0].parser)(parseCases[0].data, labels.NewSymbolTable()))
169+
gotB := testParse(b, newParser(b, parseCases[1].parser)(parseCases[1].data, labels.NewSymbolTable()))
170+
testutil.RequireEqualWithOptions(b, gotA, gotB, []cmp.Option{cmp.AllowUnexported(parsedEntry{})})
171+
172+
// For fun, OM2 parser should work with classic histogram too (TODO add separate tests).
173+
_ = testParse(b, newParser(b, parseCases[1].parser)(parseCases[0].data, labels.NewSymbolTable()))
174+
175+
for _, c := range parseCases {
176+
b.Run(fmt.Sprintf("parser=%v", c.parser), func(b *testing.B) {
177+
benchParse(b, c.data, c.parser)
178+
})
179+
}
180+
}
181+
182+
func newParser(t testing.TB, parser string) func([]byte, *labels.SymbolTable) Parser {
183+
t.Helper()
143184

144-
var newParserFn newParser
145185
switch parser {
146186
case "promtext":
147-
newParserFn = func(b []byte, st *labels.SymbolTable) Parser {
187+
return func(b []byte, st *labels.SymbolTable) Parser {
148188
return NewPromParser(b, st, false)
149189
}
150190
case "promproto":
151-
newParserFn = func(b []byte, st *labels.SymbolTable) Parser {
191+
return func(b []byte, st *labels.SymbolTable) Parser {
152192
return NewProtobufParser(b, true, false, false, st)
153193
}
154194
case "omtext":
155-
newParserFn = func(b []byte, st *labels.SymbolTable) Parser {
195+
return func(b []byte, st *labels.SymbolTable) Parser {
156196
return NewOpenMetricsParser(b, st, WithOMParserCTSeriesSkipped())
157197
}
158198
case "omtext_with_nhcb":
159-
newParserFn = func(buf []byte, st *labels.SymbolTable) Parser {
199+
return func(buf []byte, st *labels.SymbolTable) Parser {
160200
p, err := New(buf, "application/openmetrics-text", st, ParserOptions{ConvertClassicHistogramsToNHCB: true})
161-
require.NoError(b, err)
201+
require.NoError(t, err)
162202
return p
163203
}
204+
case "om2text_with_nhcb":
205+
return func(b []byte, st *labels.SymbolTable) Parser {
206+
return NewOpenMetrics2Parser(b, st, func(options *openMetrics2ParserOptions) {
207+
options.unrollComplexTypes = false
208+
})
209+
}
164210
default:
165-
b.Fatal("unknown parser", parser)
211+
t.Fatal("unknown parser", parser)
166212
}
213+
return nil
214+
}
215+
216+
func benchParse(b *testing.B, data []byte, parser string) {
217+
newParserFn := newParser(b, parser)
167218

168219
var (
169220
res labels.Labels

‎model/textparse/interface_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ func requireEntries(t *testing.T, exp, got []parsedEntry) {
226226
})
227227
}
228228

229-
func testParse(t *testing.T, p Parser) (ret []parsedEntry) {
229+
func testParse(t testing.TB, p Parser) (ret []parsedEntry) {
230230
t.Helper()
231231

232232
for {

‎model/textparse/openmetrics2lex.l‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
%{
2+
// Copyright 2018 The Prometheus Authors
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package textparse
16+
17+
import (
18+
"fmt"
19+
)
20+
21+
// Lex is called by the parser generated by "go tool yacc" to obtain each
22+
// token. The method is opened before the matching rules block and closed at
23+
// the end of the file.
24+
func (l *openMetrics2Lexer) Lex() token {
25+
if l.i >= len(l.b) {
26+
return tEOF
27+
}
28+
c := l.b[l.i]
29+
l.start = l.i
30+
31+
%}
32+
33+
D [0-9]
34+
L [a-zA-Z_]
35+
M [a-zA-Z_:]
36+
C [^\n]
37+
S [ ]
38+
39+
%x sComment sMeta1 sMeta2 sLabels sLValue sValue sTimestamp sExemplar sEValue sETimestamp sCValue
40+
41+
%yyc c
42+
%yyn c = l.next()
43+
%yyt l.state
44+
45+
46+
%%
47+
48+
#{S} l.state = sComment
49+
<sComment>HELP{S} l.state = sMeta1; return tHelp
50+
<sComment>TYPE{S} l.state = sMeta1; return tType
51+
<sComment>UNIT{S} l.state = sMeta1; return tUnit
52+
<sComment>"EOF"\n? l.state = sInit; return tEOFWord
53+
<sMeta1>\"(\\.|[^\\"])*\" l.state = sMeta2; return tMName
54+
<sMeta1>{M}({M}|{D})* l.state = sMeta2; return tMName
55+
<sMeta2>{S}{C}*\n l.state = sInit; return tText
56+
57+
{M}({M}|{D})* l.state = sValue; return tMName
58+
<sValue>\{ l.state = sLabels; return tBraceOpen
59+
\{ l.state = sLabels; return tBraceOpen
60+
<sLabels>{L}({L}|{D})* return tLName
61+
<sLabels>\"(\\.|[^\\"])*\" l.state = sLabels; return tQString
62+
<sLabels>\} l.state = sValue; return tBraceClose
63+
<sLabels>= l.state = sLValue; return tEqual
64+
<sLabels>, return tComma
65+
<sLValue>\"(\\.|[^\\"\n])*\" l.state = sLabels; return tLValue
66+
67+
<sValue>{S}\{ l.state = sCValue; return tBraceOpen
68+
<sCValue>{L}({L}|{D})* return tLName
69+
<sCValue>: return tColon
70+
<sCValue>\[ return tBracketOpen
71+
<sCValue>\] return tBracketClose
72+
<sCValue>, return tComma
73+
<sCValue>[ \t]+ // Skip whitespace inside the block
74+
<sCValue>[^ \n\t,\[\]{}:]+ return tValue
75+
<sCValue>\} l.state = sTimestamp; return tBraceClose
76+
77+
<sValue>{S}[^{ \n]+ l.state = sTimestamp; return tValue
78+
<sTimestamp>{S}[^ \n]+ return tTimestamp
79+
<sTimestamp>\n l.state = sInit; return tLinebreak
80+
<sTimestamp>{S}#{S}\{ l.state = sExemplar; return tComment
81+
82+
<sExemplar>{L}({L}|{D})* return tLName
83+
<sExemplar>\"(\\.|[^\\"\n])*\" l.state = sExemplar; return tQString
84+
<sExemplar>\} l.state = sEValue; return tBraceClose
85+
<sExemplar>= l.state = sEValue; return tEqual
86+
<sEValue>\"(\\.|[^\\"\n])*\" l.state = sExemplar; return tLValue
87+
<sExemplar>, return tComma
88+
<sEValue>{S}[^ \n]+ l.state = sETimestamp; return tValue
89+
<sETimestamp>{S}[^ \n]+ return tTimestamp
90+
<sETimestamp>\n l.state = sInit; return tLinebreak
91+
92+
%%
93+
94+
return tInvalid
95+
}

0 commit comments

Comments
 (0)