diff --git a/.buildkite/pipeline-default.yml b/.buildkite/pipeline-default.yml index 4457b0aef..244e970ab 100644 --- a/.buildkite/pipeline-default.yml +++ b/.buildkite/pipeline-default.yml @@ -35,36 +35,7 @@ agents: queue: buildkite-gcp steps: - - name: ":go: 1.21 test - %n" - parallelism: 2 - plugins: - - kubernetes: - <<: *kubernetes - podSpec: - <<: *podSpec - containers: - - <<: *commandContainer - command: - - |- - make test - - docker-compose#v3.13.0: - run: yarpc-go-1.21 - - - name: ":go: 1.21 examples" - plugins: - - kubernetes: - <<: *kubernetes - podSpec: - <<: *podSpec - containers: - - <<: *commandContainer - command: - - |- - make examples - - docker-compose#v3.13.0: - run: yarpc-go-1.21 - - - name: ":go: 1.22 test - %n" + - name: ":go: 1.23 test - %n" parallelism: 6 plugins: - kubernetes: @@ -77,9 +48,9 @@ steps: - |- make codecov - docker-compose#v3.13.0: - run: yarpc-go-1.22 + run: yarpc-go-1.23 - - name: ":go: 1.22 crossdock" + - name: ":go: 1.23 crossdock" plugins: - kubernetes: <<: *kubernetes @@ -91,9 +62,9 @@ steps: - |- make crossdock-codecov - docker-compose#v3.13.0: - run: yarpc-go-1.22 + run: yarpc-go-1.23 - - name: ":go: 1.22 lint" + - name: ":go: 1.23 lint" plugins: - kubernetes: <<: *kubernetes @@ -105,9 +76,9 @@ steps: - |- make lint - docker-compose#v3.13.0: - run: yarpc-go-1.22 + run: yarpc-go-1.23 - - name: ":go: 1.22 examples" + - name: ":go: 1.23 examples" plugins: - kubernetes: <<: *kubernetes @@ -119,4 +90,4 @@ steps: - |- make examples - docker-compose#v3.13.0: - run: yarpc-go-1.22 + run: yarpc-go-1.23 diff --git a/Dockerfile.1.23 b/Dockerfile.1.23 new file mode 100644 index 000000000..bfd1cac68 --- /dev/null +++ b/Dockerfile.1.23 @@ -0,0 +1,24 @@ +FROM golang:1.23 + +ENV SUPPRESS_DOCKER 1 +WORKDIR /yarpc +RUN apt-get update -yq && apt-get install -yq jq unzip netcat-openbsd +ADD dockerdeps.mk /yarpc/ +ADD etc/make/base.mk etc/make/deps.mk /yarpc/etc/make/ +RUN make -f dockerdeps.mk predeps +ADD etc/bin/vendor-build.sh /yarpc/etc/bin/ + +# Download and cache dependencies in the image so that we're not constantly +# re-downloading them locally. + +ADD tools_test.go go.mod go.sum /yarpc/ +RUN go mod download + +ADD internal/examples/go.mod /yarpc/internal/examples/ +RUN cd /yarpc/internal/examples && go mod download + +ADD internal/crossdock/go.mod /yarpc/internal/crossdock/ +RUN cd /yarpc/internal/crossdock && go mod download + +RUN make -f dockerdeps.mk deps +ADD . /yarpc/ diff --git a/api/encoding/call.go b/api/encoding/call.go index 363e538f7..5a399a73c 100644 --- a/api/encoding/call.go +++ b/api/encoding/call.go @@ -22,6 +22,7 @@ package encoding import ( "context" + "iter" "sort" "go.uber.org/yarpc/api/transport" @@ -140,6 +141,22 @@ func (c *Call) OriginalHeaders() map[string]string { return h } +// OriginalHeadersAll returns an iterator over the original (non-canonicalized) +// header key-value pairs provided with the request. +// The header keys are not canonicalized and suitable for case-sensitive transport like TChannel. +func (c *Call) OriginalHeadersAll() iter.Seq2[string, string] { + return func(yield func(string, string) bool) { + if c == nil { + return + } + for k, v := range c.md.Headers().OriginalItemsAll() { + if !yield(k, v) { + return + } + } + } +} + // HeaderNames returns a sorted list of the names of user defined headers // provided with this request. func (c *Call) HeaderNames() []string { @@ -156,6 +173,30 @@ func (c *Call) HeaderNames() []string { return names } +// HeaderNamesAll returns an iterator over the names of user defined headers +// provided with this request. +func (c *Call) HeaderNamesAll() iter.Seq[string] { + return func(yield func(string) bool) { + if c == nil { + return + } + for k := range c.md.Headers().All() { + if !yield(k) { + return + } + } + } +} + +// HeadersLen returns the number of user defined headers provided with this request. +// Useful for pre-allocating slices. +func (c *Call) HeadersLen() int { + if c == nil { + return 0 + } + return c.md.Headers().Len() +} + // ShardKey returns the shard key for this request. func (c *Call) ShardKey() string { if c == nil { diff --git a/api/encoding/call_test.go b/api/encoding/call_test.go index 23dcc1955..01c2b938d 100644 --- a/api/encoding/call_test.go +++ b/api/encoding/call_test.go @@ -22,6 +22,8 @@ package encoding import ( "context" + "slices" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -46,6 +48,8 @@ func TestNilCall(t *testing.T) { assert.Equal(t, "", call.OriginalHeader("foo")) assert.Empty(t, call.HeaderNames()) assert.Nil(t, call.OriginalHeaders()) + assert.Equal(t, 0, call.HeadersLen()) + assert.Len(t, slices.Collect(call.HeaderNamesAll()), 0, "nil call should yield no headers") assert.Error(t, call.WriteResponseHeader("foo", "bar")) } @@ -82,6 +86,20 @@ func TestReadFromRequest(t *testing.T) { assert.Equal(t, map[string]string{"Foo": "Bar", "foo": "bar"}, call.OriginalHeaders()) assert.Equal(t, "cp", call.CallerProcedure()) assert.Len(t, call.HeaderNames(), 1) + assert.Equal(t, 1, call.HeadersLen()) + + headerNames := call.HeaderNames() + headerNamesFromIterator := slices.Collect(call.HeaderNamesAll()) + slices.Sort(headerNames) + slices.Sort(headerNamesFromIterator) + assert.Equal(t, headerNames, headerNamesFromIterator) + + originalHeaders := call.OriginalHeaders() + headersFromIterator := make(map[string]string) + for k, v := range call.OriginalHeadersAll() { + headersFromIterator[k] = v + } + assert.Equal(t, originalHeaders, headersFromIterator) assert.NoError(t, call.WriteResponseHeader("foo2", "bar2")) assert.Equal(t, icall.resHeaders[0].k, "foo2") @@ -158,3 +176,77 @@ func TestDisabledResponseHeaders(t *testing.T) { assert.Error(t, call.WriteResponseHeader("foo", "bar")) assert.Nil(t, icall.resHeaders) } + +func BenchmarkCallHeaderNames(b *testing.B) { + benchmarkSizes := []int{1, 2, 3, 4, 5, 6, 8, 10, 25, 50, 100} + + testCalls := make(map[int]*Call) + for _, size := range benchmarkSizes { + headers := transport.NewHeadersWithCapacity(size) + for i := 0; i < size; i++ { + headers = headers.With("header-"+strconv.Itoa(i), "value-"+strconv.Itoa(i)) + } + ctx, icall := NewInboundCall(context.Background()) + icall.ReadFromRequest(&transport.Request{Headers: headers}) + testCalls[size] = CallFromContext(ctx) + } + + // Benchmark HeaderNames (with sorting). + b.Run("HeaderNames", func(b *testing.B) { + for _, size := range benchmarkSizes { + call := testCalls[size] + b.Run("size="+strconv.Itoa(size), func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + call.HeaderNames() + } + }) + } + }) + + // Benchmark HeaderNamesAll (no sorting). + b.Run("HeaderNamesAll", func(b *testing.B) { + for _, size := range benchmarkSizes { + call := testCalls[size] + b.Run("size="+strconv.Itoa(size), func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Consume the iterator. + for name := range call.HeaderNamesAll() { + _ = name + } + } + }) + } + }) + + // Benchmark OriginalHeaders (creates map copy). + b.Run("OriginalHeaders", func(b *testing.B) { + for _, size := range benchmarkSizes { + call := testCalls[size] + b.Run("size="+strconv.Itoa(size), func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + call.OriginalHeaders() + } + }) + } + }) + + // Benchmark OriginalHeadersAll (no copy). + b.Run("OriginalHeadersAll", func(b *testing.B) { + for _, size := range benchmarkSizes { + call := testCalls[size] + b.Run("size="+strconv.Itoa(size), func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Consume the iterator. + for k, v := range call.OriginalHeadersAll() { + _ = k + _ = v + } + } + }) + } + }) +} diff --git a/api/transport/header.go b/api/transport/header.go index 8215a0807..6b7d6b56d 100644 --- a/api/transport/header.go +++ b/api/transport/header.go @@ -20,7 +20,10 @@ package transport -import "strings" +import ( + "iter" + "strings" +) // CanonicalizeHeaderKey canonicalizes the given header key for storage into // Headers. @@ -103,6 +106,18 @@ func (h Headers) Items() map[string]string { return h.items } +// All returns an iterator over the header key-value pairs. +// Keys are normalized using CanonicalizeHeaderKey. +func (h Headers) All() iter.Seq2[string, string] { + return func(yield func(string, string) bool) { + for k, v := range h.items { + if !yield(k, v) { + return + } + } + } +} + // OriginalItems returns the non-canonicalized version of the underlying map // for this Headers object. The returned map MUST NOT be changed. // Doing so will result in undefined behavior. @@ -110,6 +125,18 @@ func (h Headers) OriginalItems() map[string]string { return h.originalItems } +// OriginalItemsAll returns an iterator over the original (non-canonicalized) +// header key-value pairs. +func (h Headers) OriginalItemsAll() iter.Seq2[string, string] { + return func(yield func(string, string) bool) { + for k, v := range h.originalItems { + if !yield(k, v) { + return + } + } + } +} + // HeadersFromMap builds a new Headers object from the given map of header // key-value pairs. func HeadersFromMap(m map[string]string) Headers { diff --git a/docker-compose.yml b/docker-compose.yml index 2cbe4a967..f588824e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,3 +48,24 @@ services: - BUILDKITE_PROJECT_SLUG - BUILDKITE_REPO - GO111MODULE=on + + yarpc-go-1.23: + build: + context: . + dockerfile: Dockerfile.1.23 + environment: + - TEST_TIME_SCALE=5 + - THIS_CHUNK=${BUILDKITE_PARALLEL_JOB} + - TOTAL_CHUNKS=${BUILDKITE_PARALLEL_JOB_COUNT} + - CODECOV_TOKEN + - CI=true + - BUILDKITE + - BUILDKITE_AGENT_ID + - BUILDKITE_BRANCH + - BUILDKITE_BUILD_NUMBER + - BUILDKITE_BUILD_URL + - BUILDKITE_COMMIT + - BUILDKITE_JOB_ID + - BUILDKITE_PROJECT_SLUG + - BUILDKITE_REPO + - GO111MODULE=on diff --git a/etc/make/docker.mk b/etc/make/docker.mk index b13c7987f..abd14a24e 100644 --- a/etc/make/docker.mk +++ b/etc/make/docker.mk @@ -1,4 +1,4 @@ -DOCKER_GO_VERSION ?= 1.22 +DOCKER_GO_VERSION ?= 1.23 DOCKERFILE := Dockerfile.$(DOCKER_GO_VERSION) DOCKER_IMAGE := uber/yarpc-go-$(DOCKER_GO_VERSION) diff --git a/go.mod b/go.mod index ee7d62749..c15fabf34 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module go.uber.org/yarpc -go 1.21 +go 1.23 -toolchain go1.22.2 +toolchain go1.23.0 require ( github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 diff --git a/internal/crossdock/go.mod b/internal/crossdock/go.mod index 39727bed6..c03e02503 100644 --- a/internal/crossdock/go.mod +++ b/internal/crossdock/go.mod @@ -1,8 +1,8 @@ module go.uber.org/yarpc/internal/crossdock -go 1.21 +go 1.23 -toolchain go1.22.2 +toolchain go1.23.0 require ( github.com/apache/thrift v0.13.0 diff --git a/internal/examples/go.mod b/internal/examples/go.mod index 6fea7e07c..c06732b70 100644 --- a/internal/examples/go.mod +++ b/internal/examples/go.mod @@ -1,8 +1,8 @@ module go.uber.org/yarpc/internal/examples -go 1.21 +go 1.23 -toolchain go1.22.2 +toolchain go1.23.0 require ( github.com/gogo/protobuf v1.3.2 diff --git a/yarpcconfig/spec_test.go b/yarpcconfig/spec_test.go index bd2bddf11..c25d19180 100644 --- a/yarpcconfig/spec_test.go +++ b/yarpcconfig/spec_test.go @@ -116,7 +116,7 @@ func TestCompileTransportSpec(t *testing.T) { BuildTransport: func(**struct{}, *Kit) (transport.Transport, error) { panic("kthxbye") }, BuildUnaryOutbound: func(debt, transport.Transport, *Kit) (transport.UnaryOutbound, error) { panic("kthxbye") }, }, - transportInput: reflect.PtrTo(reflect.TypeOf(&struct{}{})), + transportInput: reflect.PointerTo(reflect.TypeOf(&struct{}{})), supportsUnary: true, unaryOutboundInput: reflect.TypeOf(debt{}), }, @@ -461,7 +461,7 @@ func TestCompileTransportConfig(t *testing.T) { { desc: "valid: *struct{}", build: func(*struct{}, *Kit) (transport.Transport, error) { panic("kthxbye") }, - wantInputType: reflect.PtrTo(_typeOfEmptyStruct), + wantInputType: reflect.PointerTo(_typeOfEmptyStruct), }, } @@ -1180,10 +1180,10 @@ func TestIsDecodable(t *testing.T) { want bool }{ {give: _typeOfError, want: false}, - {give: reflect.PtrTo(_typeOfError), want: false}, + {give: reflect.PointerTo(_typeOfError), want: false}, {give: _typeOfEmptyStruct, want: true}, - {give: reflect.PtrTo(_typeOfEmptyStruct), want: true}, - {give: reflect.PtrTo(reflect.PtrTo(_typeOfEmptyStruct)), want: true}, + {give: reflect.PointerTo(_typeOfEmptyStruct), want: true}, + {give: reflect.PointerTo(reflect.PointerTo(_typeOfEmptyStruct)), want: true}, } for _, tt := range tests {