From 6009d7ddba4b498917d38279057da72041bd868f Mon Sep 17 00:00:00 2001 From: Justin Won Date: Fri, 10 Jul 2026 14:24:54 -0700 Subject: [PATCH] feat(internal/cachekey): add cache path/key construction package Adds internal/cachekey with the cache-path builders (ToShortRemote, GetGraphByTreeHash, GetTreehashCachePath, GetComparedTargetsCachePath) that today live in core/common/utils.go. This is purely additive: no existing callers are migrated and core/common is untouched, so this package is currently unused. A follow-up PR will switch controller and orchestrator over to it and remove the duplicated code from core/common. Co-Authored-By: Claude Sonnet 5 --- internal/cachekey/BUILD.bazel | 19 ++++ internal/cachekey/cachekey.go | 112 +++++++++++++++++++++ internal/cachekey/cachekey_test.go | 156 +++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+) create mode 100644 internal/cachekey/BUILD.bazel create mode 100644 internal/cachekey/cachekey.go create mode 100644 internal/cachekey/cachekey_test.go diff --git a/internal/cachekey/BUILD.bazel b/internal/cachekey/BUILD.bazel new file mode 100644 index 00000000..32afe650 --- /dev/null +++ b/internal/cachekey/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "cachekey", + srcs = ["cachekey.go"], + importpath = "github.com/uber/tango/internal/cachekey", + visibility = ["//visibility:public"], + deps = ["//tangopb"], +) + +go_test( + name = "cachekey_test", + srcs = ["cachekey_test.go"], + embed = [":cachekey"], + deps = [ + "//tangopb", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/internal/cachekey/cachekey.go b/internal/cachekey/cachekey.go new file mode 100644 index 00000000..0579b26e --- /dev/null +++ b/internal/cachekey/cachekey.go @@ -0,0 +1,112 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cachekey builds the storage keys used to cache target graphs, treehashes, and compared-target-graph results. +package cachekey + +import ( + "crypto/md5" + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/uber/tango/tangopb" +) + +// ToShortRemote returns the short remote name given a git ssh remote string. +// For example, "git@github:uber/tango" will return "uber/tango". +func ToShortRemote(remote string) string { + strs := strings.Split(remote, ":") + return strs[len(strs)-1] +} + +// GetGraphByTreeHash returns the cache path for the target graph by treehash. +// strategy is part of the key because different computation strategies (e.g. +// SHELL vs NATIVE) can produce different graphs from the same tree state. +// requestOptions is folded into the key when any of its fields affect computation +// (today: extra_exclude_files_regex). Empty/nil ⇒ legacy path unchanged. +func GetGraphByTreeHash(remote, treehash string, strategy tangopb.ComputationStrategy, requestOptions *tangopb.RequestOptions) string { + path := filepath.Join(ToShortRemote(remote), "graphs", treehash, strategy.String()) + if hash := hashRequestOptions(requestOptions); hash != "" { + path += "_requests-options-" + hash + } + return path +} + +// GetTreehashCachePath returns the cache path for the treehash mapping. +// The git treehash is purely a function of git state (base SHA + applied +// requests), so neither requestOptions nor the computation strategy is part +// of this key. +func GetTreehashCachePath(buildDescription *tangopb.BuildDescription) string { + path := filepath.Join(ToShortRemote(buildDescription.Remote), "treehashes", fmt.Sprintf("base-sha-%s", buildDescription.BaseSha)) + if len(buildDescription.Requests) > 0 { + path += "_request-urls-" + getReqURLsHash(buildDescription.Requests) + } + return path +} + +// GetComparedTargetsCachePath returns the cache path for a compared target graph result. +// treehash1 and treehash2 are the resolved treehashes of the first and second revisions. +// remote is the shared git remote for both revisions. +// requestOptions is folded into the key when any of its fields affect computation. +// Empty/nil ⇒ legacy path unchanged. +func GetComparedTargetsCachePath(remote, treehash1, treehash2 string, requestOptions *tangopb.RequestOptions) string { + path := filepath.Join(ToShortRemote(remote), "compared-targets", treehash1+"_"+treehash2) + if hash := hashRequestOptions(requestOptions); hash != "" { + path += "_requests-options-" + hash + } + return path +} + +// getReqURLsHash returns a fixed-length MD5 hash of the sorted request URLs. +// Each URL's bytes are fed into the digest individually (no separator), matching +// the Java MessageDigest.update(str.getBytes()) per-string behavior. +func getReqURLsHash(requests []*tangopb.Request) string { + if len(requests) == 0 { + return "" + } + urls := make([]string, 0, len(requests)) + for _, req := range requests { + urls = append(urls, req.GetUrl()) + } + sort.Strings(urls) + h := md5.New() + for _, url := range urls { + h.Write([]byte(url)) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// hashRequestOptions returns "" when no field of RequestOptions contributes to +// the cache (preserves legacy paths), otherwise the md5 hex digest of those +// fields. As new fields are added to RequestOptions, fold them into the digest +// here. +func hashRequestOptions(opts *tangopb.RequestOptions) string { + if opts == nil { + return "" + } + excludes := opts.GetExtraExcludeFilesRegex() + if len(excludes) == 0 { + return "" + } + sorted := make([]string, len(excludes)) + copy(sorted, excludes) + sort.Strings(sorted) + h := md5.New() + for _, r := range sorted { + h.Write([]byte(r)) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/internal/cachekey/cachekey_test.go b/internal/cachekey/cachekey_test.go new file mode 100644 index 00000000..4dd9cc3c --- /dev/null +++ b/internal/cachekey/cachekey_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cachekey + +import ( + "crypto/md5" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + pb "github.com/uber/tango/tangopb" +) + +func TestToShortRemote(t *testing.T) { + t.Parallel() + tests := []struct { + name string + remote string + want string + }{ + { + name: "ssh remote with host", + remote: "git@github:uber/tango", + want: "uber/tango", + }, + { + name: "already short", + remote: "uber/tango", + want: "uber/tango", + }, + { + name: "with nested path", + remote: "git@github:org/project/sub", + want: "org/project/sub", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, ToShortRemote(tt.remote)) + }) + } +} + +func TestGetGraphByTreeHash(t *testing.T) { + t.Parallel() + remote := "git@github:uber/tango" + treehash := "abcd1234" + strategy := pb.COMPUTATION_STRATEGY_NATIVE + + // Nil/empty options ⇒ no request-options suffix. + got := GetGraphByTreeHash(remote, treehash, strategy, nil) + assert.Equal(t, filepath.Join("uber/tango", "graphs", treehash, strategy.String()), got) + assert.Equal(t, got, GetGraphByTreeHash(remote, treehash, strategy, &pb.RequestOptions{})) + + // Different strategies ⇒ different keys. + assert.NotEqual(t, got, GetGraphByTreeHash(remote, treehash, pb.COMPUTATION_STRATEGY_SHELL, nil)) + + // Non-empty options ⇒ suffix appended; different lists ⇒ different keys. + withFoo := GetGraphByTreeHash(remote, treehash, strategy, &pb.RequestOptions{ExtraExcludeFilesRegex: []string{"foo.*"}}) + assert.NotEqual(t, got, withFoo) + assert.NotEqual(t, withFoo, GetGraphByTreeHash(remote, treehash, strategy, &pb.RequestOptions{ExtraExcludeFilesRegex: []string{"bar.*"}})) + // Order-independence: sort before hashing. + assert.Equal(t, + GetGraphByTreeHash(remote, treehash, strategy, &pb.RequestOptions{ExtraExcludeFilesRegex: []string{"a", "b"}}), + GetGraphByTreeHash(remote, treehash, strategy, &pb.RequestOptions{ExtraExcludeFilesRegex: []string{"b", "a"}}), + ) +} + +func TestGetTreehashCachePath(t *testing.T) { + t.Parallel() + reqs := []*pb.Request{ + {Url: "github://org/repo/pull/1"}, + {Url: "custom://foo/bar"}, + } + desc := &pb.BuildDescription{ + Remote: "git@github:uber/tango", + BaseSha: "deadbeef", + Requests: reqs, + } + got := GetTreehashCachePath(desc) + // URLs are sorted then fed individually into the digest (no separator) + h := md5.New() + h.Write([]byte("custom://foo/bar")) + h.Write([]byte("github://org/repo/pull/1")) + want := filepath.Join("uber/tango", "treehashes", "base-sha-deadbeef") + "_request-urls-" + fmt.Sprintf("%x", h.Sum(nil)) + assert.Equal(t, want, got) +} + +func TestGetReqURLsHash(t *testing.T) { + t.Parallel() + md5hex := func(strs ...string) string { + h := md5.New() + for _, s := range strs { + h.Write([]byte(s)) + } + return fmt.Sprintf("%x", h.Sum(nil)) + } + tests := []struct { + name string + in []*pb.Request + want string + }{ + { + name: "empty", + in: []*pb.Request{}, + want: "", + }, + { + name: "single", + in: []*pb.Request{{Url: "github://org/repo/pull/42"}}, + want: md5hex("github://org/repo/pull/42"), + }, + { + name: "multiple", + in: []*pb.Request{{Url: "a"}, {Url: "b"}}, + want: md5hex("a", "b"), + }, + { + name: "multiple sorted", + in: []*pb.Request{{Url: "b"}, {Url: "a"}}, + want: md5hex("a", "b"), + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, getReqURLsHash(tt.in)) + }) + } +} + +func TestGetComparedTargetsCachePath(t *testing.T) { + t.Parallel() + got := GetComparedTargetsCachePath("git@github:uber/tango", "abc", "def", nil) + assert.Equal(t, filepath.Join("uber/tango", "compared-targets", "abc_def"), got) + + // Nil/empty options ⇒ legacy path. + assert.Equal(t, got, GetComparedTargetsCachePath("git@github:uber/tango", "abc", "def", &pb.RequestOptions{})) + + // Different exclude lists ⇒ different keys. + assert.NotEqual(t, got, GetComparedTargetsCachePath("git@github:uber/tango", "abc", "def", &pb.RequestOptions{ExtraExcludeFilesRegex: []string{"foo.*"}})) +}