Skip to content

Commit f7ccb52

Browse files
authored
Merge pull request #82714 from porglezomp-misc/update-checkout-additional-scheme
update-checkout: Support specifying --config multiple times
2 parents 2b05dc6 + f3917b3 commit f7ccb52

File tree

5 files changed

+179
-15
lines changed

5 files changed

+179
-15
lines changed

utils/update_checkout/tests/scheme_mock.py

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,21 @@
6262
}
6363
}
6464

65+
MOCK_ADDITIONAL_SCHEME = {
66+
'branch-schemes': {
67+
'extra': {
68+
'aliases': ['extra'],
69+
'repos': {
70+
# Spell this differently just to make it distinguishable in
71+
# test output, even though we only have one branch.
72+
# TODO: Support multiple test branches in the repo instead.
73+
'repo1': 'refs/heads/main',
74+
'repo2': 'refs/heads/main',
75+
}
76+
}
77+
}
78+
}
79+
6580

6681
class CallQuietlyException(Exception):
6782
def __init__(self, command, returncode, output):
@@ -78,7 +93,7 @@ def __str__(self):
7893
def call_quietly(*args, **kwargs):
7994
kwargs['stderr'] = subprocess.STDOUT
8095
try:
81-
subprocess.check_output(*args, **kwargs)
96+
return subprocess.check_output(*args, **kwargs)
8297
except subprocess.CalledProcessError as e:
8398
raise CallQuietlyException(command=e.cmd, returncode=e.returncode,
8499
output=e.stdout) from e
@@ -97,6 +112,10 @@ def get_config_path(base_dir):
97112
return os.path.join(base_dir, 'test-config.json')
98113

99114

115+
def get_additional_config_path(base_dir):
116+
return os.path.join(base_dir, 'test-additional-config.json')
117+
118+
100119
def setup_mock_remote(base_dir, base_config):
101120
create_dir(base_dir)
102121

@@ -142,6 +161,9 @@ def setup_mock_remote(base_dir, base_config):
142161
with open(get_config_path(base_dir), 'w') as f:
143162
json.dump(base_config, f)
144163

164+
with open(get_additional_config_path(base_dir), 'w') as f:
165+
json.dump(MOCK_ADDITIONAL_SCHEME, f)
166+
145167
return (LOCAL_PATH, REMOTE_PATH)
146168

147169

@@ -164,6 +186,7 @@ def __init__(self, *args, **kwargs):
164186
raise RuntimeError('Misconfigured test suite! Environment '
165187
'variable %s must be set!' % BASEDIR_ENV_VAR)
166188
self.config_path = get_config_path(self.workspace)
189+
self.additional_config_path = get_additional_config_path(self.workspace)
167190
self.update_checkout_path = UPDATE_CHECKOUT_PATH
168191
if not os.access(self.update_checkout_path, os.X_OK):
169192
raise RuntimeError('Error! Could not find executable '
@@ -182,7 +205,7 @@ def tearDown(self):
182205

183206
def call(self, *args, **kwargs):
184207
kwargs['cwd'] = self.source_root
185-
call_quietly(*args, **kwargs)
208+
return call_quietly(*args, **kwargs)
186209

187210
def get_all_repos(self):
188211
return list(self.config["repos"].keys())

utils/update_checkout/tests/test_clone.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,17 @@ def test_simple_clone(self):
3030
repo_path = os.path.join(self.source_root, repo)
3131
self.assertTrue(os.path.isdir(repo_path))
3232

33+
def test_clone_with_additional_scheme(self):
34+
output = self.call([self.update_checkout_path,
35+
'--config', self.config_path,
36+
'--config', self.additional_config_path,
37+
'--source-root', self.source_root,
38+
'--clone',
39+
'--scheme', 'extra'])
40+
41+
# Test that we're actually checking out the 'extra' scheme based on the output
42+
self.assertIn(b"git checkout refs/heads/main", output)
43+
3344

3445
class SchemeWithMissingRepoTestCase(scheme_mock.SchemeMockTestCase):
3546
def __init__(self, *args, **kwargs):

utils/update_checkout/tests/test_dump.py

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
#
1111
# ===----------------------------------------------------------------------===#
1212

13+
import json
14+
1315
from . import scheme_mock
1416

1517

@@ -25,9 +27,16 @@ def test_dump_hashes_json(self):
2527
'--source-root', self.source_root,
2628
'--clone'])
2729

28-
# Then dump the hashes. Just make sure we don't crash. We should test
29-
# the output as well. But we should never crash.
30-
self.call([self.update_checkout_path,
31-
'--config', self.config_path,
32-
'--source-root', self.source_root,
33-
'--dump-hashes'])
30+
# Then dump the hashes.
31+
output = self.call([self.update_checkout_path,
32+
'--config', self.config_path,
33+
'--source-root', self.source_root,
34+
'--dump-hashes'])
35+
# The output should be valid JSON
36+
result = json.loads(output)
37+
38+
# And it should have some basic properties we expect from this JSON
39+
self.assertIn("https-clone-pattern", result)
40+
self.assertEqual(result["repos"], scheme_mock.MOCK_CONFIG["repos"])
41+
self.assertEqual(result["ssh-clone-pattern"], "DO_NOT_USE")
42+
self.assertSetEqual(set(result["branch-schemes"].keys()), {"repro"})
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# ===--- test_merge_config.py ---------------------------------------------===#
2+
#
3+
# This source file is part of the Swift.org open source project
4+
#
5+
# Copyright (c) 2025 Apple Inc. and the Swift project authors
6+
# Licensed under Apache License v2.0 with Runtime Library Exception
7+
#
8+
# See https:#swift.org/LICENSE.txt for license information
9+
# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
#
11+
# ===----------------------------------------------------------------------===#
12+
13+
import unittest
14+
15+
from update_checkout.update_checkout import merge_config, merge_no_duplicates
16+
17+
18+
class MergeTestCase(unittest.TestCase):
19+
def test_no_duplicates(self):
20+
self.assertEqual(merge_no_duplicates({}, {}), {})
21+
self.assertEqual(merge_no_duplicates({"a": 1}, {"b": 2}), {"a": 1, "b": 2})
22+
with self.assertRaises(ValueError):
23+
merge_no_duplicates({"a": 1, "b": 2}, {"b": 3})
24+
25+
def test_merge_config(self):
26+
default_config = {
27+
"ssh-clone-pattern": "git@1",
28+
"https-clone-pattern": "https://1",
29+
"repos": {
30+
"swift": {"remote": {"id": "swiftlang/swift"}},
31+
"llvm-project": {"remote": {"id": "swiftlang/llvm-project"}},
32+
},
33+
"default-branch-scheme": "main",
34+
"branch-schemes": {
35+
"main": {
36+
"aliases": ["swift/main", "main", "stable/20240723"],
37+
},
38+
},
39+
}
40+
41+
self.assertEqual(merge_config(default_config, {
42+
"note": "this is machine generated or something",
43+
"ssh-clone-pattern": "git@2",
44+
"repos": {
45+
"llvm-project": {"remote": {"id": "blah/llvm-project"}},
46+
"swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}},
47+
},
48+
"default-branch-scheme": "bonus",
49+
"branch-schemes": {
50+
"bonus": {
51+
"aliases": ["bonus", "also-bonus"],
52+
},
53+
},
54+
}), {
55+
"ssh-clone-pattern": "git@2",
56+
"https-clone-pattern": "https://1",
57+
"repos": {
58+
"swift": {"remote": {"id": "swiftlang/swift"}},
59+
"llvm-project": {"remote": {"id": "blah/llvm-project"}},
60+
"swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}},
61+
},
62+
"default-branch-scheme": "bonus",
63+
"branch-schemes": {
64+
"main": {
65+
"aliases": ["swift/main", "main", "stable/20240723"],
66+
},
67+
"bonus": {
68+
"aliases": ["bonus", "also-bonus"],
69+
},
70+
},
71+
"note": "this is machine generated or something",
72+
})
73+
74+
with self.assertRaises(ValueError):
75+
merge_config(default_config, default_config)

utils/update_checkout/update_checkout/update_checkout.py

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,40 @@ def print_repo_hashes(args, config):
536536
print("{:<35}: {:<35}".format(repo_name, repo_hash))
537537

538538

539+
def merge_no_duplicates(a: dict, b: dict) -> dict:
540+
result = {**a}
541+
for key, value in b.items():
542+
if key in a:
543+
raise ValueError(f"Duplicate scheme {key}")
544+
545+
result[key] = value
546+
return result
547+
548+
549+
def merge_config(config: dict, new_config: dict) -> dict:
550+
"""
551+
Merge two configs, with a 'last-wins' strategy.
552+
553+
The branch-schemes are rejected if they define duplicate schemes.
554+
"""
555+
556+
result = {**config}
557+
for key, value in new_config.items():
558+
if key == "branch-schemes":
559+
# We reject duplicates here since this is the most conservative
560+
# behavior, so it can be relaxed in the future.
561+
# TODO: Another semantics might be nicer, define that as it is needed.
562+
result[key] = merge_no_duplicates(config.get(key, {}), value)
563+
elif key == "repos":
564+
# The "repos" object is last-wins on a key-by-key basis
565+
result[key] = {**config.get(key, {}), **value}
566+
else:
567+
# Anything else is just last-wins
568+
result[key] = value
569+
570+
return result
571+
572+
539573
def validate_config(config):
540574
# Make sure that our branch-names are unique.
541575
scheme_names = config['branch-schemes'].keys()
@@ -669,9 +703,12 @@ def main():
669703
)
670704
parser.add_argument(
671705
"--config",
672-
default=os.path.join(SCRIPT_DIR, os.pardir,
673-
"update-checkout-config.json"),
674-
help="Configuration file to use")
706+
help="""The configuration file to use. Can be specified multiple times,
707+
each config will be merged together with a 'last-wins' strategy.
708+
Overwriting branch-schemes is not allowed.""",
709+
action="append",
710+
default=[],
711+
dest="configs")
675712
parser.add_argument(
676713
"--github-comment",
677714
help="""Check out related pull requests referenced in the given
@@ -734,8 +771,15 @@ def main():
734771
all_repos = args.all_repositories
735772
use_submodules = args.use_submodules
736773

737-
with open(args.config) as f:
738-
config = json.load(f)
774+
# Set the default config path if none are specified
775+
if not args.configs:
776+
default_path = os.path.join(SCRIPT_DIR, os.pardir,
777+
"update-checkout-config.json")
778+
args.configs.append(default_path)
779+
config = {}
780+
for config_path in args.configs:
781+
with open(config_path) as f:
782+
config = merge_config(config, json.load(f))
739783
validate_config(config)
740784

741785
cross_repos_pr = {}
@@ -783,8 +827,10 @@ def main():
783827
prefix="[swift] ")
784828

785829
# Re-read the config after checkout.
786-
with open(args.config) as f:
787-
config = json.load(f)
830+
config = {}
831+
for config_path in args.configs:
832+
with open(config_path) as f:
833+
config = merge_config(config, json.load(f))
788834
validate_config(config)
789835
scheme_map = get_scheme_map(config, scheme_name)
790836

0 commit comments

Comments
 (0)