Skip to content

CODEOWNERS cache builder cleanup #15476

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
danieljurek wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

CODEOWNERS cache builder cleanup #15476

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 11 additions & 20 deletions tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/OpenSourceApiClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ namespace GitHubTeamUserStore
public class OpenSourceApiClient
{
private static readonly TokenRequestContext OpenSourceApiTokenRequestContext = new([ProductAndTeamConstants.OpenSourceApiScope]);
private static readonly string[] UnsupportedPaginationHeaders = ["Link", "x-ms-continuation", "x-next-page"];
private static readonly HttpClient SharedHttpClient = new HttpClient();
private readonly TokenCredential _credential;

Expand Down Expand Up @@ -69,6 +68,8 @@ public async Task<HashSet<string>> GetPublicOrgMembers(string orgName)
throw new InvalidOperationException($"Open Source API returned a child team without a valid name and slug for '{teamSlug}'.");
}

// The current cache contract still keys teams by display name. Slug remains the traversal key for OSP.
// TODO: migrate the cache contract to use slug entirely. See https://github.com/Azure/azure-sdk-tools/issues/15474.
results.Add((childTeam.Name, childTeam.Slug));
}

Expand Down Expand Up @@ -136,31 +137,21 @@ private async Task<T> GetFromOpenSourceApi<T>(string relativePath)

using HttpResponseMessage response = await SharedHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
EnsureNoUnsupportedPagination(response, relativePath);

await using Stream responseStream = await response.Content.ReadAsStreamAsync();
return await JsonSerializer.DeserializeAsync<T>(responseStream)
?? throw new InvalidOperationException($"Open Source API returned an empty payload for '{relativePath}'.");
}

private static void EnsureNoUnsupportedPagination(HttpResponseMessage response, string relativePath)
{
foreach (string header in UnsupportedPaginationHeaders)
{
if (response.Headers.Contains(header) || response.Content.Headers.Contains(header))
{
throw new InvalidOperationException($"Open Source API returned unsupported pagination header '{header}' for '{relativePath}'.");
}
}

if (response.Content.Headers.Contains("Content-Range"))
{
throw new InvalidOperationException($"Open Source API returned unsupported pagination header 'Content-Range' for '{relativePath}'.");
}
}

private static TokenCredential CreateOpenSourceApiCredential()
{
// This mirrors azsdk-cli AzureService credential ordering for Azure DevOps and local runs:
// Azure DevOps uses AzurePipelines -> WorkloadIdentity -> AzureCli when service connection values exist,
// otherwise WorkloadIdentity -> AzureCli. Local runs use AzureCli -> AzurePowerShell ->
// AzureDeveloperCli -> VisualStudio -> ManagedIdentity.
// It intentionally omits the GitHub Actions and InteractiveBrowserCredential branches because
// this tool only runs in Azure DevOps or local dev and should fail fast when non-interactive
// credentials are unavailable.
return IsRunningInAzureDevOps()
? CreateAzureDevOpsCredential()
: CreateLocalCredential();
Expand All @@ -185,8 +176,8 @@ private static TokenCredential CreateAzureDevOpsCredential()

return new ChainedTokenCredential(
new AzurePipelinesCredential(azureSubscriptionClient, azureSubscriptionTenant, azureServiceConnection, accessToken),
new WorkloadIdentityCredential(),
new AzureCliCredential());
new WorkloadIdentityCredential(new WorkloadIdentityCredentialOptions { TenantId = azureSubscriptionTenant }),
new AzureCliCredential(new AzureCliCredentialOptions { TenantId = azureSubscriptionTenant }));
}

private static TokenCredential CreateLocalCredential()
Expand Down
56 changes: 5 additions & 51 deletions ...ithub-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/RepositoryLabelGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public static async Task<bool> GenerateAndWriteRepositoryLabels(OpenSourceApiCli

// Repository name is the key, with the list of that repository's labels as the as the value
Dictionary<string, HashSet<string>> repoLabelDict = new Dictionary<string, HashSet<string>>();
bool dataMatches = false;
bool succeeded = false;

try
{
Expand Down Expand Up @@ -46,61 +46,15 @@ public static async Task<bool> GenerateAndWriteRepositoryLabels(OpenSourceApiCli

string jsonString = JsonSerializer.Serialize(repoLabelDict);
await File.WriteAllTextAsync(repoLabelOutputPath, jsonString);
dataMatches = await VerifyWrittenRepositoryLabelData(repoLabelOutputPath, repoLabelDict);
if (dataMatches)
{
Console.WriteLine($"repository/label data written successfully to {repoLabelOutputPath}.");
}
else
{
Console.WriteLine("There were issues with the written repository/label data. See above for specifics.");
}
Console.WriteLine($"repository/label data written successfully to {repoLabelOutputPath}.");
succeeded = true;
}
finally
{
Console.WriteLine($"=== Finished repository/label cache build: {(dataMatches ? "success" : "failure")} ({repoLabelOutputPath}) ===");
}

return dataMatches;
}

private static async Task<bool> VerifyWrittenRepositoryLabelData(string repoLabelOutputPath,
Dictionary<string, HashSet<string>> repoLabelDict)
{
string rawJson = await File.ReadAllTextAsync(repoLabelOutputPath);
var writtenRepoLabelDict = JsonSerializer.Deserialize<Dictionary<string, HashSet<string>>>(rawJson)
?? throw new InvalidOperationException($"Unable to deserialize repository/label data from {repoLabelOutputPath}.");
if (repoLabelDict.Keys.Count != writtenRepoLabelDict.Keys.Count)
{
Console.WriteLine($"Error! Created repo/label dictionary has {repoLabelDict.Keys.Count} repositories and written dictionary has {writtenRepoLabelDict.Keys.Count} repositories.");
Console.WriteLine(string.Format("created list repositories {0}", string.Join(", ", repoLabelDict.Keys)));
Console.WriteLine(string.Format("written list repositories {0}", string.Join(", ", writtenRepoLabelDict.Keys)));
return false;
Console.WriteLine($"=== Finished repository/label cache build: {(succeeded ? "success" : "failure")} ({repoLabelOutputPath}) ===");
}

foreach (string repository in repoLabelDict.Keys)
{
if (!writtenRepoLabelDict.ContainsKey(repository))
{
Console.WriteLine("Error! Created repo/label dictionary has different repositories than the written dictionary.");
Console.WriteLine(string.Format("created dictionary repositories {0}", string.Join(", ", repoLabelDict.Keys)));
Console.WriteLine(string.Format("written dictionary repositories {0}", string.Join(", ", writtenRepoLabelDict.Keys)));
return false;
}
}

bool hasError = false;
foreach (string repository in repoLabelDict.Keys)
{
if (!repoLabelDict[repository].SetEquals(writtenRepoLabelDict[repository]))
{
hasError = true;
Console.WriteLine($"The created dictionary entry for {repository} has different labels than the written dictionary.");
Console.WriteLine(string.Format("created dictionary labels {0}", string.Join(", ", repoLabelDict[repository])));
Console.WriteLine(string.Format("written dictionary labels {0}", string.Join(", ", writtenRepoLabelDict[repository])));
}
}
return !hasError;
return succeeded;
}
}
}
134 changes: 16 additions & 118 deletions tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/TeamUserGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,16 @@ public class TeamUserGenerator

/// <summary>
/// Generate the team/user lists for each and every team under azure-sdk-write using the OSP team
/// children and team members endpoints. The resulting team/user data is serialized into json and written
/// to a local file using the current cache contract where the team display name is the key.
/// children and team members endpoints. Starting from the azure-sdk-write root avoids the older GitHub
/// team walk entirely and keeps the OSP traversal bounded to one known team tree instead of enumerating
/// unrelated teams and then looking up their membership, parents, or children. The resulting team/user
/// data is serialized into json and written to a local file using the current cache contract where the
/// team display name is the key.
/// </summary>
/// <param name="openSourceApiClient">Authenticated OpenSourceApiClient</param>
/// <param name="teamUserOutputPath">The file where the team/user cache will be written.</param>
/// <param name="userOrgVisibilityOutputPath">The file where the user/org visibility cache will be written.</param>
/// <returns>True if everything is written and verified successfully, false otherwise.</returns>
/// <returns>True if everything is written successfully, false otherwise.</returns>
public static async Task<bool> GenerateAndWriteTeamUserAndOrgData(OpenSourceApiClient openSourceApiClient,
string teamUserOutputPath,
string userOrgVisibilityOutputPath)
Expand All @@ -35,7 +38,7 @@ public static async Task<bool> GenerateAndWriteTeamUserAndOrgData(OpenSourceApiC
Queue<(string Name, string Slug)> teamsToProcess = new Queue<(string Name, string Slug)>();
teamsToProcess.Enqueue((ProductAndTeamConstants.AzureSdkWriteTeamName, ProductAndTeamConstants.AzureSdkWriteTeamSlug));

bool teamUserDataMatches = false;
bool teamUserDataWritten = false;
try
{
while (teamsToProcess.Count > 0)
Expand Down Expand Up @@ -69,22 +72,15 @@ public static async Task<bool> GenerateAndWriteTeamUserAndOrgData(OpenSourceApiC
var list = teamUserDict.ToList();
string jsonString = JsonSerializer.Serialize(list);
await File.WriteAllTextAsync(teamUserOutputPath, jsonString);
teamUserDataMatches = await VerifyWrittenTeamUsers(teamUserOutputPath, teamUserDict);
if (teamUserDataMatches)
{
Console.WriteLine($"team/user data written successfully to {teamUserOutputPath}.");
}
else
{
Console.WriteLine("There were issues with the written team/user data. See above for specifics.");
}
Console.WriteLine($"team/user data written successfully to {teamUserOutputPath}.");
teamUserDataWritten = true;
}
finally
{
Console.WriteLine($"=== Finished team/user cache build: {(teamUserDataMatches ? "success" : "failure")} ({teamUserOutputPath}) ===");
Console.WriteLine($"=== Finished team/user cache build: {(teamUserDataWritten ? "success" : "failure")} ({teamUserOutputPath}) ===");
}

if (!teamUserDataMatches)
if (!teamUserDataWritten)
{
return false;
}
Expand Down Expand Up @@ -134,60 +130,6 @@ private static List<string> CreateDistinctMembers(IReadOnlyList<string> teamMemb
return distinctMembers;
}

/// <summary>
/// This method is called after the team/user data is written locally. It verifies that the
/// team/user data from disk is the same as the in-memory data that was used to create the file.
/// </summary>
/// <param name="teamUserOutputPath">The file containing the written team/user data.</param>
/// <returns>True, if the data on disk matches the in-memory data that was used to create the file, otherwise, false.</returns>
private static async Task<bool> VerifyWrittenTeamUsers(string teamUserOutputPath,
Dictionary<string, List<string>> teamUserDict)
{
string rawJson = await File.ReadAllTextAsync(teamUserOutputPath);
var list = JsonSerializer.Deserialize<List<KeyValuePair<string, List<string>>>>(rawJson)
?? throw new InvalidOperationException($"Unable to deserialize team/user data from {teamUserOutputPath}.");
var writtenDictionary = list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value);

// Verify the dictionary from disk contains everything from the in-memory dictionary.
if (teamUserDict.Keys.Count != writtenDictionary.Keys.Count)
{
Console.WriteLine($"Error! Created dictionary has {teamUserDict.Keys.Count} teams and written dictionary has {writtenDictionary.Keys.Count} teams.");
Console.WriteLine(string.Format("created list teams {0}", string.Join(", ", teamUserDict.Keys)));
Console.WriteLine(string.Format("written list teams {0}", string.Join(", ", writtenDictionary.Keys)));
return false;
}

bool hasError = false;
foreach (string key in teamUserDict.Keys)
{
if (!writtenDictionary.ContainsKey(key))
{
Console.WriteLine($"Error! Written dictionary does not contain the team {key}.");
Console.WriteLine(string.Format("created list teams {0}", string.Join(", ", teamUserDict.Keys)));
Console.WriteLine(string.Format("written list teams {0}", string.Join(", ", writtenDictionary.Keys)));
return false;
}

var users = teamUserDict[key].OrderBy(user => user, StringComparer.OrdinalIgnoreCase).ToList();
var writtenUsers = writtenDictionary[key].OrderBy(user => user, StringComparer.OrdinalIgnoreCase).ToList();
if (users.Count != writtenUsers.Count)
{
hasError = true;
Console.WriteLine($"Error! Created dictionary for team {key} has {users.Count} users and written dictionary has {writtenUsers.Count} users.");
Console.WriteLine(string.Format("created list users {0}", string.Join(", ", users)));
Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUsers)));
}
else if (!users.SequenceEqual(writtenUsers, StringComparer.OrdinalIgnoreCase))
{
hasError = true;
Console.WriteLine($"Error! Created dictionary for team {key} has different users than the written dictionary.");
Console.WriteLine(string.Format("created list users {0}", string.Join(", ", users)));
Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUsers)));
}
}
return !hasError;
}

/// <summary>
/// This function requires that the team/user data is generated first. It'll use the users from
/// the azure-sdk-write group, which is the all inclusive list of users with write permissions,
Expand All @@ -202,7 +144,7 @@ private static async Task<bool> GenerateAndWriteUserOrgData(OpenSourceApiClient
Console.WriteLine($"=== Starting user/org visibility cache build: {userOrgVisibilityOutputPath} ===");

Dictionary<string, bool> userOrgDict = new Dictionary<string, bool>(StringComparer.InvariantCultureIgnoreCase);
bool dataMatches = false;
bool userOrgDataWritten = false;

try
{
Expand All @@ -222,59 +164,15 @@ private static async Task<bool> GenerateAndWriteUserOrgData(OpenSourceApiClient

string jsonString = JsonSerializer.Serialize(userOrgDict);
await File.WriteAllTextAsync(userOrgVisibilityOutputPath, jsonString);
dataMatches = await VerifyWrittenUserOrgData(userOrgVisibilityOutputPath, userOrgDict);
if (dataMatches)
{
Console.WriteLine($"user/org visibility data written successfully to {userOrgVisibilityOutputPath}.");
}
else
{
Console.WriteLine("There were issues with the written user/org visibility data. See above for specifics.");
}
Console.WriteLine($"user/org visibility data written successfully to {userOrgVisibilityOutputPath}.");
userOrgDataWritten = true;
}
finally
{
Console.WriteLine($"=== Finished user/org visibility cache build: {(dataMatches ? "success" : "failure")} ({userOrgVisibilityOutputPath}) ===");
Console.WriteLine($"=== Finished user/org visibility cache build: {(userOrgDataWritten ? "success" : "failure")} ({userOrgVisibilityOutputPath}) ===");
}

return dataMatches;
}

private static async Task<bool> VerifyWrittenUserOrgData(string userOrgVisibilityOutputPath,
Dictionary<string, bool> userOrgDict)
{
string rawJson = await File.ReadAllTextAsync(userOrgVisibilityOutputPath);
var writtenUserOrgDict = JsonSerializer.Deserialize<Dictionary<string, bool>>(rawJson)
?? throw new InvalidOperationException($"Unable to deserialize user/org visibility data from {userOrgVisibilityOutputPath}.");
if (userOrgDict.Keys.Count != writtenUserOrgDict.Keys.Count)
{
Console.WriteLine($"Error! Created user/org dictionary has {userOrgDict.Keys.Count} users and written dictionary has {writtenUserOrgDict.Keys.Count} users.");
Console.WriteLine(string.Format("created list users {0}", string.Join(", ", userOrgDict.Keys)));
Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUserOrgDict.Keys)));
return false;
}

foreach (string user in userOrgDict.Keys)
{
if (!writtenUserOrgDict.ContainsKey(user))
{
Console.WriteLine("Error! Created user/org dictionary has different users than the written dictionary.");
Console.WriteLine(string.Format("created list users {0}", string.Join(", ", userOrgDict.Keys)));
Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUserOrgDict.Keys)));
return false;
}
}

bool hasError = false;
foreach (string user in userOrgDict.Keys)
{
if (userOrgDict[user] != writtenUserOrgDict[user])
{
hasError = true;
Console.WriteLine($"The created dictionary entry for {user} is '{userOrgDict[user]}' and in written it is '{writtenUserOrgDict[user]}'");
}
}
return !hasError;
return userOrgDataWritten;
}
}
}