diff --git a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/OpenSourceApiClient.cs b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/OpenSourceApiClient.cs index 3a2a11d9021..314f83a2198 100644 --- a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/OpenSourceApiClient.cs +++ b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/OpenSourceApiClient.cs @@ -10,7 +10,6 @@ namespace GitHubTeamUserStore public class OpenSourceApiClient { private static readonly TokenRequestContext OpenSourceApiTokenRequestContext = new([ProductAndTeamConstants.OpenSourceApiScope]); - private static readonly string[] UnsupportedPaginationHeaders = ["Link", "x-ms-continuation", "x-next-page"]; private static readonly HttpClient SharedHttpClient = new HttpClient(); private readonly TokenCredential _credential; @@ -69,6 +68,8 @@ public async Task> GetPublicOrgMembers(string orgName) throw new InvalidOperationException($"Open Source API returned a child team without a valid name and slug for '{teamSlug}'."); } + // The current cache contract still keys teams by display name. Slug remains the traversal key for OSP. + // TODO: migrate the cache contract to use slug entirely. See https://github.com/Azure/azure-sdk-tools/issues/15474. results.Add((childTeam.Name, childTeam.Slug)); } @@ -136,31 +137,21 @@ private async Task GetFromOpenSourceApi(string relativePath) using HttpResponseMessage response = await SharedHttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); - EnsureNoUnsupportedPagination(response, relativePath); await using Stream responseStream = await response.Content.ReadAsStreamAsync(); return await JsonSerializer.DeserializeAsync(responseStream) ?? throw new InvalidOperationException($"Open Source API returned an empty payload for '{relativePath}'."); } - private static void EnsureNoUnsupportedPagination(HttpResponseMessage response, string relativePath) - { - foreach (string header in UnsupportedPaginationHeaders) - { - if (response.Headers.Contains(header) || response.Content.Headers.Contains(header)) - { - throw new InvalidOperationException($"Open Source API returned unsupported pagination header '{header}' for '{relativePath}'."); - } - } - - if (response.Content.Headers.Contains("Content-Range")) - { - throw new InvalidOperationException($"Open Source API returned unsupported pagination header 'Content-Range' for '{relativePath}'."); - } - } - private static TokenCredential CreateOpenSourceApiCredential() { + // This mirrors azsdk-cli AzureService credential ordering for Azure DevOps and local runs: + // Azure DevOps uses AzurePipelines -> WorkloadIdentity -> AzureCli when service connection values exist, + // otherwise WorkloadIdentity -> AzureCli. Local runs use AzureCli -> AzurePowerShell -> + // AzureDeveloperCli -> VisualStudio -> ManagedIdentity. + // It intentionally omits the GitHub Actions and InteractiveBrowserCredential branches because + // this tool only runs in Azure DevOps or local dev and should fail fast when non-interactive + // credentials are unavailable. return IsRunningInAzureDevOps() ? CreateAzureDevOpsCredential() : CreateLocalCredential(); @@ -185,8 +176,8 @@ private static TokenCredential CreateAzureDevOpsCredential() return new ChainedTokenCredential( new AzurePipelinesCredential(azureSubscriptionClient, azureSubscriptionTenant, azureServiceConnection, accessToken), - new WorkloadIdentityCredential(), - new AzureCliCredential()); + new WorkloadIdentityCredential(new WorkloadIdentityCredentialOptions { TenantId = azureSubscriptionTenant }), + new AzureCliCredential(new AzureCliCredentialOptions { TenantId = azureSubscriptionTenant })); } private static TokenCredential CreateLocalCredential() diff --git a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/RepositoryLabelGenerator.cs b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/RepositoryLabelGenerator.cs index fba9a04884d..01112b6b6c5 100644 --- a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/RepositoryLabelGenerator.cs +++ b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/RepositoryLabelGenerator.cs @@ -13,7 +13,7 @@ public static async Task GenerateAndWriteRepositoryLabels(OpenSourceApiCli // Repository name is the key, with the list of that repository's labels as the as the value Dictionary> repoLabelDict = new Dictionary>(); - bool dataMatches = false; + bool succeeded = false; try { @@ -46,61 +46,15 @@ public static async Task GenerateAndWriteRepositoryLabels(OpenSourceApiCli string jsonString = JsonSerializer.Serialize(repoLabelDict); await File.WriteAllTextAsync(repoLabelOutputPath, jsonString); - dataMatches = await VerifyWrittenRepositoryLabelData(repoLabelOutputPath, repoLabelDict); - if (dataMatches) - { - Console.WriteLine($"repository/label data written successfully to {repoLabelOutputPath}."); - } - else - { - Console.WriteLine("There were issues with the written repository/label data. See above for specifics."); - } + Console.WriteLine($"repository/label data written successfully to {repoLabelOutputPath}."); + succeeded = true; } finally { - Console.WriteLine($"=== Finished repository/label cache build: {(dataMatches ? "success" : "failure")} ({repoLabelOutputPath}) ==="); - } - - return dataMatches; - } - - private static async Task VerifyWrittenRepositoryLabelData(string repoLabelOutputPath, - Dictionary> repoLabelDict) - { - string rawJson = await File.ReadAllTextAsync(repoLabelOutputPath); - var writtenRepoLabelDict = JsonSerializer.Deserialize>>(rawJson) - ?? throw new InvalidOperationException($"Unable to deserialize repository/label data from {repoLabelOutputPath}."); - if (repoLabelDict.Keys.Count != writtenRepoLabelDict.Keys.Count) - { - Console.WriteLine($"Error! Created repo/label dictionary has {repoLabelDict.Keys.Count} repositories and written dictionary has {writtenRepoLabelDict.Keys.Count} repositories."); - Console.WriteLine(string.Format("created list repositories {0}", string.Join(", ", repoLabelDict.Keys))); - Console.WriteLine(string.Format("written list repositories {0}", string.Join(", ", writtenRepoLabelDict.Keys))); - return false; + Console.WriteLine($"=== Finished repository/label cache build: {(succeeded ? "success" : "failure")} ({repoLabelOutputPath}) ==="); } - foreach (string repository in repoLabelDict.Keys) - { - if (!writtenRepoLabelDict.ContainsKey(repository)) - { - Console.WriteLine("Error! Created repo/label dictionary has different repositories than the written dictionary."); - Console.WriteLine(string.Format("created dictionary repositories {0}", string.Join(", ", repoLabelDict.Keys))); - Console.WriteLine(string.Format("written dictionary repositories {0}", string.Join(", ", writtenRepoLabelDict.Keys))); - return false; - } - } - - bool hasError = false; - foreach (string repository in repoLabelDict.Keys) - { - if (!repoLabelDict[repository].SetEquals(writtenRepoLabelDict[repository])) - { - hasError = true; - Console.WriteLine($"The created dictionary entry for {repository} has different labels than the written dictionary."); - Console.WriteLine(string.Format("created dictionary labels {0}", string.Join(", ", repoLabelDict[repository]))); - Console.WriteLine(string.Format("written dictionary labels {0}", string.Join(", ", writtenRepoLabelDict[repository]))); - } - } - return !hasError; + return succeeded; } } } diff --git a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/TeamUserGenerator.cs b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/TeamUserGenerator.cs index c3b37f86884..18d0092cab9 100644 --- a/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/TeamUserGenerator.cs +++ b/tools/github-team-user-store/GitHubTeamUserStore/GitHubTeamUserStore/TeamUserGenerator.cs @@ -8,13 +8,16 @@ public class TeamUserGenerator /// /// Generate the team/user lists for each and every team under azure-sdk-write using the OSP team - /// children and team members endpoints. The resulting team/user data is serialized into json and written - /// to a local file using the current cache contract where the team display name is the key. + /// children and team members endpoints. Starting from the azure-sdk-write root avoids the older GitHub + /// team walk entirely and keeps the OSP traversal bounded to one known team tree instead of enumerating + /// unrelated teams and then looking up their membership, parents, or children. The resulting team/user + /// data is serialized into json and written to a local file using the current cache contract where the + /// team display name is the key. /// /// Authenticated OpenSourceApiClient /// The file where the team/user cache will be written. /// The file where the user/org visibility cache will be written. - /// True if everything is written and verified successfully, false otherwise. + /// True if everything is written successfully, false otherwise. public static async Task GenerateAndWriteTeamUserAndOrgData(OpenSourceApiClient openSourceApiClient, string teamUserOutputPath, string userOrgVisibilityOutputPath) @@ -35,7 +38,7 @@ public static async Task GenerateAndWriteTeamUserAndOrgData(OpenSourceApiC Queue<(string Name, string Slug)> teamsToProcess = new Queue<(string Name, string Slug)>(); teamsToProcess.Enqueue((ProductAndTeamConstants.AzureSdkWriteTeamName, ProductAndTeamConstants.AzureSdkWriteTeamSlug)); - bool teamUserDataMatches = false; + bool teamUserDataWritten = false; try { while (teamsToProcess.Count > 0) @@ -69,22 +72,15 @@ public static async Task GenerateAndWriteTeamUserAndOrgData(OpenSourceApiC var list = teamUserDict.ToList(); string jsonString = JsonSerializer.Serialize(list); await File.WriteAllTextAsync(teamUserOutputPath, jsonString); - teamUserDataMatches = await VerifyWrittenTeamUsers(teamUserOutputPath, teamUserDict); - if (teamUserDataMatches) - { - Console.WriteLine($"team/user data written successfully to {teamUserOutputPath}."); - } - else - { - Console.WriteLine("There were issues with the written team/user data. See above for specifics."); - } + Console.WriteLine($"team/user data written successfully to {teamUserOutputPath}."); + teamUserDataWritten = true; } finally { - Console.WriteLine($"=== Finished team/user cache build: {(teamUserDataMatches ? "success" : "failure")} ({teamUserOutputPath}) ==="); + Console.WriteLine($"=== Finished team/user cache build: {(teamUserDataWritten ? "success" : "failure")} ({teamUserOutputPath}) ==="); } - if (!teamUserDataMatches) + if (!teamUserDataWritten) { return false; } @@ -134,60 +130,6 @@ private static List CreateDistinctMembers(IReadOnlyList teamMemb return distinctMembers; } - /// - /// This method is called after the team/user data is written locally. It verifies that the - /// team/user data from disk is the same as the in-memory data that was used to create the file. - /// - /// The file containing the written team/user data. - /// True, if the data on disk matches the in-memory data that was used to create the file, otherwise, false. - private static async Task VerifyWrittenTeamUsers(string teamUserOutputPath, - Dictionary> teamUserDict) - { - string rawJson = await File.ReadAllTextAsync(teamUserOutputPath); - var list = JsonSerializer.Deserialize>>>(rawJson) - ?? throw new InvalidOperationException($"Unable to deserialize team/user data from {teamUserOutputPath}."); - var writtenDictionary = list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value); - - // Verify the dictionary from disk contains everything from the in-memory dictionary. - if (teamUserDict.Keys.Count != writtenDictionary.Keys.Count) - { - Console.WriteLine($"Error! Created dictionary has {teamUserDict.Keys.Count} teams and written dictionary has {writtenDictionary.Keys.Count} teams."); - Console.WriteLine(string.Format("created list teams {0}", string.Join(", ", teamUserDict.Keys))); - Console.WriteLine(string.Format("written list teams {0}", string.Join(", ", writtenDictionary.Keys))); - return false; - } - - bool hasError = false; - foreach (string key in teamUserDict.Keys) - { - if (!writtenDictionary.ContainsKey(key)) - { - Console.WriteLine($"Error! Written dictionary does not contain the team {key}."); - Console.WriteLine(string.Format("created list teams {0}", string.Join(", ", teamUserDict.Keys))); - Console.WriteLine(string.Format("written list teams {0}", string.Join(", ", writtenDictionary.Keys))); - return false; - } - - var users = teamUserDict[key].OrderBy(user => user, StringComparer.OrdinalIgnoreCase).ToList(); - var writtenUsers = writtenDictionary[key].OrderBy(user => user, StringComparer.OrdinalIgnoreCase).ToList(); - if (users.Count != writtenUsers.Count) - { - hasError = true; - Console.WriteLine($"Error! Created dictionary for team {key} has {users.Count} users and written dictionary has {writtenUsers.Count} users."); - Console.WriteLine(string.Format("created list users {0}", string.Join(", ", users))); - Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUsers))); - } - else if (!users.SequenceEqual(writtenUsers, StringComparer.OrdinalIgnoreCase)) - { - hasError = true; - Console.WriteLine($"Error! Created dictionary for team {key} has different users than the written dictionary."); - Console.WriteLine(string.Format("created list users {0}", string.Join(", ", users))); - Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUsers))); - } - } - return !hasError; - } - /// /// This function requires that the team/user data is generated first. It'll use the users from /// the azure-sdk-write group, which is the all inclusive list of users with write permissions, @@ -202,7 +144,7 @@ private static async Task GenerateAndWriteUserOrgData(OpenSourceApiClient Console.WriteLine($"=== Starting user/org visibility cache build: {userOrgVisibilityOutputPath} ==="); Dictionary userOrgDict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - bool dataMatches = false; + bool userOrgDataWritten = false; try { @@ -222,59 +164,15 @@ private static async Task GenerateAndWriteUserOrgData(OpenSourceApiClient string jsonString = JsonSerializer.Serialize(userOrgDict); await File.WriteAllTextAsync(userOrgVisibilityOutputPath, jsonString); - dataMatches = await VerifyWrittenUserOrgData(userOrgVisibilityOutputPath, userOrgDict); - if (dataMatches) - { - Console.WriteLine($"user/org visibility data written successfully to {userOrgVisibilityOutputPath}."); - } - else - { - Console.WriteLine("There were issues with the written user/org visibility data. See above for specifics."); - } + Console.WriteLine($"user/org visibility data written successfully to {userOrgVisibilityOutputPath}."); + userOrgDataWritten = true; } finally { - Console.WriteLine($"=== Finished user/org visibility cache build: {(dataMatches ? "success" : "failure")} ({userOrgVisibilityOutputPath}) ==="); + Console.WriteLine($"=== Finished user/org visibility cache build: {(userOrgDataWritten ? "success" : "failure")} ({userOrgVisibilityOutputPath}) ==="); } - return dataMatches; - } - - private static async Task VerifyWrittenUserOrgData(string userOrgVisibilityOutputPath, - Dictionary userOrgDict) - { - string rawJson = await File.ReadAllTextAsync(userOrgVisibilityOutputPath); - var writtenUserOrgDict = JsonSerializer.Deserialize>(rawJson) - ?? throw new InvalidOperationException($"Unable to deserialize user/org visibility data from {userOrgVisibilityOutputPath}."); - if (userOrgDict.Keys.Count != writtenUserOrgDict.Keys.Count) - { - Console.WriteLine($"Error! Created user/org dictionary has {userOrgDict.Keys.Count} users and written dictionary has {writtenUserOrgDict.Keys.Count} users."); - Console.WriteLine(string.Format("created list users {0}", string.Join(", ", userOrgDict.Keys))); - Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUserOrgDict.Keys))); - return false; - } - - foreach (string user in userOrgDict.Keys) - { - if (!writtenUserOrgDict.ContainsKey(user)) - { - Console.WriteLine("Error! Created user/org dictionary has different users than the written dictionary."); - Console.WriteLine(string.Format("created list users {0}", string.Join(", ", userOrgDict.Keys))); - Console.WriteLine(string.Format("written list users {0}", string.Join(", ", writtenUserOrgDict.Keys))); - return false; - } - } - - bool hasError = false; - foreach (string user in userOrgDict.Keys) - { - if (userOrgDict[user] != writtenUserOrgDict[user]) - { - hasError = true; - Console.WriteLine($"The created dictionary entry for {user} is '{userOrgDict[user]}' and in written it is '{writtenUserOrgDict[user]}'"); - } - } - return !hasError; + return userOrgDataWritten; } } }