From 358e5a8b7e80320eee9d6b00eaf3fddc9bd8473d Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 15:44:35 +0300 Subject: [PATCH 1/6] Add test fix history matching --- .../ignite/ci/tcbot/issue/IssueDetector.java | 2 + .../rest/monitoring/MonitoringService.java | 155 +++ .../rest/testfixes/TestFixesRestService.java | 67 + .../tracked/GetTrackedBranchTestResults.java | 9 +- .../src/main/webapp/current.html | 6 +- .../src/main/webapp/guard.html | 2 - .../src/main/webapp/js/common-1.7.js | 1 + .../src/main/webapp/js/prs-1.3.js | 5 + .../src/main/webapp/js/testfails-2.3.js | 32 + .../src/main/webapp/monitoring.html | 48 + .../src/main/webapp/testfixes.html | 374 ++++++ .../chain/TrackedBranchProcessorTest.java | 2 +- .../guice/GuiceTcBotApplicationContext.java | 2 + .../tcbot/engine/TcBotEngineModule.java | 2 + .../tcbot/common/conf/IJiraServerConfig.java | 14 + .../tcbot/engine/conf/JiraServerConfig.java | 38 + .../tcbot/engine/pr/PrChainsProcessor.java | 11 +- .../tcbot/engine/testfixes/TestFixMatch.java | 98 ++ .../tcbot/engine/testfixes/TestFixRefUi.java | 80 ++ .../engine/testfixes/TestFixesService.java | 1186 +++++++++++++++++ .../IDetailedStatusForTrackedBranch.java | 2 + .../tracked/TrackedBranchChainsProcessor.java | 7 + .../ignite/tcbot/engine/ui/ShortSuiteUi.java | 4 + .../tcbot/engine/ui/ShortTestFailureUi.java | 6 + .../testfixes/TestFixesServiceTest.java | 175 +++ .../githubignited/GitHubConnIgnitedImpl.java | 70 + .../githubignited/IGitHubConnIgnited.java | 8 + .../apache/ignite/ci/github/PullRequest.java | 38 +- .../githubservice/GitHubConnectionImpl.java | 20 + .../githubservice/IGitHubConnection.java | 7 + .../ci/jira/ignited/TicketCompacted.java | 47 +- .../ignite/jiraignited/JiraTicketSync.java | 81 +- .../apache/ignite/jiraservice/IFields.java | 14 +- .../apache/ignite/jiraservice/v2/Fields.java | 27 + .../ignite/jiraservice/v3/FieldsV3.java | 26 + .../tcignited/history/ISuiteRunHistory.java | 6 + .../tcignited/history/SuiteHistory.java | 5 + 37 files changed, 2654 insertions(+), 23 deletions(-) create mode 100644 ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java create mode 100644 ignite-tc-helper-web/src/main/webapp/testfixes.html create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java create mode 100644 tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java index eb775bf12..a129641e7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java @@ -632,6 +632,7 @@ protected String checkFailuresEx(String brachName) { false, null, null, + null, DisplayMode.None, null, -1, false, false); @@ -645,6 +646,7 @@ protected String checkFailuresEx(String brachName) { false, null, null, + null, DisplayMode.OnlyFailures, null, -1, false, false); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java index 6a6937c56..51841678f 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java @@ -16,6 +16,10 @@ */ package org.apache.ignite.ci.web.rest.monitoring; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import java.io.BufferedReader; import java.io.File; @@ -33,16 +37,21 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.security.RolesAllowed; +import javax.cache.Cache; import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BadRequestException; import javax.ws.rs.ClientErrorException; +import javax.ws.rs.ForbiddenException; import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.NotFoundException; @@ -57,6 +66,8 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.ci.user.ITcBotUserCreds; +import org.apache.ignite.ci.user.TcHelperUser; import org.apache.ignite.ci.web.CtxListener; import org.apache.ignite.ci.web.auth.AuthenticationFilter; import org.apache.ignite.ci.web.model.SimpleResult; @@ -69,6 +80,7 @@ import org.apache.ignite.tcbot.engine.conf.NotificationsConfig; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; import org.apache.ignite.tcbot.engine.process.BotProcessStatus; +import org.apache.ignite.tcbot.engine.user.IUserStorage; import org.apache.ignite.tcbot.notify.IEmailSender; import org.apache.ignite.tcbot.notify.ISendEmailConfig; import org.apache.ignite.tcbot.notify.ISlackSender; @@ -110,6 +122,16 @@ public class MonitoringService { /** Max summary length. */ private static final int SUMMARY_LIMIT = 240; + /** Default number of cache entries to preview. */ + private static final int DFLT_CACHE_PEEK_LIMIT = 20; + + /** Hard cache preview cap. */ + private static final int MAX_CACHE_PEEK_LIMIT = 100; + + /** JSON mapper for raw cache entry values. */ + private static final ObjectMapper CACHE_PEEK_MAPPER = new ObjectMapper() + .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + /** Log timestamp format. */ private static final DateTimeFormatter LOG_TS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); @@ -117,6 +139,10 @@ public class MonitoringService { @Context private ServletContext ctx; + /** Request. */ + @Context + private HttpServletRequest req; + @GET @Path("tasks") public List getTaskMonitoring() { @@ -602,6 +628,135 @@ public List getCacheStat() { return res; } + @GET + @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) + @Produces(MediaType.TEXT_PLAIN) + @Path("cachePeek") + public String cachePeek(@QueryParam("name") String name, @QueryParam("limit") Integer limit) { + if (Strings.isNullOrEmpty(name)) + throw new BadRequestException("Cache name is required"); + + ensureCanPeekCache(name); + + Ignite ignite = instance(Ignite.class); + IgniteCache cache = ignite.cache(name); + + if (cache == null) + throw new NotFoundException("Cache not found: " + name); + + int actualLimit = normalizeCachePeekLimit(limit); + StringBuilder res = new StringBuilder(); + int shown = 0; + boolean truncated = false; + + res.append("Cache: ").append(name).append('\n'); + res.append("Size: ").append(cache.size()).append('\n'); + res.append("Limit: ").append(actualLimit).append("\n\n"); + + for (Cache.Entry entry : cache) { + if (shown >= actualLimit) { + truncated = true; + + break; + } + + shown++; + + res.append("Entry #").append(shown).append('\n'); + res.append("Key class: ").append(className(entry.getKey())).append('\n'); + res.append(toJsonNode(entry.getKey()).toPrettyString()).append('\n'); + res.append("Value class: ").append(className(entry.getValue())).append('\n'); + res.append(toJsonNode(entry.getValue()).toPrettyString()).append("\n\n"); + } + + res.append("Entries shown: ").append(shown).append('\n'); + res.append("Truncated: ").append(truncated).append('\n'); + + return res.toString(); + } + + /** + * @param limit Requested limit. + */ + private static int normalizeCachePeekLimit(Integer limit) { + if (limit == null || limit <= 0) + return DFLT_CACHE_PEEK_LIMIT; + + return Math.min(limit, MAX_CACHE_PEEK_LIMIT); + } + + /** + * @param obj Object. + */ + private static String className(Object obj) { + return obj == null ? "null" : obj.getClass().getName(); + } + + /** + * @param name Cache name. + */ + private void ensureCanPeekCache(String name) { + if (!isUsersCache(name)) + return; + + ITcBotUserCreds creds = ITcBotUserCreds.get(req); + TcHelperUser user = creds == null ? null : instance(IUserStorage.class).getUser(creds.getPrincipalId()); + + if (!isUserAdmin(user, instance(ITcBotConfig.class))) + throw new ForbiddenException("Only user admin can inspect Users caches"); + } + + /** + * @param name Cache name. + */ + private static boolean isUsersCache(String name) { + return !Strings.isNullOrEmpty(name) && name.toLowerCase(Locale.ROOT).contains("users"); + } + + /** + * @param user User. + * @param cfg Config. + */ + private static boolean isUserAdmin(TcHelperUser user, ITcBotConfig cfg) { + return user != null && (user.isUserAdmin() || isConfigUserAdmin(user.username, cfg)); + } + + /** + * @param username Username. + * @param cfg Config. + */ + private static boolean isConfigUserAdmin(String username, ITcBotConfig cfg) { + if (Strings.isNullOrEmpty(username)) + return false; + + Collection cfgUserAdmins = cfg.userAdmins(); + + if (cfgUserAdmins == null) + return false; + + String normalized = username.toLowerCase(Locale.ROOT); + + return cfgUserAdmins.stream() + .filter(Objects::nonNull) + .map(s -> s.toLowerCase(Locale.ROOT)) + .anyMatch(normalized::equals); + } + + /** + * @param obj Object. + */ + private static JsonNode toJsonNode(Object obj) { + try { + return CACHE_PEEK_MAPPER.valueToTree(obj); + } + catch (RuntimeException e) { + return CACHE_PEEK_MAPPER.createObjectNode() + .put("serializationError", e.getClass().getSimpleName() + ": " + e.getMessage()) + .put("class", className(obj)) + .put("toString", String.valueOf(obj)); + } + } + @POST @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) @Path("resetCache") diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java new file mode 100644 index 000000000..1f705f02b --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.web.rest.testfixes; + +import java.util.List; +import javax.servlet.ServletContext; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; + +/** + * Test fix history REST service. + */ +@Path("testFixes") +@Produces(MediaType.APPLICATION_JSON) +public class TestFixesRestService { + /** Servlet Context. */ + @Context private ServletContext ctx; + + /** + * @param limit Max rows. + */ + @GET + @Path("recent") + public List recent(@QueryParam("limit") Integer limit) { + int actualLimit = limit == null ? 0 : limit; + + return CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class).recent(actualLimit); + } + + /** + * Schedules an immediate refresh and returns currently cached rows. + * + * @param limit Max rows. + * @param processId Optional user-visible process id. + */ + @GET + @Path("refresh") + public List refresh(@QueryParam("limit") Integer limit, @QueryParam("processId") Long processId) { + TestFixesService svc = CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class); + + svc.requestMatchNow(processId); + + return recent(limit); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java index 571abaa02..2d1feafc8 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java @@ -124,6 +124,7 @@ public DsSummaryUi getTestFailsResultsNoSync( @Nullable @QueryParam("trustedTests") Boolean trustedTests, @Nullable @QueryParam("tagSelected") String tagSelected, @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected, + @Nullable @QueryParam("suiteId") String suiteId, @Nullable @QueryParam("displayMode") String displayMode, @Nullable @QueryParam("sortOption") String sortOption, @Nullable @QueryParam("count") Integer mergeCnt, @@ -131,7 +132,7 @@ public DsSummaryUi getTestFailsResultsNoSync( @Nullable @QueryParam("muted") Boolean showMuted, @Nullable @QueryParam("ignored") Boolean showIgnored) { return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected, - SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); + suiteId, SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); } @GET @@ -143,6 +144,7 @@ public DsSummaryUi getTestFailsNoCache( @Nullable @QueryParam("trustedTests") Boolean trustedTests, @Nullable @QueryParam("tagSelected") String tagSelected, @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected, + @Nullable @QueryParam("suiteId") String suiteId, @Nullable @QueryParam("displayMode") String displayMode, @Nullable @QueryParam("sortOption") String sortOption, @Nullable @QueryParam("count") Integer mergeCnt, @@ -150,7 +152,8 @@ public DsSummaryUi getTestFailsNoCache( @Nullable @QueryParam("muted") Boolean showMuted, @Nullable @QueryParam("ignored") Boolean showIgnored) { return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected, - SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); + suiteId, SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, + showIgnored); } @NotNull private DsSummaryUi latestBuildResults( @@ -159,6 +162,7 @@ public DsSummaryUi getTestFailsNoCache( @Nullable Boolean trustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nonnull SyncMode mode, @Nullable String displayMode, @Nullable String sortOption, @@ -185,6 +189,7 @@ public DsSummaryUi getTestFailsNoCache( Boolean.TRUE.equals(trustedTests), tagSelected, tagForHistSelected, + suiteId, DisplayMode.parseStringValue(displayMode), SortOption.parseStringValue(sortOption), maxDurationSec, diff --git a/ignite-tc-helper-web/src/main/webapp/current.html b/ignite-tc-helper-web/src/main/webapp/current.html index 07ca05899..11384f510 100644 --- a/ignite-tc-helper-web/src/main/webapp/current.html +++ b/ignite-tc-helper-web/src/main/webapp/current.html @@ -204,7 +204,11 @@ let tagForHistSelected = gVue.$data.tagForHistSelected; if (tagForHistSelected != null && tagForHistSelected !== "") - curReqParms += "&tagForHistSelected=" + tagForHistSelected; + curReqParms += "&tagForHistSelected=" + encodeURIComponent(tagForHistSelected); + + let suiteId = findGetParameter("suiteId"); + if (suiteId != null && suiteId !== "") + curReqParms += "&suiteId=" + encodeURIComponent(suiteId); let runTime = gVue.$data.showTestLongerThan; if (runTime != null && runTime > 0) diff --git a/ignite-tc-helper-web/src/main/webapp/guard.html b/ignite-tc-helper-web/src/main/webapp/guard.html index cdb9dbdb1..430d84a24 100644 --- a/ignite-tc-helper-web/src/main/webapp/guard.html +++ b/ignite-tc-helper-web/src/main/webapp/guard.html @@ -4,7 +4,6 @@ - @@ -19,7 +18,6 @@ .guard-page { margin: 22px 18px 34px 18px; - max-width: 1320px; padding: 0; } diff --git a/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js b/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js index 9de453ca6..d05365991 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js +++ b/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js @@ -732,6 +732,7 @@ function showMenu(menuData) { res += "Compare builds"; res += "Issues history"; res += "Visas history"; + res += "Test fixes history"; res += "Muted tests"; res += "Muted issues"; res += "Board"; diff --git a/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js b/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js index 09bddd1e6..68bb98788 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js +++ b/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js @@ -411,6 +411,11 @@ function claimGithubAuthorHtml(srvId, row) { let explicitlyConfigured = explicitlyConfiguredGithubLoginsByServer.get(srvId); + if (isDefinedAndFilled(explicitlyConfigured) && explicitlyConfigured.has(String(row.prAuthor).toLowerCase())) + return ""; + + let explicitlyConfigured = explicitlyConfiguredGithubLoginsByServer.get(srvId); + if (isDefinedAndFilled(explicitlyConfigured) && explicitlyConfigured.has(String(row.prAuthor).toLowerCase())) return ""; diff --git a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js index 5c4a6ad15..55c3a99a2 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js +++ b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js @@ -1801,6 +1801,8 @@ function showSuiteData(suite, settings, prNum) { res += "title='Open AI prompt with TeamCity context for this suite'>[AI Prompt]"; } + res += testFixRefsHtml(suite.fixRefs); + if(isDefinedAndFilled(suite.tags)) { for (let i = 0; i < suite.tags.length; i++) { const tag = suite.tags[i]; @@ -2068,6 +2070,8 @@ function showTestFailData(testFail, isFailureShown, settings) { res += "title='Open AI prompt with TeamCity context for this test'>[AI Prompt]"; } + res += testFixRefsHtml(testFail.fixRefs); + var histContent = ""; //see class TestHistory @@ -2153,6 +2157,34 @@ function showTestFailData(testFail, isFailureShown, settings) { return res; } +function testFixRefsHtml(refs) { + if (!isDefinedAndFilled(refs) || refs.length === 0) + return ""; + + var res = ""; + + for (var i = 0; i < refs.length; i++) { + var ref = refs[i]; + var text = ref.sourceType === "github" ? "GH" : "JIRA"; + var title = "Matched fix"; + + if (isDefinedAndFilled(ref.title)) + title += ": " + ref.title; + + if (isDefinedAndFilled(ref.status)) + title += " [" + ref.status + "]"; + + if (isDefinedAndFilled(ref.closedDate)) + title += " closed " + ref.closedDate; + + res += " " + escapeHtml(text) + ""; + } + + return res; +} + function drawLatestRuns(latestRuns) { if(isDefinedAndFilled(findGetParameter("reportMode"))) return ""; diff --git a/ignite-tc-helper-web/src/main/webapp/monitoring.html b/ignite-tc-helper-web/src/main/webapp/monitoring.html index 68c5bdb1a..6695c6047 100644 --- a/ignite-tc-helper-web/src/main/webapp/monitoring.html +++ b/ignite-tc-helper-web/src/main/webapp/monitoring.html @@ -17,6 +17,7 @@ + + + + + + + + +
+ +
+ Show latest + + matched fixes + +
+ + + + + + + + + + + +
Fix / commitResolved testsAuthorStatusDates
+ +
+ + + + diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java index 492654ffc..4321c2db7 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java @@ -121,7 +121,7 @@ public void testTrackedBranchChainsProcessor() { false, 1, mock, SyncMode.RELOAD_QUEUED, - false, null, null, DisplayMode.OnlyFailures, null, + false, null, null, null, DisplayMode.OnlyFailures, null, -1, false, false); Gson gson = new GsonBuilder().setPrettyPrinting().create(); diff --git a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java index 49167a01e..87232a0dd 100644 --- a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java +++ b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java @@ -38,6 +38,7 @@ import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.NotificationsConfig; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.notify.ISlackSender; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.apache.ignite.tcservice.http.TeamcityRecorder; @@ -91,6 +92,7 @@ private void startBackgroundServicesWhenReady(Future igniteFuture) { getInstance(BuildObserver.class); getInstance(UserAdminRefreshService.class).start(); + getInstance(TestFixesService.class).start(); ready.set(true); } catch (Exception e) { diff --git a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java index b3d1abeea..7b495dcfd 100644 --- a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java +++ b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java @@ -29,6 +29,7 @@ import org.apache.ignite.tcbot.engine.issue.IssuesStorage; import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch; import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor; import org.apache.ignite.tcbot.engine.user.IUserStorage; @@ -55,6 +56,7 @@ public class TcBotEngineModule extends AbstractModule { bind(MutedIssuesDao.class).in(Scopes.SINGLETON); bind(NewTestsStorage.class).in(Scopes.SINGLETON); bind(BotProcessMonitor.class).in(Scopes.SINGLETON); + bind(TestFixesService.class).in(Scopes.SINGLETON); install(new TcBotCommonModule()); } diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java index bf1fd2982..916f8c999 100644 --- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java @@ -60,6 +60,20 @@ public interface IJiraServerConfig { */ @Nullable String decodedHttpAuthToken(); + /** + * JIRA label used to explicitly mark test-fix tickets. + */ + default String testFixesLabel() { + return "MakeTeamcityGreenAgain"; + } + + /** + * Number of recent days to scan for test-fix tickets. Default matches base branch run-history horizon. + */ + default int testFixesLookbackDays() { + return org.apache.ignite.tcbot.common.TcBotConst.HISTORY_MAX_DAYS; + } + /** * @return {@code True} if JIRA authorization token is available. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java index 72d94e322..20f3ddf3b 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java @@ -37,6 +37,12 @@ public class JiraServerConfig implements IJiraServerConfig { /** JIRA URL to build links to tickets. */ public static final String JIRA_URL = "jira.url"; + /** Explicit JIRA label for test-fix tickets. */ + public static final String JIRA_TEST_FIXES_LABEL = "jira.test_fixes_label"; + + /** Recent days window for test-fix ticket matching. */ + public static final String JIRA_TEST_FIXES_LOOKBACK_DAYS = "jira.test_fixes_lookback_days"; + /** Prefix for JIRA ticket names. */ @Deprecated public static final String JIRA_TICKET_TEMPLATE = "jira.ticket_template"; @@ -79,6 +85,12 @@ public class JiraServerConfig implements IJiraServerConfig { **/ private JiraApiVersion apiVersion = JiraApiVersion.defaultApiVersion(); + /** Explicit JIRA label for test-fix tickets. */ + private String testFixesLabel; + + /** Recent days window for test-fix ticket matching. */ + @Nullable private Integer testFixesLookbackDays; + public JiraServerConfig() { } @@ -139,6 +151,32 @@ public JiraServerConfig code(String code) { return Strings.emptyToNull(branchNumPrefix); } + /** {@inheritDoc} */ + @Override public String testFixesLabel() { + if (!Strings.isNullOrEmpty(testFixesLabel)) + return testFixesLabel; + + return props != null + ? props.getProperty(JIRA_TEST_FIXES_LABEL, IJiraServerConfig.super.testFixesLabel()) + : IJiraServerConfig.super.testFixesLabel(); + } + + /** {@inheritDoc} */ + @Override public int testFixesLookbackDays() { + if (testFixesLookbackDays != null) + return testFixesLookbackDays; + + if (props == null) + return IJiraServerConfig.super.testFixesLookbackDays(); + + String val = props.getProperty(JIRA_TEST_FIXES_LOOKBACK_DAYS); + + if (Strings.isNullOrEmpty(val)) + return IJiraServerConfig.super.testFixesLookbackDays(); + + return Integer.parseInt(val); + } + /** {@inheritDoc} */ @Nullable @Override diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java index 54c4de5bd..ca27a6564 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java @@ -61,6 +61,7 @@ import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; @@ -135,6 +136,9 @@ private static class Action { /** User-visible process monitor. */ @Inject private BotProcessMonitor processMonitor; + /** Test fix matcher. */ + @Inject private TestFixesService testFixesService; + /** * @param creds Credentials. * @param srvCodeOrAlias Server code or alias. @@ -217,6 +221,7 @@ else if (Action.CHAIN.equals(act)) //fail rate reference is always default (master) chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false, null, null, -1, null, false, false); // don't need for PR + chainStatus.suites.forEach(testFixesService::decorate); chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor, newTestsStorage); initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited); } @@ -674,9 +679,13 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, // test failure based blockers and/or blocker found by suite results if (!failures.isEmpty() || !Strings.isNullOrEmpty(suiteComment)) { - return new ShortSuiteUi() + ShortSuiteUi suite = new ShortSuiteUi() .testShortFailures(failures) .initFrom(ctx, tcIgnited, compactor, statInBaseBranch); + + testFixesService.decorate(suite); + + return suite; } return null; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java new file mode 100644 index 000000000..791915376 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Persisted detected relation between a test/suite name and a fix source. + */ +@Persisted +public class TestFixMatch implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Matched test or suite name. */ + public String entityName; + + /** Matched suite name, when known. */ + @Nullable public String suiteName; + + /** Matched suite id, when known. */ + @Nullable public String suiteId; + + /** Matched test name, when known. */ + @Nullable public String testName; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current master TeamCity suite status URL. */ + @Nullable public String currentStatusUrl; + + /** Source type: jira or github. */ + public String sourceType; + + /** Source id. */ + public String sourceId; + + /** Source URL. */ + public String sourceUrl; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated timestamp. */ + @Nullable public Long updatedTs; + + /** Closed/resolved timestamp. */ + @Nullable public Long closedTs; + + /** Commit SHA, when known. */ + @Nullable public String commitSha; + + /** Commit URL, when known. */ + @Nullable public String commitUrl; + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java new file mode 100644 index 000000000..13dffc955 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; + +/** + * Link to a ticket or PR mentioning a test/suite fix. + */ +@SuppressWarnings("PublicField") +public class TestFixRefUi { + /** Matched test or suite name. */ + public String entityName; + + /** Suite id. */ + @Nullable public String suiteId; + + /** Suite display name. */ + @Nullable public String suiteName; + + /** Test display name. */ + @Nullable public String testName; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current tracked branch status URL. */ + @Nullable public String currentStatusUrl; + + /** Source type: jira or github. */ + public String sourceType; + + /** Short source text, for example IGNITE-12345 or PR #123. */ + public String text; + + /** Source URL. */ + public String url; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated date. */ + @Nullable public String updatedDate; + + /** Closed/resolved date. */ + @Nullable public String closedDate; + + /** Closing or best-known fix commit URL. */ + @Nullable public String commitUrl; + + /** Closing or best-known fix commit text. */ + @Nullable public String commitText; +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java new file mode 100644 index 000000000..eb78893dd --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java @@ -0,0 +1,1186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import com.google.common.base.Strings; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; +import javax.cache.Cache; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.ci.github.GitHubBranch; +import org.apache.ignite.ci.github.GitHubUser; +import org.apache.ignite.ci.github.PullRequest; +import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeCompacted; +import org.apache.ignite.ci.teamcity.ignited.buildtype.SnapshotDependencyCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnited; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; +import org.apache.ignite.jiraservice.JiraTicketStatusCode; +import org.apache.ignite.jiraservice.Ticket; +import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; +import org.apache.ignite.tcbot.engine.conf.ITrackedBranch; +import org.apache.ignite.tcbot.engine.conf.ITrackedChain; +import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.ui.ShortSuiteUi; +import org.apache.ignite.tcbot.engine.ui.ShortTestFailureUi; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcbot.persistence.CacheConfigs; +import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; +import org.apache.ignite.tcbot.persistence.scheduler.MaintenanceActionRegistry; +import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; +import org.apache.ignite.tcservice.model.conf.BuildType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch; + +/** + * Matches JIRA tickets and GitHub PRs mentioning tests or suites that are being fixed. + */ +public class TestFixesService { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(TestFixesService.class); + + /** Cache name. */ + private static final String CACHE_NAME = "testFixMatches"; + + /** Background scheduled task name. */ + private static final String TASK_NAME = TestFixesService.class.getSimpleName() + ".refresh"; + + /** Source update signal cache name. Also used by JIRA/GitHub sync modules. */ + private static final String SOURCE_UPDATES_CACHE_NAME = "testFixSourceUpdates"; + + /** Ignite provider. */ + @Inject private Provider igniteProvider; + + /** Scheduler. */ + @Inject private IScheduler scheduler; + + /** Admin maintenance actions. */ + @Inject private MaintenanceActionRegistry maintenanceActions; + + /** Config. */ + @Inject private ITcBotConfig cfg; + + /** JIRA provider. */ + @Inject private IJiraIgnitedProvider jiraProvider; + + /** GitHub provider. */ + @Inject private IGitHubConnIgnitedProvider ghProvider; + + /** TeamCity provider. */ + @Inject private ITeamcityIgnitedProvider tcProvider; + + /** User-visible process monitor. */ + @Inject private BotProcessMonitor processMonitor; + + /** String compactor. */ + @Inject private IStringCompactor compactor; + + /** Cache. */ + private volatile IgniteCache cache; + + /** Source update signals cache. */ + private volatile IgniteCache signalCache; + + /** Last source update signal timestamp. */ + private volatile long lastSignalTs; + + /** + * Starts rare background refresh. + */ + public void start() { + maintenanceActions.register(TASK_NAME, "Refresh JIRA/GitHub test-fix mapping", this::refresh); + + scheduler.invokeLater(this::refreshAndReschedule, 2, TimeUnit.MINUTES); + scheduler.invokeLater(this::watchSourceUpdates, 3, TimeUnit.MINUTES); + } + + /** + * Schedules earlier matching after data update signal. + */ + public void requestMatchSoon() { + scheduler.sheduleNamed(TASK_NAME + ".soon", this::safeRefresh, 3, TimeUnit.MINUTES); + } + + /** + * Schedules an immediate manual refresh. + * + * @param processId Optional user-visible process id. + * @return {@code true} if refresh was accepted. + */ + public boolean requestMatchNow(@Nullable Long processId) { + processMonitor.start(processId, "testFixesRefresh", "Test fix mapping refresh queued."); + + boolean accepted = scheduler.runNamedNow(TASK_NAME + ".manual", () -> safeRefresh(processId), processId); + + if (!accepted) + processMonitor.fail(processId, "Test fix mapping refresh is already queued or running."); + + return accepted; + } + + /** + * @param suite Suite UI. + */ + public void decorate(ShortSuiteUi suite) { + if (suite == null) + return; + + suite.fixRefs = findRefs(suite.name); + + for (ShortTestFailureUi failure : suite.testFailures()) + decorate(failure); + } + + /** + * @param failure Test failure UI. + */ + public void decorate(ShortTestFailureUi failure) { + if (failure == null) + return; + + List refs = new ArrayList<>(); + + refs.addAll(findRefs(failure.testName)); + refs.addAll(findRefs(failure.name)); + + failure.fixRefs = uniqueRefs(refs); + } + + /** + * @param limit Max rows. + */ + public List recent(int limit) { + ensureCache(); + + java.util.stream.Stream stream = StreamSupport.stream(cache.spliterator(), false) + .map(Cache.Entry::getValue) + .filter(TestFixesService::isPlausibleMatch) + .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()); + + if (limit > 0) + stream = stream.limit(limit); + + return stream + .map(this::toUi) + .collect(Collectors.toList()); + } + + /** + * Refreshes match cache. + */ + @AutoProfiling + public String refresh() { + return refresh(null); + } + + /** + * Refreshes match cache. + * + * @param processId Optional user-visible process id. + */ + @AutoProfiling + public String refresh(@Nullable Long processId) { + ensureCache(); + + RefreshStats stats = new RefreshStats(); + + publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); + Map> candidatesByServer = candidatesByServer(); + stats.collectCandidateStats(candidatesByServer); + publishRefreshStatus(processId, "collected test names from tracked suite history", stats); + + for (String srvCode : serverCodes()) { + List srvCandidates = candidatesByServer.get(srvCode); + + if (srvCandidates == null || srvCandidates.isEmpty()) + continue; + + publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); + List recentPrs = recentPullRequests(srvCode); + stats.githubLoaded += recentPrs.size(); + publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + + srvCode, stats); + + refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); + refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); + } + + return stats.finishText(); + } + + /** + * Refreshes and schedules next rare run. + */ + private void refreshAndReschedule() { + safeRefresh(); + scheduler.sheduleNamed(TASK_NAME, this::refreshAndReschedule, 6, TimeUnit.HOURS); + } + + /** + * Watches cheap update signals from JIRA/GitHub cache refreshes. + */ + private void watchSourceUpdates() { + try { + long signalTs = latestSignalTs(); + + if (signalTs > lastSignalTs) { + lastSignalTs = signalTs; + requestMatchSoon(); + } + } + catch (RuntimeException e) { + logger.debug("Failed to watch test fix update signals", e); + } + + scheduler.sheduleNamed(TASK_NAME + ".watchUpdates", this::watchSourceUpdates, 5, TimeUnit.MINUTES); + } + + /** + * Safe refresh wrapper for background tasks. + */ + private void safeRefresh() { + safeRefresh(null); + } + + /** + * Safe refresh wrapper for background tasks. + * + * @param processId Optional user-visible process id. + */ + private void safeRefresh(@Nullable Long processId) { + try { + String res = refresh(processId); + + processMonitor.finish(processId, res); + logger.info(res); + } + catch (RuntimeException e) { + processMonitor.fail(processId, e); + logger.warn("Failed to refresh test fix matches", e); + } + } + + /** + * @param srvCode Server code. + * @param recentPrs Recent GitHub pull requests. + */ + private void refreshJira(String srvCode, List candidates, List recentPrs, + RefreshStats stats, @Nullable Long processId) { + try { + IJiraIgnited jira = jiraProvider.server(srvCode); + IJiraServerConfig jiraCfg = jira.config(); + String label = jiraCfg.testFixesLabel(); + long cutoffTs = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(jiraCfg.testFixesLookbackDays()); + + publishRefreshStatus(processId, "checking JIRA tickets for " + srvCode, stats); + for (Ticket ticket : jira.getTickets()) { + if (ticket == null || ticket.fields == null) + continue; + + stats.jiraChecked++; + + if (stats.jiraChecked % 100 == 0) + publishRefreshStatus(processId, "checked JIRA tickets for " + srvCode, stats); + + Long updatedTs = parseDate(ticket.fields.updated()); + Long closedTs = parseDate(ticket.fields.resolutionDate()); + boolean labeled = ticket.fields.labels().stream().anyMatch(label::equalsIgnoreCase); + + if (!labeled && updatedTs != null && updatedTs < cutoffTs) + continue; + + String text = Strings.nullToEmpty(ticket.fields.summary()) + "\n" + + Strings.nullToEmpty(ticket.fields.description()); + + List linkedPrs = relatedPullRequests(ticket, recentPrs); + + for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { + TestFixMatch match = candidate.newMatch(); + match.sourceType = "jira"; + match.sourceId = ticket.key; + match.sourceUrl = jira.generateTicketUrl(ticket.key); + match.title = ticket.fields.summary(); + match.status = statusText(ticket.status()); + match.updatedTs = updatedTs; + match.closedTs = closedTs; + + cache.put(cacheKey(match), match); + stats.jiraSaved++; + + for (PullRequest pr : linkedPrs) { + TestFixMatch linkedPrMatch = githubMatch(candidate, pr); + + cache.put(cacheKey(linkedPrMatch), linkedPrMatch); + stats.githubSaved++; + } + } + } + + publishRefreshStatus(processId, "checked JIRA tickets for " + srvCode, stats); + } + catch (RuntimeException e) { + logger.debug("Skipping JIRA test fix matching for " + srvCode, e); + publishRefreshStatus(processId, "skipped JIRA tickets for " + srvCode + ": " + e.getMessage(), stats); + } + } + + /** + * @param srvCode Server code. + * @param recentPrs Recent GitHub pull requests. + */ + private void refreshGithub(String srvCode, List candidates, List recentPrs, + RefreshStats stats, @Nullable Long processId) { + try { + publishRefreshStatus(processId, "checking GitHub pull requests for " + srvCode, stats); + for (PullRequest pr : recentPrs) { + stats.githubChecked++; + + if (stats.githubChecked % 25 == 0) + publishRefreshStatus(processId, "checked GitHub pull requests for " + srvCode, stats); + + String text = Strings.nullToEmpty(pr.getTitle()) + "\n" + Strings.nullToEmpty(pr.getBody()); + + for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { + TestFixMatch match = githubMatch(candidate, pr); + + cache.put(cacheKey(match), match); + stats.githubSaved++; + } + } + + publishRefreshStatus(processId, "checked GitHub pull requests for " + srvCode, stats); + } + catch (RuntimeException e) { + logger.debug("Skipping GitHub test fix matching for " + srvCode, e); + publishRefreshStatus(processId, "skipped GitHub pull requests for " + srvCode + ": " + e.getMessage(), + stats); + } + } + + /** + * @param srvCode Server code. + */ + private List recentPullRequests(String srvCode) { + try { + IJiraServerConfig jiraCfg = cfg.getJiraConfig(srvCode); + IGitHubConnIgnited gh = ghProvider.server(srvCode); + + return gh.getRecentPullRequests(jiraCfg.testFixesLookbackDays()); + } + catch (RuntimeException e) { + logger.debug("Skipping GitHub PR loading for test fix matching for " + srvCode, e); + + return new ArrayList<>(); + } + } + + /** + * @param candidate Candidate. + * @param pr Pull request. + */ + private static TestFixMatch githubMatch(TestFixCandidate candidate, PullRequest pr) { + TestFixMatch match = candidate.newMatch(); + + match.sourceType = "github"; + match.sourceId = "PR #" + pr.getNumber(); + match.sourceUrl = pr.htmlUrl(); + match.title = pr.getTitle(); + match.status = pr.getState(); + match.updatedTs = parseDate(pr.getTimeUpdate()); + match.closedTs = parseDate(pr.mergedAt()); + + fillGithubAuthor(match, pr.gitHubUser()); + fillCommit(match, pr); + + return match; + } + + /** + * @param ticket JIRA ticket. + * @param prs Recent pull requests. + */ + private static List relatedPullRequests(Ticket ticket, List prs) { + if (ticket == null || Strings.isNullOrEmpty(ticket.key) || prs.isEmpty()) + return new ArrayList<>(); + + String ticketKey = ticket.key.toUpperCase(Locale.ROOT); + + return prs.stream() + .filter(pr -> mentionsTicket(pr, ticketKey)) + .collect(Collectors.toList()); + } + + /** + * @param pr Pull request. + * @param ticketKey Uppercase JIRA ticket key. + */ + private static boolean mentionsTicket(PullRequest pr, String ticketKey) { + GitHubBranch head = pr.head(); + + return containsIgnoreCase(pr.getTitle(), ticketKey) || + containsIgnoreCase(pr.getBody(), ticketKey) || + (head != null && containsIgnoreCase(head.ref(), ticketKey)); + } + + /** + * @param text Text. + * @param token Uppercase token. + */ + private static boolean containsIgnoreCase(@Nullable String text, String token) { + return !Strings.isNullOrEmpty(text) && text.toUpperCase(Locale.ROOT).contains(token); + } + + /** + * @param match Match. + * @param user GitHub user. + */ + private static void fillGithubAuthor(TestFixMatch match, @Nullable GitHubUser user) { + if (user == null) + return; + + match.author = user.login(); + match.authorUrl = Strings.isNullOrEmpty(user.login()) ? null : "https://github.com/" + user.login(); + match.authorAvatarUrl = user.avatarUrl(); + } + + /** + * @param match Match. + * @param pr Pull request. + */ + private static void fillCommit(TestFixMatch match, PullRequest pr) { + String sha = pr.mergeCommitSha(); + + if (Strings.isNullOrEmpty(sha)) { + GitHubBranch head = pr.head(); + + if (head == null || Strings.isNullOrEmpty(head.sha())) + return; + + sha = head.sha(); + } + + match.commitSha = sha; + match.commitUrl = commitUrl(pr.htmlUrl(), sha); + } + + /** + * @param prUrl Pull request URL. + * @param sha Commit SHA. + */ + @Nullable private static String commitUrl(@Nullable String prUrl, String sha) { + if (Strings.isNullOrEmpty(prUrl)) + return null; + + int pullIdx = prUrl.indexOf("/pull/"); + + if (pullIdx < 0) + return null; + + return prUrl.substring(0, pullIdx) + "/commit/" + sha; + } + + /** + * @param name Entity name. + */ + private List findRefs(@Nullable String name) { + if (Strings.isNullOrEmpty(name)) + return new ArrayList<>(); + + String exact = normalize(name); + + ensureCache(); + + List res = StreamSupport.stream(cache.spliterator(), false) + .map(Cache.Entry::getValue) + .filter(TestFixesService::isPlausibleMatch) + .filter(match -> exact.equals(normalize(match.entityName)) || + exact.equals(normalize(match.testName)) || + exact.equals(normalize(match.suiteName))) + .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()) + .limit(5) + .map(this::toUi) + .collect(Collectors.toList()); + + return uniqueRefs(res); + } + + /** + * @param refs Refs. + */ + private static List uniqueRefs(List refs) { + Map res = new HashMap<>(); + + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); + + return new ArrayList<>(res.values()); + } + + /** + * @param match Match. + */ + private TestFixRefUi toUi(TestFixMatch match) { + TestFixRefUi res = new TestFixRefUi(); + + res.entityName = match.entityName; + res.suiteId = match.suiteId; + res.suiteName = match.suiteName; + res.testName = match.testName; + res.trackedBranch = match.trackedBranch; + res.currentStatusUrl = match.currentStatusUrl; + res.sourceType = match.sourceType; + res.text = match.sourceId; + res.url = match.sourceUrl; + res.title = match.title; + res.status = match.status; + res.author = match.author; + res.authorUrl = match.authorUrl; + res.authorAvatarUrl = match.authorAvatarUrl; + res.updatedDate = formatDate(match.updatedTs); + res.closedDate = formatDate(match.closedTs); + res.commitUrl = match.commitUrl; + res.commitText = Strings.isNullOrEmpty(match.commitSha) ? null : shortSha(match.commitSha); + + return res; + } + + /** + * @param status Status. + */ + private static String statusText(@Nullable JiraTicketStatusCode status) { + return JiraTicketStatusCode.text(status); + } + + /** + * @param sha SHA. + */ + private static String shortSha(String sha) { + return sha.length() <= PullRequest.INCLUDE_SHORT_VER ? sha : sha.substring(0, PullRequest.INCLUDE_SHORT_VER); + } + + /** + * @param match Match. + */ + private static String cacheKey(TestFixMatch match) { + return match.sourceType + ":" + match.sourceId + ":" + normalize(match.trackedBranch) + ":" + + normalize(match.suiteId) + ":" + normalize(match.entityName); + } + + /** + * @param value Value. + */ + private static String normalize(@Nullable String value) { + return Strings.nullToEmpty(value).trim().toLowerCase(Locale.ROOT); + } + + /** + * @param match Match. + */ + private static long sortTs(TestFixMatch match) { + if (match.closedTs != null) + return match.closedTs; + + if (match.updatedTs != null) + return match.updatedTs; + + return 0; + } + + /** + * @param date Date string. + */ + @Nullable private static Long parseDate(@Nullable String date) { + if (Strings.isNullOrEmpty(date)) + return null; + + try { + return Instant.parse(date).toEpochMilli(); + } + catch (DateTimeParseException ignored) { + // Try JIRA offset format below. + } + + try { + return OffsetDateTime.parse(date, DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")) + .toInstant().toEpochMilli(); + } + catch (DateTimeParseException ignored) { + return null; + } + } + + /** + * @param ts Timestamp. + */ + @Nullable private static String formatDate(@Nullable Long ts) { + if (ts == null) + return null; + + return DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.ofEpochMilli(ts).atZone(java.time.ZoneOffset.UTC)); + } + + /** + * @return Server codes. + */ + private Collection serverCodes() { + Set res = new HashSet<>(cfg.getServerIds()); + + if (!Strings.isNullOrEmpty(cfg.primaryServerCode())) + res.add(cfg.primaryServerCode()); + + return res; + } + + /** + * @return Match candidates grouped by TeamCity/GitHub/JIRA server code. + */ + private Map> candidatesByServer() { + Map> res = new HashMap<>(); + + cfg.getTrackedBranches().branchesStream().forEach(branch -> collectCandidates(branch, res)); + + return res; + } + + /** + * @param branch Tracked branch. + * @param res Target map. + */ + private void collectCandidates(ITrackedBranch branch, Map> res) { + branch.chainsStream().forEach(chain -> { + try { + res.computeIfAbsent(chain.serverCode(), k -> new ArrayList<>()) + .addAll(candidates(branch, chain)); + } + catch (RuntimeException e) { + logger.debug("Failed to collect test fix candidates [branch={}, server={}, suite={}]", + branch.name(), chain.serverCode(), chain.tcSuiteId(), e); + } + }); + } + + /** + * @param branch Tracked branch. + * @param chain Tracked chain. + */ + private List candidates(ITrackedBranch branch, ITrackedChain chain) { + ITeamcityIgnited tc = tcProvider.server(chain.serverCode(), null); + String baseBranch = chain.tcBaseBranch().orElse(chain.tcBranch()); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(baseBranch)); + + if (baseBranchId == null) + return new ArrayList<>(); + + List res = new ArrayList<>(); + + for (String suiteId : suiteIdsForHistory(tc, chain.tcSuiteId(), baseBranch)) + res.addAll(candidates(branch, tc, baseBranchId, suiteId)); + + return res; + } + + /** + * @param branch Tracked branch. + * @param tc TeamCity storage. + * @param baseBranchId Compacted base branch id. + * @param suiteId TeamCity build configuration id. + */ + private List candidates(ITrackedBranch branch, ITeamcityIgnited tc, Integer baseBranchId, + String suiteId) { + Integer compactedSuiteId = compactor.getStringIdIfPresent(suiteId); + + if (compactedSuiteId == null) + return new ArrayList<>(); + + ISuiteRunHistory hist = tc.getSuiteRunHist(compactedSuiteId, baseBranchId); + + if (hist == null) + return new ArrayList<>(); + + List res = new ArrayList<>(); + + for (Integer testNameId : hist.testNames()) { + String fullTestName = compactor.getStringFromId(testNameId); + + if (Strings.isNullOrEmpty(fullTestName)) + continue; + + String shortTestName = ShortTestFailureUi.extractTest(fullTestName); + + if (!isPlausibleTestName(shortTestName)) + continue; + + String suiteName = testSuiteName(fullTestName, shortTestName); + + String currentStatusUrl = tc.host() + "buildConfiguration/" + suiteId + "?branch=" + + org.apache.ignite.tcbot.common.util.UrlUtil.escape(ITcServerConfig.DEFAULT_TRACKED_BRANCH_NAME); + + res.add(new TestFixCandidate(branch.name(), suiteId, suiteName, fullTestName, shortTestName, + currentStatusUrl)); + } + + return res; + } + + /** + * @param tc TeamCity storage. + * @param rootSuiteId Root tracked chain build configuration id. + * @param baseBranch Base branch in TeamCity history. + */ + private List suiteIdsForHistory(ITeamcityIgnited tc, String rootSuiteId, String baseBranch) { + LinkedHashSet res = new LinkedHashSet<>(); + + collectSuiteIds(tc, rootSuiteId, res, new HashSet<>()); + collectSuiteIdsFromRecentBuilds(tc, rootSuiteId, baseBranch, res); + + return new ArrayList<>(res); + } + + /** + * @param tc TeamCity storage. + * @param suiteId TeamCity build configuration id. + * @param res Collected suite ids. + * @param visited Visited build configurations. + */ + private void collectSuiteIds(ITeamcityIgnited tc, @Nullable String suiteId, LinkedHashSet res, + Set visited) { + if (Strings.isNullOrEmpty(suiteId) || !visited.add(suiteId)) + return; + + res.add(suiteId); + + BuildTypeCompacted buildType = tc.getBuildType(suiteId); + + if (buildType == null) + return; + + for (SnapshotDependencyCompacted dep : buildType.snapshotDependencies()) { + BuildType depBuildType = dep.toSnapshotDependency(compactor).bt(); + + if (depBuildType != null) + collectSuiteIds(tc, depBuildType.getId(), res, visited); + } + } + + /** + * @param tc TeamCity storage. + * @param rootSuiteId Root tracked chain build configuration id. + * @param baseBranch Base branch in TeamCity history. + * @param res Collected suite ids. + */ + private void collectSuiteIdsFromRecentBuilds(ITeamcityIgnited tc, String rootSuiteId, String baseBranch, + LinkedHashSet res) { + try { + for (Integer buildId : tc.getLastNBuildsFromHistory(rootSuiteId, baseBranch, 3)) + collectSuiteIdsFromBuild(tc, buildId, res, new HashSet<>()); + } + catch (RuntimeException e) { + logger.debug("Failed to collect test fix suite ids from recent builds [suite={}, branch={}]", + rootSuiteId, baseBranch, e); + } + } + + /** + * @param tc TeamCity storage. + * @param buildId Build id. + * @param res Collected suite ids. + * @param visitedBuilds Visited build ids. + */ + private void collectSuiteIdsFromBuild(ITeamcityIgnited tc, @Nullable Integer buildId, LinkedHashSet res, + Set visitedBuilds) { + if (buildId == null || !visitedBuilds.add(buildId)) + return; + + FatBuildCompacted build = tc.getFatBuild(buildId); + + if (build == null || build.isFakeStub()) + return; + + String suiteId = build.buildTypeId(compactor); + + if (!Strings.isNullOrEmpty(suiteId)) + res.add(suiteId); + + for (int depId : build.snapshotDependencies()) + collectSuiteIdsFromBuild(tc, depId, res, visitedBuilds); + } + + /** + * @param fullTestName Full TeamCity test name. + * @param shortTestName Short test name in Class.method form. + */ + @Nullable private static String testSuiteName(String fullTestName, @Nullable String shortTestName) { + if (!Strings.isNullOrEmpty(shortTestName)) { + int dot = shortTestName.lastIndexOf('.'); + + if (dot > 0) + return shortTestName.substring(0, dot); + } + + return ShortTestFailureUi.extractSuite(fullTestName); + } + + /** + * @param text Source title/body/description. + * @param candidates Candidates from tracked branch histories. + */ + static Collection matchingCandidates(String text, List candidates) { + String rawSrc = normalize(text); + String src = mentionKey(rawSrc); + Map res = new LinkedHashMap<>(); + + for (TestFixCandidate candidate : candidates) { + if (candidate.matchesQualified(src)) + res.putIfAbsent(candidate.key(), candidate); + } + + if (!res.isEmpty()) + return res.values(); + + for (TestFixCandidate candidate : candidates) { + if (candidate.matchesUnqualified(src, rawSrc)) + res.putIfAbsent(candidate.key(), candidate); + } + + return res.values(); + } + + /** + * @param match Persisted match. + */ + private static boolean isPlausibleMatch(TestFixMatch match) { + return match != null && isPlausibleTestName(match.testName) && !Strings.isNullOrEmpty(match.suiteId); + } + + /** + * @param testName Short test name in Class.method form. + */ + static boolean isPlausibleTestName(@Nullable String testName) { + if (Strings.isNullOrEmpty(testName)) + return false; + + return testName.matches("[A-Za-z_$][A-Za-z0-9_$]*[.][A-Za-z_$][A-Za-z0-9_$]*(?:\\[[^\\]]+\\])?"); + } + + /** + * @param value Source text or a candidate alias. + */ + private static String mentionKey(@Nullable String value) { + String normalized = normalize(value); + StringBuilder res = new StringBuilder(normalized.length()); + boolean lastSeparator = true; + + for (int i = 0; i < normalized.length(); i++) { + char ch = normalized.charAt(i); + + if (Character.isLetterOrDigit(ch)) { + res.append(ch); + lastSeparator = false; + } + else if (!lastSeparator) { + res.append('.'); + lastSeparator = true; + } + } + + int len = res.length(); + + if (len > 0 && res.charAt(len - 1) == '.') + res.deleteCharAt(len - 1); + + return res.toString(); + } + + /** + * @param src Canonical source text. + * @param token Source token. + */ + private static boolean containsMention(String src, @Nullable String token) { + if (Strings.isNullOrEmpty(token)) + return false; + + String needle = mentionKey(token); + + if (Strings.isNullOrEmpty(needle)) + return false; + + int start = 0; + + while (start <= src.length() - needle.length()) { + int idx = src.indexOf(needle, start); + + if (idx < 0) + return false; + + int end = idx + needle.length(); + boolean leftBoundary = idx == 0 || src.charAt(idx - 1) == '.'; + boolean rightBoundary = end == src.length() || src.charAt(end) == '.'; + + if (leftBoundary && rightBoundary) + return true; + + start = idx + 1; + } + + return false; + } + + /** + * @param src Normalized source text. + * @param token Standalone name token. + */ + private static boolean containsStandaloneName(String src, @Nullable String token) { + String needle = normalize(token); + + if (Strings.isNullOrEmpty(needle)) + return false; + + int start = 0; + + while (start <= src.length() - needle.length()) { + int idx = src.indexOf(needle, start); + + if (idx < 0) + return false; + + int end = idx + needle.length(); + boolean leftBoundary = idx == 0 || !Character.isLetterOrDigit(src.charAt(idx - 1)); + boolean rightBoundary = end == src.length() || !Character.isLetterOrDigit(src.charAt(end)); + boolean followedByMethod = end + 1 < src.length() && (src.charAt(end) == '.' || src.charAt(end) == '#') && + Character.isLetterOrDigit(src.charAt(end + 1)); + + if (leftBoundary && rightBoundary && !followedByMethod) + return true; + + start = idx + 1; + } + + return false; + } + + /** + * Initializes cache. + */ + private void ensureCache() { + if (cache != null) + return; + + synchronized (this) { + if (cache == null) { + cache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(CACHE_NAME)); + signalCache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig(SOURCE_UPDATES_CACHE_NAME)); + } + } + } + + /** + * @return Latest source update signal timestamp. + */ + private long latestSignalTs() { + ensureCache(); + + long res = 0; + + for (Cache.Entry entry : signalCache) { + if (entry.getValue() != null) + res = Math.max(res, entry.getValue()); + } + + return res; + } + + /** + * @param processId Optional user-visible process id. + * @param stage Current refresh stage. + * @param stats Refresh stats. + */ + private void publishRefreshStatus(@Nullable Long processId, String stage, RefreshStats stats) { + processMonitor.status(processId, "Test fix mapping: " + stage + ". " + stats.progressText()); + } + + /** Test fix refresh counters. */ + private static class RefreshStats { + /** Candidate test names from tracked suite histories. */ + private int candidates; + + /** Unique suites with history candidates. */ + private int suites; + + /** Unique full test names with history candidates. */ + private int tests; + + /** Recent GitHub PRs loaded from cache. */ + private int githubLoaded; + + /** GitHub PRs checked against candidates. */ + private int githubChecked; + + /** JIRA tickets checked against candidates. */ + private int jiraChecked; + + /** Saved JIRA matches. */ + private int jiraSaved; + + /** Saved GitHub matches. */ + private int githubSaved; + + /** + * @param candidatesByServer Candidates by server. + */ + private void collectCandidateStats(Map> candidatesByServer) { + Set suiteIds = new HashSet<>(); + Set testNames = new HashSet<>(); + + for (Map.Entry> entry : candidatesByServer.entrySet()) { + for (TestFixCandidate candidate : entry.getValue()) { + candidates++; + suiteIds.add(entry.getKey() + ":" + candidate.suiteId); + testNames.add(candidate.fullTestName); + } + } + + suites = suiteIds.size(); + tests = testNames.size(); + } + + /** */ + private int saved() { + return jiraSaved + githubSaved; + } + + /** */ + private String progressText() { + return "Analyzed " + tests + " unique test names (" + candidates + " candidate rows) in " + suites + + " suites. Checked sources: GitHub " + githubChecked + "/" + githubLoaded + " PRs, JIRA " + + jiraChecked + " tickets. Matches saved: " + saved() + " (JIRA " + jiraSaved + ", GitHub " + + githubSaved + ")."; + } + + /** */ + private String finishText() { + return "Test fix matches refreshed: " + saved() + " (tests=" + tests + ", candidates=" + candidates + + ", suites=" + suites + ", githubChecked=" + githubChecked + "/" + githubLoaded + + ", jiraChecked=" + jiraChecked + ", jira=" + jiraSaved + ", github=" + githubSaved + ")"; + } + } + + /** Test fix candidate resolved from tracked branch suite history. */ + static class TestFixCandidate { + /** Tracked branch. */ + private final String trackedBranch; + + /** Suite id. */ + private final String suiteId; + + /** Suite display name. */ + private final String suiteName; + + /** Full test name. */ + private final String fullTestName; + + /** Short test name. */ + private final String shortTestName; + + /** Current master TeamCity suite status URL. */ + private final String currentStatusUrl; + + /** + * @param trackedBranch Tracked branch. + * @param suiteId Suite id. + * @param suiteName Suite display name. + * @param fullTestName Full test name. + * @param shortTestName Short test name. + */ + TestFixCandidate(String trackedBranch, String suiteId, @Nullable String suiteName, String fullTestName, + @Nullable String shortTestName, String currentStatusUrl) { + this.trackedBranch = trackedBranch; + this.suiteId = suiteId; + this.suiteName = suiteName; + this.fullTestName = fullTestName; + this.shortTestName = shortTestName; + this.currentStatusUrl = currentStatusUrl; + } + + /** + * @param src Canonical source text. + */ + private boolean matchesQualified(String src) { + return containsMention(src, suiteId + "." + shortTestName) || + containsMention(src, suiteId + "." + fullTestName); + } + + /** + * @param src Canonical source text. + * @param rawSrc Normalized source text. + */ + private boolean matchesUnqualified(String src, String rawSrc) { + return containsMention(src, fullTestName) || + containsMention(src, shortTestName); + } + + /** */ + private TestFixMatch newMatch() { + TestFixMatch match = new TestFixMatch(); + + match.entityName = Strings.isNullOrEmpty(shortTestName) ? fullTestName : shortTestName; + match.trackedBranch = trackedBranch; + match.suiteId = suiteId; + match.suiteName = suiteName; + match.testName = Strings.isNullOrEmpty(shortTestName) ? fullTestName : shortTestName; + match.currentStatusUrl = currentStatusUrl; + + return match; + } + + /** */ + private String key() { + return normalize(suiteId) + ":" + normalize(fullTestName); + } + + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java index 13baae23d..9a85bfe5b 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java @@ -37,6 +37,7 @@ public interface IDetailedStatusForTrackedBranch { * @param calcTrustedTests Calculate trusted tests count. * @param tagSelected Selected tag based filter. If null or empty all data is returned. * @param tagForHistSelected Selected tag for filtering history (applicable to reruns and history stripe). + * @param suiteId Suite id to include. If null or empty all suites are returned. * @param displayMode Suites and tests display mode. Default - failures only. * @param sortOption Sort mode * @param maxDurationSec Show test as failed if duration is greater than provided seconds count. @@ -52,6 +53,7 @@ public DsSummaryUi getTrackedBranchTestFailures( boolean calcTrustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nullable DisplayMode displayMode, @Nullable SortOption sortOption, int maxDurationSec, diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index a63688387..e3dcc6444 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -53,6 +53,7 @@ import org.apache.ignite.tcbot.engine.conf.ITrackedChain; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; @@ -112,6 +113,9 @@ public class TrackedBranchChainsProcessor implements IDetailedStatusForTrackedBr /** User-visible process monitor. */ @Inject private BotProcessMonitor processMonitor; + /** Test fix matcher. */ + @Inject private TestFixesService testFixesService; + /** * @param branch Branch. * @param buildResMergeCnt Build results merge count. @@ -356,6 +360,7 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { boolean calcTrustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nullable DisplayMode displayMode, @Nullable SortOption sortOption, int maxDurationSec, @@ -379,6 +384,7 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { List accessibleChains = tracked.chainsStream() .filter(chainTracked -> tcIgnitedProv.hasAccess(chainTracked.serverCode(), creds)) + .filter(chainTracked -> Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainTracked.tcSuiteId())) .collect(Collectors.toList()); for (ITrackedChain chainTracked : accessibleChains) { @@ -443,6 +449,7 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests, tagSelected, displayMode, maxDurationSec, requireParamVal, showMuted, showIgnored); + chainStatus.suites.forEach(testFixesService::decorate); uiInitNanos += System.nanoTime() - stepStart; res.addChainOnServer(chainStatus); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java index 8f042a742..14e75c880 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistory; @@ -44,6 +45,9 @@ public class ShortSuiteUi extends DsHistoryStatUi { public List testShortFailures = new ArrayList<>(); + /** Matched tickets/PRs that mention fixing this suite. */ + public List fixRefs = new ArrayList<>(); + /** Web Href. to suite particular run */ public String webToBuild = ""; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java index a1a8560eb..b3c0a14ea 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java @@ -17,11 +17,14 @@ package org.apache.ignite.tcbot.engine.ui; import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistory; @@ -41,6 +44,9 @@ public class ShortTestFailureUi { /** Blocker comment: indicates test seems to be introduced failure. */ @Nullable public String blockerComment; + /** Matched tickets/PRs that mention fixing this test. */ + public List fixRefs = new ArrayList<>(); + /** * */ diff --git a/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java new file mode 100644 index 000000000..7fe4697b3 --- /dev/null +++ b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import java.util.Collection; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for test-fix source text matching. + */ +public class TestFixesServiceTest { + /** Test suite/class name. */ + private static final String SUITE = "SqlPlanHistoryIntegrationTest"; + + /** Test method name. */ + private static final String TEST = "cacheQuery"; + + /** Short test name. */ + private static final String SHORT_TEST = SUITE + "." + TEST; + + /** Full TeamCity test name. */ + private static final String FULL_TEST = + "org.apache.ignite.internal.processors.query.calcite.integration." + SHORT_TEST; + + /** Cache run configuration. */ + private static final TestFixesService.TestFixCandidate CACHE8 = candidate("IgniteTests24Java8_Cache8"); + + /** Queries run configuration. */ + private static final TestFixesService.TestFixCandidate QUERIES1 = candidate("IgniteTests24Java8_Queries1"); + + /** */ + @Test + public void jiraTextMatchesSuiteDotTestInAllRunConfigurations() { + Collection matches = jiraMatches("Fix " + SHORT_TEST + " flakiness"); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void jiraTextMatchesSuiteHashTestInAllRunConfigurations() { + Collection matches = jiraMatches("Fix " + SUITE + "#" + TEST); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void jiraTextMatchesQualifiedRunConfigurationOnly() { + Collection matches = jiraMatches( + "Fix IgniteTests24Java8_Cache8::" + SUITE + "#" + TEST); + + assertEquals(1, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertFalse(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesSuiteDotTestInAllRunConfigurations() { + Collection matches = prMatches( + "IGNITE-1 Fix test", + "This PR fixes " + SHORT_TEST + " in master."); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesSuiteHashTestInAllRunConfigurations() { + Collection matches = prMatches( + "IGNITE-1 Fix " + SUITE + "#" + TEST, + "No comments from code review should be needed."); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesQualifiedRunConfigurationOnly() { + Collection matches = prMatches( + "IGNITE-1 Fix IgniteTests24Java8_Cache8::" + SUITE + "#" + TEST, + "The same test exists in another suite, but this PR names the run configuration."); + + assertEquals(1, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertFalse(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void partialWordsDoNotMatch() { + Collection matches = jiraMatches( + "Fix Not" + SHORT_TEST + " and " + SHORT_TEST + "Extra"); + + assertTrue(matches.isEmpty()); + } + + /** */ + @Test + public void suiteNameAloneDoesNotMatch() { + Collection matches = jiraMatches( + "Fix " + SUITE + " instability in master"); + + assertTrue(matches.isEmpty()); + } + + /** */ + @Test + public void garbageTeamcityNamesAreNotPlausibleTests() { + assertFalse(TestFixesService.isPlausibleTestName("0.0\\\\, Culture=neutral\\\\, PublicKeyToken=123\\\\]\\\\]\")")); + assertFalse(TestFixesService.isPlausibleTestName("Configuration.ConfigurationException: x)")); + assertFalse(TestFixesService.isPlausibleTestName("Data.PropertyCollection)")); + assertFalse(TestFixesService.isPlausibleTestName("Log.CustomLoggerTest+CustomEnum)")); + + assertTrue(TestFixesService.isPlausibleTestName("OpenCensusSqlNativeTracingTest.testNextPageRequestFailure")); + assertTrue(TestFixesService.isPlausibleTestName("SqlPlanHistoryIntegrationTest.cacheQuery")); + } + + /** + * @param text JIRA summary plus description. + */ + private static Collection jiraMatches(String text) { + return TestFixesService.matchingCandidates(text, candidates()); + } + + /** + * @param title PR title. + * @param body PR body. + */ + private static Collection prMatches(String title, String body) { + return TestFixesService.matchingCandidates(title + "\n" + body, candidates()); + } + + /** */ + private static List candidates() { + return List.of(CACHE8, QUERIES1); + } + + /** + * @param suiteId Run configuration id. + */ + private static TestFixesService.TestFixCandidate candidate(String suiteId) { + return new TestFixesService.TestFixCandidate("master", suiteId, SUITE, FULL_TEST, SHORT_TEST, + "https://ci.example/buildConfiguration/" + suiteId + "?branch=%3Cdefault%3E"); + } +} diff --git a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java index 117dd11f8..2bff1ee1d 100644 --- a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java @@ -18,6 +18,8 @@ import java.io.FileNotFoundException; import java.io.UncheckedIOException; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -154,6 +156,12 @@ public void init(IGitHubConnection conn) { return runActualizePrs(srvCode, false); } + /** {@inheritDoc} */ + @AutoProfiling + @Override public List getRecentPullRequests(int lookbackDays) { + return loadRecentPullRequests(Math.max(1, lookbackDays)); + } + /** {@inheritDoc} */ @AutoProfiling @Override public List getBranches() { @@ -198,6 +206,55 @@ private void actualizePrs() { 5, TimeUnit.MINUTES); } + /** + * @param lookbackDays Lookback window. + */ + private List loadRecentPullRequests(int lookbackDays) { + long cutoffMs = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(lookbackDays); + AtomicReference outLinkNext = new AtomicReference<>(); + List res = new java.util.ArrayList<>(); + String nextPageUrl = null; + + do { + List page = conn.getRecentPullRequestsPage(nextPageUrl, outLinkNext); + + enrichPullRequestAuthors(page); + savePrsChunk(page); + + for (PullRequest pr : page) { + Long updatedTs = parseGithubTime(pr.getTimeUpdate()); + + if (updatedTs == null || updatedTs >= cutoffMs) + res.add(pr); + } + + boolean reachedOldPage = page.stream() + .map(PullRequest::getTimeUpdate) + .map(GitHubConnIgnitedImpl::parseGithubTime) + .anyMatch(ts -> ts != null && ts < cutoffMs); + + nextPageUrl = reachedOldPage ? null : outLinkNext.get(); + } + while (nextPageUrl != null); + + return res; + } + + /** + * @param time GitHub ISO time. + */ + @Nullable private static Long parseGithubTime(@Nullable String time) { + if (time == null || time.isEmpty()) + return null; + + try { + return Instant.parse(time).toEpochMilli(); + } + catch (DateTimeParseException e) { + return null; + } + } + /** * */ @@ -244,9 +301,22 @@ protected String runActualizePrs(String srvId, boolean fullReindex) { if (fullReindex) refreshOutdatedPrs(srvId, actualPrs); + if (cntSaved > 0) + signalSourceUpdate("github:" + srvId); + return "Entries saved " + cntSaved + " PRs checked " + totalChecked; } + /** + * @param key Source key. + */ + private void signalSourceUpdate(String key) { + IgniteCache cache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig("testFixSourceUpdates")); + + cache.put(key, System.currentTimeMillis()); + } + /** * Loads full public GitHub profiles for PR authors when GitHub /pulls returned compact users without email. * diff --git a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java index 6327b8311..f3205e6ad 100644 --- a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java @@ -53,6 +53,14 @@ public interface IGitHubConnIgnited { */ public String refreshPullRequests(); + /** + * Loads recently updated pull requests, including closed/merged PRs. + * + * @param lookbackDays Number of recent days. + * @return Recent pull requests. + */ + public List getRecentPullRequests(int lookbackDays); + /** * Reloads branches from GitHub immediately. * diff --git a/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java index a04928165..8757ada19 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java @@ -39,7 +39,7 @@ public class PullRequest implements IVersionedEntity { public static final int INCLUDE_SHORT_VER = 7; /** Entitiy current (latest) version. */ - private static final int LATEST_VERSION = 7; + private static final int LATEST_VERSION = 10; /** Entity version. */ @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; @@ -53,10 +53,15 @@ public class PullRequest implements IVersionedEntity { /** Pull Request title. */ private String title; + /** Pull Request body. */ + private String body; + @SerializedName("html_url") private String htmlUrl; @SerializedName("updated_at") private String updatedAt; + @SerializedName("merged_at") private String mergedAt; + /** Pull Request statuses URL. */ @SerializedName("statuses_url") private String statusesUrl; @@ -66,6 +71,8 @@ public class PullRequest implements IVersionedEntity { @SerializedName("base") private GitHubBranch base; + @SerializedName("merge_commit_sha") private String mergeCommitSha; + /** * @return Pull Request time update. */ @@ -73,6 +80,13 @@ public String getTimeUpdate() { return updatedAt; } + /** + * @return Pull Request merge time. + */ + @Nullable public String mergedAt() { + return mergedAt; + } + /** * @return Pull Request number. */ @@ -94,6 +108,13 @@ public String getTitle() { return title; } + /** + * @return Pull Request body. + */ + public String getBody() { + return body; + } + /** * @return Url to get PR statuses. */ @@ -129,6 +150,13 @@ public GitHubBranch head() { return head; } + /** + * @return Merge commit SHA. + */ + @Nullable public String mergeCommitSha() { + return mergeCommitSha; + } + /** {@inheritDoc} */ @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -152,17 +180,21 @@ public GitHubBranch head() { Objects.equals(_ver, req._ver) && Objects.equals(state, req.state) && Objects.equals(title, req.title) && + Objects.equals(body, req.body) && Objects.equals(htmlUrl, req.htmlUrl) && Objects.equals(updatedAt, req.updatedAt) && + Objects.equals(mergedAt, req.mergedAt) && Objects.equals(statusesUrl, req.statusesUrl) && Objects.equals(gitHubUser, req.gitHubUser) && Objects.equals(head, req.head) && - Objects.equals(base, req.base); + Objects.equals(base, req.base) && + Objects.equals(mergeCommitSha, req.mergeCommitSha); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(_ver, num, state, title, htmlUrl, updatedAt, statusesUrl, gitHubUser, head, base); + return Objects.hash(_ver, num, state, title, body, htmlUrl, updatedAt, mergedAt, statusesUrl, gitHubUser, head, base, + mergeCommitSha); } /** {@inheritDoc} */ diff --git a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java index ab0496d8c..a8055a4ab 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java @@ -273,6 +273,26 @@ class GitHubConnectionImpl implements IGitHubConnection { return readOnePage(outLinkNext, url, rspHeaders, tok); } + /** {@inheritDoc} */ + @AutoProfiling + @Override public List getRecentPullRequestsPage(@Nullable String fullUrl, + @Nullable AtomicReference outLinkNext) { + String gitApiUrl = getApiUrlMandatory(); + + String url = fullUrl != null ? fullUrl : gitApiUrl + "pulls?state=all&sort=updated&direction=desc&per_page=100"; + + HashMap rspHeaders = new HashMap<>(); + if (outLinkNext != null) { + outLinkNext.set(null); + rspHeaders.put("Link", null); + } + + TypeToken> tok = new TypeToken>() { + }; + + return readOnePage(outLinkNext, url, rspHeaders, tok); + } + @Nonnull public String getApiUrlMandatory() { String gitApiUrl = config().gitApiUrl(); diff --git a/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java index cbd81dc08..34c16fd29 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java @@ -50,6 +50,13 @@ public interface IGitHubConnection { */ public List getPullRequestsPage(@Nullable String fullUrl, @Nullable AtomicReference outLinkNext); + /** + * @param fullUrl Full url - null for first page, not null for next page. + * @param outLinkNext Out link for return next page full url. + */ + public List getRecentPullRequestsPage(@Nullable String fullUrl, + @Nullable AtomicReference outLinkNext); + /** * @param prNum Pull request number. * @return Pull request issue comments. diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java index 797174019..dad0fa66e 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java @@ -18,6 +18,8 @@ package org.apache.ignite.ci.jira.ignited; import java.util.Objects; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nullable; import org.apache.ignite.ci.tcbot.common.StringFieldCompacted; import org.apache.ignite.jiraservice.Status; @@ -56,6 +58,15 @@ public class TicketCompacted { /** Internal JIRA id of ticket status, see {@link org.apache.ignite.jiraservice.JiraTicketStatusCode}. */ public int statusCodeId; + /** Labels joined by new lines, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted labels = new StringFieldCompacted(); + + /** Last updated date as returned by JIRA, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted updated = new StringFieldCompacted(); + + /** Resolution date as returned by JIRA, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted resolutiondate = new StringFieldCompacted(); + /** * @param ticket Jira ticket. * @param comp Compactor. @@ -68,6 +79,9 @@ public TicketCompacted(Ticket ticket, IStringCompactor comp, String projectCode) summary.setValue(ticket.fields.summary()); customfield_11050.setValue(ticket.fields.igniteLink()/*customfield_11050*/); description.setValue(ticket.fields.description()); + labels.setValue(String.join("\n", ticket.fields.labels())); + updated.setValue(ticket.fields.updated()); + resolutiondate.setValue(ticket.fields.resolutionDate()); } /** @@ -85,12 +99,37 @@ public Ticket toTicket(IStringCompactor comp, String projectCode) { fields.summary = summary != null ? summary.getValue() : null; fields.customfield_11050 = customfield_11050 != null ? customfield_11050.getValue() : null; fields.description = description != null ? description.getValue() : null; + fields.labels = splitLines(labels); + fields.updated = updated != null ? updated.getValue() : null; + fields.resolutiondate = resolutiondate != null ? resolutiondate.getValue() : null; ticket.fields = fields; return ticket; } + /** + * @param val Compacted string field. + */ + private static List splitLines(@Nullable StringFieldCompacted val) { + List res = new ArrayList<>(); + + if (val == null) + return res; + + String text = val.getValue(); + + if (text == null || text.isEmpty()) + return res; + + for (String label : text.split("\\n")) { + if (!label.isEmpty()) + res.add(label); + } + + return res; + } + /** {@inheritDoc} */ @Override public boolean equals(Object o) { if (this == o) @@ -106,11 +145,15 @@ public Ticket toTicket(IStringCompactor comp, String projectCode) { statusCodeId == compacted.statusCodeId && Objects.equals(summary, compacted.summary) && Objects.equals(customfield_11050, compacted.customfield_11050) && - Objects.equals(description, compacted.description); + Objects.equals(description, compacted.description) && + Objects.equals(labels, compacted.labels) && + Objects.equals(updated, compacted.updated) && + Objects.equals(resolutiondate, compacted.resolutiondate); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(id, igniteId, statusCodeId, summary, customfield_11050, description); + return Objects.hash(id, igniteId, statusCodeId, summary, customfield_11050, description, labels, updated, + resolutiondate); } } diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java index b4edb515a..f7c483784 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java @@ -21,6 +21,9 @@ import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.IJiraIntegration; @@ -29,6 +32,7 @@ import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; +import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +49,9 @@ public class JiraTicketSync { /** Scheduler. */ @Inject private IScheduler scheduler; + /** Ignite provider. */ + @Inject private Provider igniteProvider; + /** Mute DAO. */ @Inject private JiraTicketDao jiraDao; @@ -98,11 +105,54 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { IJiraServerConfig cfg = jira.config(); String projectCode = cfg.projectCodeForVisa(); - String baseUrl = cfg.getApiVersion().searchUrl() + escape("project=" + projectCode + " order by updated DESC") - + "&" + - "fields=" + reqFields + - "&maxResults=100"; + String baseUrl = searchUrl(cfg, "project=" + projectCode + " order by updated DESC", reqFields); + + SyncStats stats = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, baseUrl, fullResync); + SyncStats recent = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND updated >= -" + cfg.testFixesLookbackDays() + + "d order by updated DESC", reqFields), true); + SyncStats labeled = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND labels = " + cfg.testFixesLabel() + + " order by updated DESC", reqFields), true); + + int saved = stats.saved + recent.saved + labeled.saved; + + if (saved > 0) + signalSourceUpdate("jira:" + srvCode); + + return "Jira tickets saved " + saved + " from " + + (stats.processed + recent.processed + labeled.processed) + " checked for service " + srvCode; + } + + /** + * @param key Source key. + */ + private void signalSourceUpdate(String key) { + IgniteCache cache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig("testFixSourceUpdates")); + + cache.put(key, System.currentTimeMillis()); + } + /** + * @param cfg JIRA config. + * @param jql JQL query. + * @param reqFields Requested fields. + */ + private String searchUrl(IJiraServerConfig cfg, String jql, String reqFields) { + return cfg.getApiVersion().searchUrl() + escape(jql) + "&fields=" + reqFields + "&maxResults=100"; + } + + /** + * @param jira JIRA facade. + * @param cfg JIRA config. + * @param srvIdMaskHigh Server id mask high. + * @param projectCode Project code. + * @param baseUrl Search URL. + * @param fullResync Full resync flag. + */ + private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int srvIdMaskHigh, String projectCode, + String baseUrl, boolean fullResync) { String url = baseUrl; logger.info("Requesting JIRA tickets using URL " + url + ("\n" + cfg.restApiUrl() + url)); @@ -110,12 +160,14 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { Collection page = tickets.issues(); if (F.isEmpty(page)) - return "Something went wrong - no tickets found. Check jira availability: " + - "[project=" + projectCode + ", url=" + url + "]"; + return new SyncStats(); + + SyncStats res = new SyncStats(); int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); - int ticketsProcessed = page.size(); + res.saved += ticketsSaved; + res.processed += page.size(); if (ticketsSaved != 0 || fullResync) { while (tickets.hasNextPage()) { @@ -131,14 +183,23 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); - ticketsSaved += savedNow; - ticketsProcessed += page.size(); + res.saved += savedNow; + res.processed += page.size(); if (savedNow == 0 && !fullResync) break; // find not updated chunk and exit } } - return "Jira tickets saved " + ticketsSaved + " from " + ticketsProcessed + " checked for service " + srvCode; + return res; + } + + /** Sync stats. */ + private static class SyncStats { + /** Saved count. */ + private int saved; + + /** Processed count. */ + private int processed; } } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java index 9e0666929..3916633d9 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java @@ -24,7 +24,10 @@ public interface IFields { "status", "summary", "customfield_11050", - "description" + "description", + "labels", + "updated", + "resolutiondate" ); /** Ticket status. */ @@ -38,4 +41,13 @@ public interface IFields { /** Description. */ String description(); + + /** Labels. */ + List labels(); + + /** Last updated date as returned by JIRA. */ + String updated(); + + /** Resolution date as returned by JIRA. */ + String resolutionDate(); } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java index 616b9474f..a1ef8d4fa 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java @@ -18,6 +18,8 @@ package org.apache.ignite.jiraservice.v2; import com.google.common.base.MoreObjects; +import java.util.Collections; +import java.util.List; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.Status; @@ -37,6 +39,15 @@ public class Fields implements IFields { /** Description. */ public String description; + /** Labels. */ + public List labels; + + /** Last updated date. */ + public String updated; + + /** Resolution date. */ + public String resolutiondate; + /** {@inheritDoc} */ @Override public Status status() { return status; @@ -57,12 +68,28 @@ public class Fields implements IFields { return description; } + /** {@inheritDoc} */ + @Override public List labels() { + return labels == null ? Collections.emptyList() : labels; + } + + /** {@inheritDoc} */ + @Override public String updated() { + return updated; + } + + /** {@inheritDoc} */ + @Override public String resolutionDate() { + return resolutiondate; + } + /** {@inheritDoc} */ @Override public String toString() { return MoreObjects.toStringHelper(this) .add("status", status) .add("summary", summary) .add("customfield_11050", customfield_11050) + .add("updated", updated) .toString(); } } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java index ad221f93d..876907795 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java @@ -18,6 +18,7 @@ package org.apache.ignite.jiraservice.v3; import com.google.common.base.MoreObjects; +import java.util.Collections; import java.util.List; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.Status; @@ -39,6 +40,15 @@ public class FieldsV3 implements IFields { /** Description. */ private Description description; + /** Labels. */ + private List labels; + + /** Last updated date. */ + private String updated; + + /** Resolution date. */ + private String resolutiondate; + /** {@inheritDoc} */ @Override public Status status() { return status; @@ -65,11 +75,27 @@ public class FieldsV3 implements IFields { return sb.toString(); } + /** {@inheritDoc} */ + @Override public List labels() { + return labels == null ? Collections.emptyList() : labels; + } + + /** {@inheritDoc} */ + @Override public String updated() { + return updated; + } + + /** {@inheritDoc} */ + @Override public String resolutionDate() { + return resolutiondate; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("status", status) .add("summary", summary) .add("customfield_11050", customfield_11050) + .add("updated", updated) .toString(); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java index 1fae99acd..120325991 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.tcignited.history; +import java.util.Set; import java.util.Map; import javax.annotation.Nullable; @@ -23,5 +24,10 @@ public interface ISuiteRunHistory { IRunHistory self(); @Nullable IRunHistory getTestRunHist(int testName); + /** + * @return Test name ids present in this suite history. + */ + Set testNames(); + ISuiteRunHistory filter(Map requireParameters); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java index 51b40a6e0..d59c40455 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java @@ -101,6 +101,11 @@ private void addSuiteInvocation(SuiteInvocation suiteInv, Map testNames() { + return testsInvStatues.keySet(); + } + /** {@inheritDoc} */ @Override public ISuiteRunHistory filter(Map requireParameters) { RunHistCompacted suitesFiltered = suiteHist.filterSuiteInvByParms(requireParameters); From e46cf5124b155d9f0dbf74afa64c8abf447b6e27 Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 20:20:25 +0300 Subject: [PATCH 2/6] Merge master and add integration coverage --- .run/TC Bot Server WAR.run.xml | 11 + .run/TC Bot Server.run.xml | 11 + .../rest/monitoring/MonitoringService.java | 66 +---- .../rest/testfixes/TestFixesRestService.java | 3 + .../engine/chain/BuildChainProcessor.java | 7 +- ...TestFixesAndCancelledOrderingFlowTest.java | 274 ++++++++++++++++++ .../ignite/jiraignited/JiraTicketSync.java | 165 ++++++++++- 7 files changed, 468 insertions(+), 69 deletions(-) create mode 100644 .run/TC Bot Server WAR.run.xml create mode 100644 .run/TC Bot Server.run.xml create mode 100644 tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java diff --git a/.run/TC Bot Server WAR.run.xml b/.run/TC Bot Server WAR.run.xml new file mode 100644 index 000000000..9c286685d --- /dev/null +++ b/.run/TC Bot Server WAR.run.xml @@ -0,0 +1,11 @@ + + + + diff --git a/.run/TC Bot Server.run.xml b/.run/TC Bot Server.run.xml new file mode 100644 index 000000000..76962d53f --- /dev/null +++ b/.run/TC Bot Server.run.xml @@ -0,0 +1,11 @@ + + + + diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java index 51841678f..bae9e1521 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java @@ -37,9 +37,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -48,7 +46,6 @@ import javax.annotation.security.RolesAllowed; import javax.cache.Cache; import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; import javax.ws.rs.BadRequestException; import javax.ws.rs.ClientErrorException; import javax.ws.rs.ForbiddenException; @@ -66,8 +63,6 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMetrics; import org.apache.ignite.cache.affinity.Affinity; -import org.apache.ignite.ci.user.ITcBotUserCreds; -import org.apache.ignite.ci.user.TcHelperUser; import org.apache.ignite.ci.web.CtxListener; import org.apache.ignite.ci.web.auth.AuthenticationFilter; import org.apache.ignite.ci.web.model.SimpleResult; @@ -80,7 +75,6 @@ import org.apache.ignite.tcbot.engine.conf.NotificationsConfig; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; import org.apache.ignite.tcbot.engine.process.BotProcessStatus; -import org.apache.ignite.tcbot.engine.user.IUserStorage; import org.apache.ignite.tcbot.notify.IEmailSender; import org.apache.ignite.tcbot.notify.ISendEmailConfig; import org.apache.ignite.tcbot.notify.ISlackSender; @@ -123,10 +117,13 @@ public class MonitoringService { private static final int SUMMARY_LIMIT = 240; /** Default number of cache entries to preview. */ - private static final int DFLT_CACHE_PEEK_LIMIT = 20; + private static final int DFLT_CACHE_PEEK_LIMIT = 5; /** Hard cache preview cap. */ - private static final int MAX_CACHE_PEEK_LIMIT = 100; + private static final int MAX_CACHE_PEEK_LIMIT = 20; + + /** System property with comma-separated exact cache names allowed for raw preview. */ + private static final String CACHE_PEEK_ALLOWED_CACHES = "tcbot.monitoring.cachePeek.allowedCaches"; /** JSON mapper for raw cache entry values. */ private static final ObjectMapper CACHE_PEEK_MAPPER = new ObjectMapper() @@ -139,10 +136,6 @@ public class MonitoringService { @Context private ServletContext ctx; - /** Request. */ - @Context - private HttpServletRequest req; - @GET @Path("tasks") public List getTaskMonitoring() { @@ -650,7 +643,7 @@ public String cachePeek(@QueryParam("name") String name, @QueryParam("limit") In boolean truncated = false; res.append("Cache: ").append(name).append('\n'); - res.append("Size: ").append(cache.size()).append('\n'); + res.append("Size: not calculated by cachePeek").append('\n'); res.append("Limit: ").append(actualLimit).append("\n\n"); for (Cache.Entry entry : cache) { @@ -696,50 +689,21 @@ private static String className(Object obj) { * @param name Cache name. */ private void ensureCanPeekCache(String name) { - if (!isUsersCache(name)) + if (cachePeekAllowedCaches().contains(name)) return; - ITcBotUserCreds creds = ITcBotUserCreds.get(req); - TcHelperUser user = creds == null ? null : instance(IUserStorage.class).getUser(creds.getPrincipalId()); - - if (!isUserAdmin(user, instance(ITcBotConfig.class))) - throw new ForbiddenException("Only user admin can inspect Users caches"); - } - - /** - * @param name Cache name. - */ - private static boolean isUsersCache(String name) { - return !Strings.isNullOrEmpty(name) && name.toLowerCase(Locale.ROOT).contains("users"); + throw new ForbiddenException("Cache peek is not allowed for cache: " + name + + ". Add the exact cache name to " + CACHE_PEEK_ALLOWED_CACHES + " to enable it."); } /** - * @param user User. - * @param cfg Config. + * @return Exact cache names allowed for raw preview. */ - private static boolean isUserAdmin(TcHelperUser user, ITcBotConfig cfg) { - return user != null && (user.isUserAdmin() || isConfigUserAdmin(user.username, cfg)); - } - - /** - * @param username Username. - * @param cfg Config. - */ - private static boolean isConfigUserAdmin(String username, ITcBotConfig cfg) { - if (Strings.isNullOrEmpty(username)) - return false; - - Collection cfgUserAdmins = cfg.userAdmins(); - - if (cfgUserAdmins == null) - return false; - - String normalized = username.toLowerCase(Locale.ROOT); - - return cfgUserAdmins.stream() - .filter(Objects::nonNull) - .map(s -> s.toLowerCase(Locale.ROOT)) - .anyMatch(normalized::equals); + private static Set cachePeekAllowedCaches() { + return Arrays.stream(Strings.nullToEmpty(System.getProperty(CACHE_PEEK_ALLOWED_CACHES)).split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toSet()); } /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java index 1f705f02b..04536f006 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java @@ -18,6 +18,7 @@ package org.apache.ignite.ci.web.rest.testfixes; import java.util.List; +import javax.annotation.security.RolesAllowed; import javax.servlet.ServletContext; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -26,6 +27,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.ci.web.auth.AuthenticationFilter; import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; @@ -56,6 +58,7 @@ public List recent(@QueryParam("limit") Integer limit) { * @param processId Optional user-visible process id. */ @GET + @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) @Path("refresh") public List refresh(@QueryParam("limit") Integer limit, @QueryParam("processId") Long processId) { TestFixesService svc = CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java index 02a99fb46..f36add13a 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java @@ -242,8 +242,11 @@ else if (sortOption == SortOption.SuiteDuration) { }; } - if (function != null) - contexts.sort(Comparator.comparing(function).reversed()); + if (function != null) { + contexts.sort(Comparator + .comparing(MultBuildRunCtx::onlyCancelledBuilds) + .thenComparing(Comparator.comparing(function).reversed())); + } fullChainRunCtx.addAllSuites(contexts); diff --git a/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java new file mode 100644 index 000000000..b6881dd35 --- /dev/null +++ b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.integration; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.regex.Pattern; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.tcbot.integration.IntegrationTestEnvironment.request; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Black-box coverage for test-fix matching and cancelled suite ordering. + */ +public class TestFixesAndCancelledOrderingFlowTest { + /** */ + private static final Pattern FINISHED_PROCESS = Pattern.compile("\"finished\"\\s*:\\s*[1-9][0-9]*"); + + /** */ + private static IntegrationTestEnvironment env; + + /** */ + @BeforeClass + public static void startSuite() throws Exception { + env = IntegrationTestEnvironment.get(); + } + + /** */ + @Before + public void resetEmulators() throws Exception { + env.resetEmulators(); + } + + /** */ + @Test + public void testFixRefreshReturnsOnlyOpenAndResolvedJiraTasksForMatchedTests() throws Exception { + String token = env.login(); + + createIssue("IGNITE-20101", + "Fix SqlRetryTest.testRetryOnTopologyChange flakiness", + "Open", + "The fix is being prepared for SqlRetryTest.testRetryOnTopologyChange.", + null); + createIssue("IGNITE-20102", + "Fix CacheRebalanceTest.testHistoricalRebalance instability", + "Resolved", + "Resolved test CacheRebalanceTest.testHistoricalRebalance after master history check.", + "2026-05-09T12:00:00.000+0000"); + + runTestOnlyAction(token, "refresh-jira", 710000001L); + runTestOnlyAction(token, "refresh-teamcity-builds", 710000002L); + + IntegrationTestEnvironment.HttpResponse master = request("GET", + env.botUrl + "/rest/tracked/results?branch=master", "Token " + token, null, null); + + assertEquals(master.body, 200, master.status); + assertTrue(master.body, master.body.contains("SqlRetryTest.testRetryOnTopologyChange")); + assertTrue(master.body, master.body.contains("CacheRebalanceTest.testHistoricalRebalance")); + + startTestFixRefresh(token, 710000003L); + waitForProcess(710000003L, token); + + String rows = waitForTestFixRows(token, "IGNITE-20101", "IGNITE-20102"); + + assertEquals(rows, 1, count(rows, "\"text\":\"IGNITE-20101\"")); + assertEquals(rows, 1, count(rows, "\"text\":\"IGNITE-20102\"")); + assertEquals(rows, 2, count(rows, "\"sourceType\":\"jira\"")); + assertEquals(rows, 0, count(rows, "\"sourceType\":\"github\"")); + assertTrue(rows, rows.contains("\"status\":\"Open\"")); + assertTrue(rows, rows.contains("\"status\":\"Resolved\"")); + assertTrue(rows, rows.contains("\"closedDate\":\"2026-05-09\"")); + } + + /** */ + @Test + public void cancelledSuitesAreAfterBuildFailureInPrReportAndVisaComment() throws Exception { + String token = env.login(); + + runTestOnlyAction(token, "refresh-jira", 710000011L); + runTestOnlyAction(token, "refresh-github", 710000012L); + runPrBuildRefsRefresh(token, 710000013L); + + IntegrationTestEnvironment.HttpResponse prResults = request("GET", env.botUrl + + "/rest/pr/results?serverId=apache" + + "&suiteId=" + enc("IgniteTests24Java17_RunAll") + + "&branchForTc=" + enc("pull/12007/head") + + "&action=" + enc("Latest"), + "Token " + token, null, null); + + assertEquals(prResults.body, 200, prResults.status); + assertBefore(prResults.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Cache1"); + assertBefore(prResults.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Sql"); + assertTrue(prResults.body, prResults.body.contains("CANCELLED")); + + String status = commentBuildAnalysis(token, 710000014L, "pull/12007/head", "IGNITE-20007", 12007); + + assertTrue(status, status.contains("JIRA ticket commented: IGNITE-20007")); + assertTrue(status, status.contains("GitHub PR commented: PR #12007")); + + IntegrationTestEnvironment.HttpResponse comments = request("GET", + env.githubUrl + "/repos/apache/ignite/issues/12007/comments", "Bearer github-test-token", null, null); + + assertEquals(comments.body, 200, comments.status); + assertBefore(comments.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Cache1"); + assertBefore(comments.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Sql"); + assertTrue(comments.body, comments.body.contains("CANCELLED")); + } + + /** */ + private static void createIssue(String key, String summary, String status, String description, + String resolutionDate) throws Exception { + String body = "{\"key\":\"" + key + "\"," + + "\"summary\":\"" + summary + "\"," + + "\"status\":\"" + status + "\"," + + "\"description\":\"" + description + "\"," + + "\"updated\":\"2026-05-10T10:00:00.000+0000\"" + + (resolutionDate == null ? "" : ",\"resolutiondate\":\"" + resolutionDate + "\"") + + "}"; + + IntegrationTestEnvironment.HttpResponse issue = request("POST", env.jiraUrl + "/__test__/jira/create-issue", + null, "application/json", body); + + assertEquals(issue.body, 201, issue.status); + } + + /** */ + private static void startTestFixRefresh(String token, long processId) throws Exception { + IntegrationTestEnvironment.HttpResponse refresh = request("GET", env.botUrl + + "/rest/testFixes/refresh?limit=0&processId=" + processId, + "Token " + token, null, null); + + assertEquals(refresh.body, 200, refresh.status); + } + + /** */ + private static String waitForTestFixRows(String token, String... expected) throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + String lastBody = ""; + + while (System.nanoTime() < deadline) { + IntegrationTestEnvironment.HttpResponse response = request("GET", env.botUrl + + "/rest/testFixes/recent?limit=0", "Token " + token, null, null); + lastBody = response.status + " " + response.body; + + boolean found = response.status == 200; + + for (String item : expected) + found &= response.body.contains(item); + + if (found) + return response.body; + + Thread.sleep(500); + } + + throw new IllegalStateException("Test fix rows were not found, last response: " + lastBody); + } + + /** */ + private static String runTestOnlyAction(String token, String action, long processId) throws Exception { + IntegrationTestEnvironment.HttpResponse start = request("POST", env.botUrl + + "/rest/__test__/bot/" + action + "?serverId=apache&processId=" + processId, + "Token " + token, null, null); + + assertEquals(start.body, 200, start.status); + assertTrue(start.body, start.body.contains("\"queued\":true")); + + return waitForProcess(processId, token); + } + + /** */ + private static String runPrBuildRefsRefresh(String token, long processId) throws Exception { + IntegrationTestEnvironment.HttpResponse start = request("POST", env.botUrl + + "/rest/pr/actualizeBuildRefs?serverId=apache&processId=" + processId, + "Token " + token, null, null); + + assertEquals(start.body, 200, start.status); + assertTrue(start.body, start.body.contains("TeamCity build refs refresh queued")); + + return waitForProcess(processId, token); + } + + /** */ + private static String commentBuildAnalysis(String token, long processId, String branch, String ticket, int prNum) + throws Exception { + IntegrationTestEnvironment.HttpResponse start = request("GET", env.botUrl + + "/rest/build/commentBuildAnalysis?serverId=apache" + + "&branchName=" + enc(branch) + + "&suiteId=" + enc("IgniteTests24Java17_RunAll") + + "&ticketId=" + enc(ticket) + + "&comment=" + enc("JIRA,GITHUB") + + "&prNum=" + prNum + + "&commentOnlyIfNoBlockers=false" + + "&processId=" + processId, + "Token " + token, null, null); + + assertEquals(start.body, 200, start.status); + assertTrue(start.body, start.body.contains("Comment process started")); + + return waitForProcess(processId, token); + } + + /** */ + private static String waitForProcess(long processId, String token) throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(90).toNanos(); + String url = env.botUrl + "/rest/process/status?id=" + processId; + String lastBody = ""; + + while (System.nanoTime() < deadline) { + IntegrationTestEnvironment.HttpResponse response = request("GET", url, "Token " + token, null, null); + lastBody = response.status + " " + response.body; + + if (response.status == 200 && FINISHED_PROCESS.matcher(response.body).find()) + return response.body; + + Thread.sleep(250); + } + + throw new IllegalStateException("Process did not finish: " + processId + ", last status: " + lastBody); + } + + /** */ + private static void assertBefore(String text, String first, String second) { + int firstIdx = text.indexOf(first); + int secondIdx = text.indexOf(second); + + assertTrue("Expected to find " + first + " in: " + text, firstIdx >= 0); + assertTrue("Expected to find " + second + " in: " + text, secondIdx >= 0); + assertTrue("Expected " + first + " before " + second + " in: " + text, firstIdx < secondIdx); + } + + /** */ + private static int count(String text, String needle) { + int res = 0; + int start = 0; + + while (start < text.length()) { + int idx = text.indexOf(needle, start); + + if (idx < 0) + return res; + + res++; + start = idx + needle.length(); + } + + return res; + } + + /** */ + private static String enc(String val) { + return URLEncoder.encode(val, StandardCharsets.UTF_8); + } +} diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java index f7c483784..23282f321 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java @@ -17,7 +17,10 @@ package org.apache.ignite.jiraignited; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.inject.Inject; @@ -46,6 +49,18 @@ public class JiraTicketSync { /** Logger. */ private static final Logger logger = LoggerFactory.getLogger(JiraTicketSync.class); + /** Source update signal cache name. Also used by test-fix matching. */ + private static final String SOURCE_UPDATES_CACHE_NAME = "testFixSourceUpdates"; + + /** Test-fix JIRA sync state cache name. */ + private static final String TEST_FIX_SYNC_STATE_CACHE_NAME = "jiraTestFixSyncState"; + + /** Labeled test-fix catch-up period. */ + private static final long TEST_FIX_LABEL_SYNC_PERIOD_MS = TimeUnit.DAYS.toMillis(1); + + /** Max age for labeled test-fix catch-up. */ + private static final int TEST_FIX_LABEL_LOOKBACK_DAYS = 365; + /** Scheduler. */ @Inject private IScheduler scheduler; @@ -106,14 +121,33 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { IJiraServerConfig cfg = jira.config(); String projectCode = cfg.projectCodeForVisa(); String baseUrl = searchUrl(cfg, "project=" + projectCode + " order by updated DESC", reqFields); - - SyncStats stats = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, baseUrl, fullResync); - SyncStats recent = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, - searchUrl(cfg, "project=" + projectCode + " AND updated >= -" + cfg.testFixesLookbackDays() + - "d order by updated DESC", reqFields), true); - SyncStats labeled = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, - searchUrl(cfg, "project=" + projectCode + " AND labels = " + cfg.testFixesLabel() + - " order by updated DESC", reqFields), true); + Set processedKeys = new HashSet<>(); + + SyncStats stats = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, baseUrl, fullResync, processedKeys); + SyncStats recent = new SyncStats(); + SyncStats labeled = new SyncStats(); + int recentDays = 0; + boolean labelSync = false; + + if (fullResync) + markTestFixJiraSyncComplete(srvCode); + else { + recentDays = recentLookbackDays(srvCode, cfg); + recent = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND updated >= -" + recentDays + + "d order by updated DESC", reqFields), true, processedKeys); + markTestFixRecentJiraSyncComplete(srvCode); + + labelSync = shouldSyncTestFixLabels(srvCode); + + if (labelSync) { + labeled = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND labels = " + cfg.testFixesLabel() + + " AND updated >= -" + TEST_FIX_LABEL_LOOKBACK_DAYS + "d order by updated DESC", reqFields), + true, processedKeys); + markTestFixLabelJiraSyncComplete(srvCode); + } + } int saved = stats.saved + recent.saved + labeled.saved; @@ -121,7 +155,9 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { signalSourceUpdate("jira:" + srvCode); return "Jira tickets saved " + saved + " from " + - (stats.processed + recent.processed + labeled.processed) + " checked for service " + srvCode; + (stats.processed + recent.processed + labeled.processed) + " checked for service " + srvCode + + " (recentDays=" + recentDays + ", labelSync=" + labelSync + ", duplicatesSkipped=" + + (stats.duplicatesSkipped + recent.duplicatesSkipped + labeled.duplicatesSkipped) + ")"; } /** @@ -129,11 +165,81 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { */ private void signalSourceUpdate(String key) { IgniteCache cache = igniteProvider.get().getOrCreateCache( - CacheConfigs.getCache8PartsConfig("testFixSourceUpdates")); + CacheConfigs.getCache8PartsConfig(SOURCE_UPDATES_CACHE_NAME)); cache.put(key, System.currentTimeMillis()); } + /** + * @param srvCode Server code. + * @param cfg JIRA config. + */ + private int recentLookbackDays(String srvCode, IJiraServerConfig cfg) { + Long lastSyncTs = testFixSyncStateCache().get(testFixRecentSyncKey(srvCode)); + int maxDays = Math.max(1, cfg.testFixesLookbackDays()); + + if (lastSyncTs == null) + return maxDays; + + long elapsedMs = Math.max(0, System.currentTimeMillis() - lastSyncTs); + int elapsedDays = (int)(elapsedMs / TimeUnit.DAYS.toMillis(1)) + 1; + + return Math.min(maxDays, Math.max(1, elapsedDays)); + } + + /** + * @param srvCode Server code. + */ + private boolean shouldSyncTestFixLabels(String srvCode) { + Long lastSyncTs = testFixSyncStateCache().get(testFixLabelSyncKey(srvCode)); + + return lastSyncTs == null || System.currentTimeMillis() - lastSyncTs >= TEST_FIX_LABEL_SYNC_PERIOD_MS; + } + + /** + * @param srvCode Server code. + */ + private void markTestFixJiraSyncComplete(String srvCode) { + long now = System.currentTimeMillis(); + IgniteCache cache = testFixSyncStateCache(); + + cache.put(testFixRecentSyncKey(srvCode), now); + cache.put(testFixLabelSyncKey(srvCode), now); + } + + /** + * @param srvCode Server code. + */ + private void markTestFixRecentJiraSyncComplete(String srvCode) { + testFixSyncStateCache().put(testFixRecentSyncKey(srvCode), System.currentTimeMillis()); + } + + /** + * @param srvCode Server code. + */ + private void markTestFixLabelJiraSyncComplete(String srvCode) { + testFixSyncStateCache().put(testFixLabelSyncKey(srvCode), System.currentTimeMillis()); + } + + /** */ + private IgniteCache testFixSyncStateCache() { + return igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(TEST_FIX_SYNC_STATE_CACHE_NAME)); + } + + /** + * @param srvCode Server code. + */ + private static String testFixRecentSyncKey(String srvCode) { + return "recent:" + srvCode; + } + + /** + * @param srvCode Server code. + */ + private static String testFixLabelSyncKey(String srvCode) { + return "label:" + srvCode; + } + /** * @param cfg JIRA config. * @param jql JQL query. @@ -152,7 +258,7 @@ private String searchUrl(IJiraServerConfig cfg, String jql, String reqFields) { * @param fullResync Full resync flag. */ private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int srvIdMaskHigh, String projectCode, - String baseUrl, boolean fullResync) { + String baseUrl, boolean fullResync, Set processedKeys) { String url = baseUrl; logger.info("Requesting JIRA tickets using URL " + url + ("\n" + cfg.restApiUrl() + url)); @@ -163,11 +269,11 @@ private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int sr return new SyncStats(); SyncStats res = new SyncStats(); - - int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); + Collection uniquePage = filterProcessedTickets(srvIdMaskHigh, page, projectCode, processedKeys, res); + int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, uniquePage, projectCode); res.saved += ticketsSaved; - res.processed += page.size(); + res.processed += uniquePage.size(); if (ticketsSaved != 0 || fullResync) { while (tickets.hasNextPage()) { @@ -181,10 +287,11 @@ private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int sr if (F.isEmpty(page)) break; - int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); + uniquePage = filterProcessedTickets(srvIdMaskHigh, page, projectCode, processedKeys, res); + int savedNow = jiraDao.saveChunk(srvIdMaskHigh, uniquePage, projectCode); res.saved += savedNow; - res.processed += page.size(); + res.processed += uniquePage.size(); if (savedNow == 0 && !fullResync) break; // find not updated chunk and exit @@ -194,6 +301,29 @@ private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int sr return res; } + /** + * @param srvIdMaskHigh Server id mask high. + * @param page Tickets page. + * @param projectCode Project code. + * @param processedKeys Already processed cache keys. + * @param stats Sync stats. + */ + private static Collection filterProcessedTickets(int srvIdMaskHigh, Collection page, + String projectCode, Set processedKeys, SyncStats stats) { + Collection res = new ArrayList<>(); + + for (Ticket ticket : page) { + long key = JiraTicketDao.ticketToCacheKey(srvIdMaskHigh, ticket.keyWithoutProject(projectCode)); + + if (processedKeys.add(key)) + res.add(ticket); + else + stats.duplicatesSkipped++; + } + + return res; + } + /** Sync stats. */ private static class SyncStats { /** Saved count. */ @@ -201,5 +331,8 @@ private static class SyncStats { /** Processed count. */ private int processed; + + /** Duplicate tickets skipped before save. */ + private int duplicatesSkipped; } } From 4352d8ad2f89ece3b5f05e17997ffc6c41cf2117 Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 21:23:56 +0300 Subject: [PATCH 3/6] Index test fix decoration lookups --- .../rest/monitoring/MonitoringService.java | 36 +++- .../tcbot/engine/conf/ITcBotConfig.java | 2 +- .../tcbot/engine/pr/PrChainsProcessor.java | 8 +- .../engine/testfixes/TestFixLookupIndex.java | 137 ++++++++++++++ .../tcbot/engine/testfixes/TestFixMatch.java | 3 + .../engine/testfixes/TestFixesService.java | 167 ++++++++++++++---- .../integrationTest/resources/branches.json | 2 +- 7 files changed, 311 insertions(+), 44 deletions(-) create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java index bae9e1521..d45418885 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java @@ -35,7 +35,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -125,6 +127,32 @@ public class MonitoringService { /** System property with comma-separated exact cache names allowed for raw preview. */ private static final String CACHE_PEEK_ALLOWED_CACHES = "tcbot.monitoring.cachePeek.allowedCaches"; + /** Built-in exact cache names allowed for raw preview. */ + private static final Set DFLT_CACHE_PEEK_ALLOWED_CACHES = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList( + "botDetectedDefects", + "botDetectedIssues", + "buildLogCheckResult", + "buildsConditions", + "compactVisasHistoryCacheV2", + "gitHubBranch", + "gitHubPr", + "jiraTestFixSyncState", + "mutedIssues", + "newTestsCache", + "teamcityBuildRef", + "teamcityBuildStartTime", + "teamcityBuildTypeRef", + "teamcityChange", + "teamcityFatBuild", + "teamcityFatBuildType", + "teamcityMute", + "teamcitySuiteHistory", + "testFixMatches", + "testFixMatchesV2", + "testFixSourceUpdates" + ))); + /** JSON mapper for raw cache entry values. */ private static final ObjectMapper CACHE_PEEK_MAPPER = new ObjectMapper() .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); @@ -700,10 +728,14 @@ private void ensureCanPeekCache(String name) { * @return Exact cache names allowed for raw preview. */ private static Set cachePeekAllowedCaches() { - return Arrays.stream(Strings.nullToEmpty(System.getProperty(CACHE_PEEK_ALLOWED_CACHES)).split(",")) + Set res = new HashSet<>(DFLT_CACHE_PEEK_ALLOWED_CACHES); + + Arrays.stream(Strings.nullToEmpty(System.getProperty(CACHE_PEEK_ALLOWED_CACHES)).split(",")) .map(String::trim) .filter(s -> !s.isEmpty()) - .collect(Collectors.toSet()); + .forEach(res::add); + + return res; } /** diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java index 8bbb26b9a..028cf9b4d 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java @@ -42,7 +42,7 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier { String DEFAULT_BOT_ADMIN_GROUP = "IGNITE_COMMITER"; /** Caches safe to reset from monitoring UI by default. */ - List DEFAULT_RESETTABLE_CACHES = List.of("testFixMatches", "testFixSourceUpdates"); + List DEFAULT_RESETTABLE_CACHES = List.of("testFixMatches", "testFixMatchesV2", "testFixSourceUpdates"); /** */ String primaryServerCode(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java index ca27a6564..ab1e21387 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java @@ -661,7 +661,7 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, Predicate filter = suite -> suite.isFailed() || suite.hasTestToReport(tcIgnited, baseBranchId, false, false); - return fullChainRunCtx + List suites = fullChainRunCtx .filteredChildSuites(filter) .map((ctx) -> { IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId, null); @@ -683,8 +683,6 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, .testShortFailures(failures) .initFrom(ctx, tcIgnited, compactor, statInBaseBranch); - testFixesService.decorate(suite); - return suite; } @@ -692,6 +690,10 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, }) .filter(Objects::nonNull) .collect(Collectors.toList()); + + testFixesService.decorate(suites); + + return suites; } /** diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java new file mode 100644 index 000000000..6e255f39b --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * In-memory lookup for decorating suites and test failures with known fix refs. + */ +class TestFixLookupIndex { + /** Max refs shown for a suite/test. */ + private static final int MAX_REFS = 5; + + /** Matches by normalized test/suite spellings. */ + private final Map> refsByName = new HashMap<>(); + + /** */ + public void add(String name, TestFixRefUi ref) { + if (Strings.isNullOrEmpty(name) || ref == null) + return; + + for (String key : lookupKeys(name)) + refsByName.computeIfAbsent(key, unused -> new ArrayList<>()).add(ref); + } + + /** */ + public void finish() { + for (Map.Entry> entry : refsByName.entrySet()) + entry.setValue(limit(uniqueRefs(entry.getValue()))); + } + + /** */ + public List find(String name) { + if (Strings.isNullOrEmpty(name)) + return new ArrayList<>(); + + List refs = new ArrayList<>(); + + for (String key : lookupKeys(name)) { + List found = refsByName.get(key); + + if (found != null) + refs.addAll(found); + } + + return limit(uniqueRefs(refs)); + } + + /** */ + private static List uniqueRefs(List refs) { + Map res = new LinkedHashMap<>(); + + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); + + return new ArrayList<>(res.values()); + } + + /** */ + private static List limit(List refs) { + if (refs.size() <= MAX_REFS) + return refs; + + return new ArrayList<>(refs.subList(0, MAX_REFS)); + } + + /** */ + private static Set lookupKeys(String name) { + Set res = new LinkedHashSet<>(); + + addKeys(res, name); + + int runCfgSep = name.indexOf("::"); + + if (runCfgSep >= 0) + addKeys(res, name.substring(runCfgSep + 2)); + + return res; + } + + /** */ + private static void addKeys(Set res, String name) { + String normalized = normalize(name); + + if (normalized.isEmpty()) + return; + + res.add(normalized); + + String dotForm = normalized.replace('#', '.'); + + res.add(dotForm); + + String classAndMethod = classAndMethod(dotForm); + + if (!classAndMethod.isEmpty()) + res.add(classAndMethod); + } + + /** */ + private static String classAndMethod(String name) { + String[] parts = name.split("\\."); + + if (parts.length < 2) + return ""; + + return parts[parts.length - 2] + "." + parts[parts.length - 1]; + } + + /** */ + private static String normalize(String name) { + return Strings.nullToEmpty(name).trim().toLowerCase(Locale.ROOT); + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java index 791915376..bd3859c40 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java @@ -18,6 +18,7 @@ package org.apache.ignite.tcbot.engine.testfixes; import javax.annotation.Nullable; +import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.tcbot.persistence.IVersionedEntity; import org.apache.ignite.tcbot.persistence.Persisted; @@ -51,9 +52,11 @@ public class TestFixMatch implements IVersionedEntity { @Nullable public String currentStatusUrl; /** Source type: jira or github. */ + @QuerySqlField(index = true, orderedGroups = {@QuerySqlField.Group(name = "source", order = 0)}) public String sourceType; /** Source id. */ + @QuerySqlField(index = true, orderedGroups = {@QuerySqlField.Group(name = "source", order = 1)}) public String sourceId; /** Source URL. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java index eb78893dd..2d8d82e91 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java @@ -50,6 +50,7 @@ import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.githubignited.IGitHubConnIgnited; import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.jiraignited.IJiraIgnited; import org.apache.ignite.jiraignited.IJiraIgnitedProvider; import org.apache.ignite.jiraservice.JiraTicketStatusCode; @@ -84,7 +85,7 @@ public class TestFixesService { private static final Logger logger = LoggerFactory.getLogger(TestFixesService.class); /** Cache name. */ - private static final String CACHE_NAME = "testFixMatches"; + private static final String CACHE_NAME = "testFixMatchesV2"; /** Background scheduled task name. */ private static final String TASK_NAME = TestFixesService.class.getSimpleName() + ".refresh"; @@ -125,6 +126,9 @@ public class TestFixesService { /** Source update signals cache. */ private volatile IgniteCache signalCache; + /** Lookup index built from match cache for UI decoration. */ + private volatile TestFixLookupIndex lookupIndex; + /** Last source update signal timestamp. */ private volatile long lastSignalTs; @@ -169,10 +173,36 @@ public void decorate(ShortSuiteUi suite) { if (suite == null) return; - suite.fixRefs = findRefs(suite.name); + TestFixLookupIndex idx = lookupIndex(); + + decorate(suite, idx); + } + + /** + * @param suites Suites UI. + */ + public void decorate(Collection suites) { + if (suites == null || suites.isEmpty()) + return; + + TestFixLookupIndex idx = lookupIndex(); + + for (ShortSuiteUi suite : suites) + decorate(suite, idx); + } + + /** + * @param suite Suite UI. + * @param idx Test fix lookup index. + */ + private void decorate(ShortSuiteUi suite, TestFixLookupIndex idx) { + if (suite == null) + return; + + suite.fixRefs = findRefs(suite.name, idx); for (ShortTestFailureUi failure : suite.testFailures()) - decorate(failure); + decorate(failure, idx); } /** @@ -182,10 +212,21 @@ public void decorate(ShortTestFailureUi failure) { if (failure == null) return; + decorate(failure, lookupIndex()); + } + + /** + * @param failure Test failure UI. + * @param idx Test fix lookup index. + */ + private void decorate(ShortTestFailureUi failure, TestFixLookupIndex idx) { + if (failure == null) + return; + List refs = new ArrayList<>(); - refs.addAll(findRefs(failure.testName)); - refs.addAll(findRefs(failure.name)); + refs.addAll(findRefs(failure.testName, idx)); + refs.addAll(findRefs(failure.name, idx)); failure.fixRefs = uniqueRefs(refs); } @@ -228,28 +269,33 @@ public String refresh(@Nullable Long processId) { RefreshStats stats = new RefreshStats(); - publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); - Map> candidatesByServer = candidatesByServer(); - stats.collectCandidateStats(candidatesByServer); - publishRefreshStatus(processId, "collected test names from tracked suite history", stats); + try { + publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); + Map> candidatesByServer = candidatesByServer(); + stats.collectCandidateStats(candidatesByServer); + publishRefreshStatus(processId, "collected test names from tracked suite history", stats); - for (String srvCode : serverCodes()) { - List srvCandidates = candidatesByServer.get(srvCode); + for (String srvCode : serverCodes()) { + List srvCandidates = candidatesByServer.get(srvCode); - if (srvCandidates == null || srvCandidates.isEmpty()) - continue; + if (srvCandidates == null || srvCandidates.isEmpty()) + continue; - publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); - List recentPrs = recentPullRequests(srvCode); - stats.githubLoaded += recentPrs.size(); - publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + - srvCode, stats); + publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); + List recentPrs = recentPullRequests(srvCode); + stats.githubLoaded += recentPrs.size(); + publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + + srvCode, stats); - refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); - refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); - } + refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); + refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); + } - return stats.finishText(); + return stats.finishText(); + } + finally { + invalidateLookupIndex(); + } } /** @@ -530,22 +576,18 @@ private List findRefs(@Nullable String name) { if (Strings.isNullOrEmpty(name)) return new ArrayList<>(); - String exact = normalize(name); - - ensureCache(); + return findRefs(name, lookupIndex()); + } - List res = StreamSupport.stream(cache.spliterator(), false) - .map(Cache.Entry::getValue) - .filter(TestFixesService::isPlausibleMatch) - .filter(match -> exact.equals(normalize(match.entityName)) || - exact.equals(normalize(match.testName)) || - exact.equals(normalize(match.suiteName))) - .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()) - .limit(5) - .map(this::toUi) - .collect(Collectors.toList()); + /** + * @param name Entity name. + * @param idx Test fix lookup index. + */ + private List findRefs(@Nullable String name, TestFixLookupIndex idx) { + if (Strings.isNullOrEmpty(name)) + return new ArrayList<>(); - return uniqueRefs(res); + return idx.find(name); } /** @@ -610,6 +652,53 @@ private static String cacheKey(TestFixMatch match) { normalize(match.suiteId) + ":" + normalize(match.entityName); } + /** */ + private void invalidateLookupIndex() { + lookupIndex = null; + } + + /** */ + private TestFixLookupIndex lookupIndex() { + TestFixLookupIndex idx = lookupIndex; + + if (idx != null) + return idx; + + ensureCache(); + + synchronized (this) { + idx = lookupIndex; + + if (idx == null) { + idx = buildLookupIndex(); + lookupIndex = idx; + } + + return idx; + } + } + + /** */ + private TestFixLookupIndex buildLookupIndex() { + TestFixLookupIndex idx = new TestFixLookupIndex(); + + StreamSupport.stream(cache.spliterator(), false) + .map(Cache.Entry::getValue) + .filter(TestFixesService::isPlausibleMatch) + .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()) + .forEach(match -> { + TestFixRefUi ref = toUi(match); + + idx.add(match.entityName, ref); + idx.add(match.testName, ref); + idx.add(match.suiteName, ref); + }); + + idx.finish(); + + return idx; + } + /** * @param value Value. */ @@ -1011,7 +1100,11 @@ private void ensureCache() { synchronized (this) { if (cache == null) { - cache = igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(CACHE_NAME)); + CacheConfiguration cacheCfg = CacheConfigs.getCache8PartsConfig(CACHE_NAME); + + cacheCfg.setIndexedTypes(String.class, TestFixMatch.class); + + cache = igniteProvider.get().getOrCreateCache(cacheCfg); signalCache = igniteProvider.get().getOrCreateCache( CacheConfigs.getCache8PartsConfig(SOURCE_UPDATES_CACHE_NAME)); } diff --git a/tcbot-integration-tests/src/integrationTest/resources/branches.json b/tcbot-integration-tests/src/integrationTest/resources/branches.json index 93ea9aaf0..92b7bb1c9 100644 --- a/tcbot-integration-tests/src/integrationTest/resources/branches.json +++ b/tcbot-integration-tests/src/integrationTest/resources/branches.json @@ -1,7 +1,7 @@ { "primaryServerCode": "apache", "botAdminGroups": ["IGNITE_COMMITTER"], - "resettableCaches": ["testFixMatches", "testFixSourceUpdates"], + "resettableCaches": ["testFixMatches", "testFixMatchesV2", "testFixSourceUpdates"], "confidence": 0.995, "tcServers": [ { From 0cfbae15176cf8737575213463ecd4bc73ec8cfb Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 21:39:56 +0300 Subject: [PATCH 4/6] Use reverse cache for test fix lookups --- conf/branches.json | 9 +- .../rest/monitoring/MonitoringService.java | 5 +- .../tcbot/engine/chain/TestCompactedMult.java | 4 + .../tcbot/engine/conf/ITcBotConfig.java | 10 +- .../engine/testfixes/TestFixLookupIndex.java | 108 ++--- .../tcbot/engine/testfixes/TestFixMatch.java | 6 +- .../tcbot/engine/testfixes/TestFixRefs.java | 70 +++ .../tcbot/engine/testfixes/TestFixSource.java | 80 ++++ .../engine/testfixes/TestFixesService.java | 449 +++++++++++------- .../ignite/tcbot/engine/ui/ShortSuiteUi.java | 4 + .../tcbot/engine/ui/ShortTestFailureUi.java | 8 + .../testfixes/TestFixesServiceTest.java | 2 +- .../githubignited/GitHubConnIgnitedImpl.java | 13 - .../integrationTest/resources/branches.json | 9 +- .../ignite/jiraignited/JiraTicketSync.java | 16 - 15 files changed, 510 insertions(+), 283 deletions(-) create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java create mode 100644 tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java diff --git a/conf/branches.json b/conf/branches.json index efbc5da47..67a48fe0b 100644 --- a/conf/branches.json +++ b/conf/branches.json @@ -5,7 +5,14 @@ // Apache Ignite test admins are exposed by TeamCity REST as key="IGNITE_COMMITER". "botAdminGroups": ["IGNITE_COMMITER"], // Ignite cache names admins may reset from monitoring UI. - "resettableCaches": ["testFixMatches", "testFixSourceUpdates"], + "resettableCaches": [ + "testFixRefsByTest", + "testFixSourcesById", + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ], "confidence": 0.995, "cleanerConfig": { "numOfItemsToDel": 100000, diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java index d45418885..6c061de9c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java @@ -148,9 +148,8 @@ public class MonitoringService { "teamcityFatBuildType", "teamcityMute", "teamcitySuiteHistory", - "testFixMatches", - "testFixMatchesV2", - "testFixSourceUpdates" + "testFixRefsByTest", + "testFixSourcesById" ))); /** JSON mapper for raw cache entry values. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java index 199bdeaf0..1da0f822d 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java @@ -60,6 +60,10 @@ public static void resetCached() { @Nullable public Integer testName() { return occurrences.isEmpty() ? null : occurrences.iterator().next().testName(); } + + @Nullable public String suiteId() { + return ctx == null ? null : ctx.suiteId(); + } public String getName() { return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java index 028cf9b4d..413b6c8f6 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java @@ -42,7 +42,15 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier { String DEFAULT_BOT_ADMIN_GROUP = "IGNITE_COMMITER"; /** Caches safe to reset from monitoring UI by default. */ - List DEFAULT_RESETTABLE_CACHES = List.of("testFixMatches", "testFixMatchesV2", "testFixSourceUpdates"); + List DEFAULT_RESETTABLE_CACHES = List.of( + "testFixRefsByTest", + "testFixSourcesById", + // Old test-fix cache names may still be restored from persistence after cache layout changes. + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ); /** */ String primaryServerCode(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java index 6e255f39b..c74d69733 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java @@ -21,117 +21,93 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Set; +import javax.annotation.Nullable; /** - * In-memory lookup for decorating suites and test failures with known fix refs. + * Response-local lookup for decorating suites and test failures with known fix refs. */ class TestFixLookupIndex { + /** Suite-level lookup entity. */ + private static final String SUITE_ENTITY = "suite"; + /** Max refs shown for a suite/test. */ private static final int MAX_REFS = 5; - /** Matches by normalized test/suite spellings. */ - private final Map> refsByName = new HashMap<>(); + /** Matches by suite id and compacted test id lookup key. */ + private final Map> refsByKey = new HashMap<>(); /** */ - public void add(String name, TestFixRefUi ref) { - if (Strings.isNullOrEmpty(name) || ref == null) + public void add(String lookupKey, List refs) { + if (Strings.isNullOrEmpty(lookupKey) || refs == null || refs.isEmpty()) return; - for (String key : lookupKeys(name)) - refsByName.computeIfAbsent(key, unused -> new ArrayList<>()).add(ref); + refsByKey.computeIfAbsent(lookupKey, unused -> new ArrayList<>()).addAll(refs); } /** */ public void finish() { - for (Map.Entry> entry : refsByName.entrySet()) + for (Map.Entry> entry : refsByKey.entrySet()) entry.setValue(limit(uniqueRefs(entry.getValue()))); } /** */ - public List find(String name) { - if (Strings.isNullOrEmpty(name)) - return new ArrayList<>(); - - List refs = new ArrayList<>(); - - for (String key : lookupKeys(name)) { - List found = refsByName.get(key); - - if (found != null) - refs.addAll(found); - } - - return limit(uniqueRefs(refs)); + public List findSuite(@Nullable String suiteId) { + return find(suiteLookupKey(suiteId)); } /** */ - private static List uniqueRefs(List refs) { - Map res = new LinkedHashMap<>(); - - for (TestFixRefUi ref : refs) - res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); - - return new ArrayList<>(res.values()); + public List findTest(@Nullable String suiteId, @Nullable Integer testNameId) { + return find(testLookupKey(suiteId, testNameId)); } /** */ - private static List limit(List refs) { - if (refs.size() <= MAX_REFS) - return refs; - - return new ArrayList<>(refs.subList(0, MAX_REFS)); + static String suiteLookupKey(@Nullable String suiteId) { + return lookupKey(suiteId, SUITE_ENTITY); } /** */ - private static Set lookupKeys(String name) { - Set res = new LinkedHashSet<>(); - - addKeys(res, name); - - int runCfgSep = name.indexOf("::"); - - if (runCfgSep >= 0) - addKeys(res, name.substring(runCfgSep + 2)); + static String testLookupKey(@Nullable String suiteId, @Nullable Integer testNameId) { + if (testNameId == null) + return ""; - return res; + return lookupKey(suiteId, "test:" + testNameId); } /** */ - private static void addKeys(Set res, String name) { - String normalized = normalize(name); - - if (normalized.isEmpty()) - return; - - res.add(normalized); + private List find(String lookupKey) { + if (Strings.isNullOrEmpty(lookupKey)) + return new ArrayList<>(); - String dotForm = normalized.replace('#', '.'); + List found = refsByKey.get(lookupKey); - res.add(dotForm); + return found == null ? new ArrayList<>() : new ArrayList<>(found); + } - String classAndMethod = classAndMethod(dotForm); + /** */ + private static String lookupKey(@Nullable String suiteId, String entityKey) { + if (Strings.isNullOrEmpty(suiteId)) + return ""; - if (!classAndMethod.isEmpty()) - res.add(classAndMethod); + return suiteId + "::" + entityKey; } /** */ - private static String classAndMethod(String name) { - String[] parts = name.split("\\."); + private static List uniqueRefs(List refs) { + Map res = new LinkedHashMap<>(); - if (parts.length < 2) - return ""; + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); - return parts[parts.length - 2] + "." + parts[parts.length - 1]; + return new ArrayList<>(res.values()); } /** */ - private static String normalize(String name) { - return Strings.nullToEmpty(name).trim().toLowerCase(Locale.ROOT); + private static List limit(List refs) { + if (refs.size() <= MAX_REFS) + return refs; + + return new ArrayList<>(refs.subList(0, MAX_REFS)); } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java index bd3859c40..56410579b 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java @@ -18,7 +18,6 @@ package org.apache.ignite.tcbot.engine.testfixes; import javax.annotation.Nullable; -import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.tcbot.persistence.IVersionedEntity; import org.apache.ignite.tcbot.persistence.Persisted; @@ -45,6 +44,9 @@ public class TestFixMatch implements IVersionedEntity { /** Matched test name, when known. */ @Nullable public String testName; + /** Compacted test name id, when known. */ + @Nullable public Integer testNameId; + /** Tracked branch used to resolve this match. */ @Nullable public String trackedBranch; @@ -52,11 +54,9 @@ public class TestFixMatch implements IVersionedEntity { @Nullable public String currentStatusUrl; /** Source type: jira or github. */ - @QuerySqlField(index = true, orderedGroups = {@QuerySqlField.Group(name = "source", order = 0)}) public String sourceType; /** Source id. */ - @QuerySqlField(index = true, orderedGroups = {@QuerySqlField.Group(name = "source", order = 1)}) public String sourceId; /** Source URL. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java new file mode 100644 index 000000000..07d12caea --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Reverse lookup from a suite/test spelling to source fix ids. + */ +@Persisted +public class TestFixRefs implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Matched test or suite name. */ + public String entityName; + + /** Matched suite name, when known. */ + @Nullable public String suiteName; + + /** Matched suite id, when known. */ + @Nullable public String suiteId; + + /** Matched test name, when known. */ + @Nullable public String testName; + + /** Compacted test name id, when known. */ + @Nullable public Integer testNameId; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current master TeamCity suite status URL. */ + @Nullable public String currentStatusUrl; + + /** Synthetic source ids. */ + public List sourceIds = new ArrayList<>(); + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java new file mode 100644 index 000000000..44cdc561e --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Persisted fix source data keyed by synthetic source id. + */ +@Persisted +public class TestFixSource implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Source type: jira or github. */ + public String sourceType; + + /** Source id. */ + public String sourceId; + + /** Source URL. */ + public String sourceUrl; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated timestamp. */ + @Nullable public Long updatedTs; + + /** Closed/resolved timestamp. */ + @Nullable public Long closedTs; + + /** Commit SHA, when known. */ + @Nullable public String commitSha; + + /** Commit URL, when known. */ + @Nullable public String commitUrl; + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java index 2d8d82e91..989c8342e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java @@ -19,12 +19,14 @@ import com.google.common.base.Strings; import java.time.Instant; +import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -50,7 +52,6 @@ import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; import org.apache.ignite.githubignited.IGitHubConnIgnited; import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.jiraignited.IJiraIgnited; import org.apache.ignite.jiraignited.IJiraIgnitedProvider; import org.apache.ignite.jiraservice.JiraTicketStatusCode; @@ -84,15 +85,15 @@ public class TestFixesService { /** Logger. */ private static final Logger logger = LoggerFactory.getLogger(TestFixesService.class); - /** Cache name. */ - private static final String CACHE_NAME = "testFixMatchesV2"; + /** Reverse lookup cache name. */ + private static final String REFS_CACHE_NAME = "testFixRefsByTest"; + + /** Source fix cache name. */ + private static final String SOURCE_CACHE_NAME = "testFixSourcesById"; /** Background scheduled task name. */ private static final String TASK_NAME = TestFixesService.class.getSimpleName() + ".refresh"; - /** Source update signal cache name. Also used by JIRA/GitHub sync modules. */ - private static final String SOURCE_UPDATES_CACHE_NAME = "testFixSourceUpdates"; - /** Ignite provider. */ @Inject private Provider igniteProvider; @@ -120,17 +121,11 @@ public class TestFixesService { /** String compactor. */ @Inject private IStringCompactor compactor; - /** Cache. */ - private volatile IgniteCache cache; - - /** Source update signals cache. */ - private volatile IgniteCache signalCache; + /** Source fix cache keyed by synthetic source id. */ + private volatile IgniteCache sourceCache; - /** Lookup index built from match cache for UI decoration. */ - private volatile TestFixLookupIndex lookupIndex; - - /** Last source update signal timestamp. */ - private volatile long lastSignalTs; + /** Reverse lookup cache keyed by suite/test spelling. */ + private volatile IgniteCache lookupCache; /** * Starts rare background refresh. @@ -139,7 +134,6 @@ public void start() { maintenanceActions.register(TASK_NAME, "Refresh JIRA/GitHub test-fix mapping", this::refresh); scheduler.invokeLater(this::refreshAndReschedule, 2, TimeUnit.MINUTES); - scheduler.invokeLater(this::watchSourceUpdates, 3, TimeUnit.MINUTES); } /** @@ -173,9 +167,7 @@ public void decorate(ShortSuiteUi suite) { if (suite == null) return; - TestFixLookupIndex idx = lookupIndex(); - - decorate(suite, idx); + decorate(Collections.singletonList(suite)); } /** @@ -185,7 +177,7 @@ public void decorate(Collection suites) { if (suites == null || suites.isEmpty()) return; - TestFixLookupIndex idx = lookupIndex(); + TestFixLookupIndex idx = lookupIndex(suites); for (ShortSuiteUi suite : suites) decorate(suite, idx); @@ -199,10 +191,10 @@ private void decorate(ShortSuiteUi suite, TestFixLookupIndex idx) { if (suite == null) return; - suite.fixRefs = findRefs(suite.name, idx); + suite.fixRefs = idx.findSuite(suite.suiteId); for (ShortTestFailureUi failure : suite.testFailures()) - decorate(failure, idx); + decorate(failure, suite.suiteId, idx); } /** @@ -212,23 +204,23 @@ public void decorate(ShortTestFailureUi failure) { if (failure == null) return; - decorate(failure, lookupIndex()); + Set keys = new LinkedHashSet<>(); + + addLookupKey(keys, TestFixLookupIndex.testLookupKey(failure.suiteId, failure.testNameId)); + + decorate(failure, failure.suiteId, lookupIndex(keys)); } /** * @param failure Test failure UI. + * @param suiteId Suite id. * @param idx Test fix lookup index. */ - private void decorate(ShortTestFailureUi failure, TestFixLookupIndex idx) { + private void decorate(ShortTestFailureUi failure, @Nullable String suiteId, TestFixLookupIndex idx) { if (failure == null) return; - List refs = new ArrayList<>(); - - refs.addAll(findRefs(failure.testName, idx)); - refs.addAll(findRefs(failure.name, idx)); - - failure.fixRefs = uniqueRefs(refs); + failure.fixRefs = idx.findTest(suiteId, failure.testNameId); } /** @@ -237,17 +229,20 @@ private void decorate(ShortTestFailureUi failure, TestFixLookupIndex idx) { public List recent(int limit) { ensureCache(); - java.util.stream.Stream stream = StreamSupport.stream(cache.spliterator(), false) + List mappings = StreamSupport.stream(lookupCache.spliterator(), false) .map(Cache.Entry::getValue) - .filter(TestFixesService::isPlausibleMatch) - .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()); + .collect(Collectors.toList()); + + Map sources = sources(sourceIds(mappings)); + + java.util.stream.Stream stream = mappings.stream() + .flatMap(mapping -> refsFor(mapping, sources).stream()) + .sorted(Comparator.comparingLong((TestFixRefUi ref) -> sortTs(ref)).reversed()); if (limit > 0) stream = stream.limit(limit); - return stream - .map(this::toUi) - .collect(Collectors.toList()); + return uniqueRefs(stream.collect(Collectors.toList())); } /** @@ -269,33 +264,28 @@ public String refresh(@Nullable Long processId) { RefreshStats stats = new RefreshStats(); - try { - publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); - Map> candidatesByServer = candidatesByServer(); - stats.collectCandidateStats(candidatesByServer); - publishRefreshStatus(processId, "collected test names from tracked suite history", stats); - - for (String srvCode : serverCodes()) { - List srvCandidates = candidatesByServer.get(srvCode); + publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); + Map> candidatesByServer = candidatesByServer(); + stats.collectCandidateStats(candidatesByServer); + publishRefreshStatus(processId, "collected test names from tracked suite history", stats); - if (srvCandidates == null || srvCandidates.isEmpty()) - continue; + for (String srvCode : serverCodes()) { + List srvCandidates = candidatesByServer.get(srvCode); - publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); - List recentPrs = recentPullRequests(srvCode); - stats.githubLoaded += recentPrs.size(); - publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + - srvCode, stats); + if (srvCandidates == null || srvCandidates.isEmpty()) + continue; - refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); - refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); - } + publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); + List recentPrs = recentPullRequests(srvCode); + stats.githubLoaded += recentPrs.size(); + publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + + srvCode, stats); - return stats.finishText(); - } - finally { - invalidateLookupIndex(); + refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); + refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); } + + return stats.finishText(); } /** @@ -306,25 +296,6 @@ private void refreshAndReschedule() { scheduler.sheduleNamed(TASK_NAME, this::refreshAndReschedule, 6, TimeUnit.HOURS); } - /** - * Watches cheap update signals from JIRA/GitHub cache refreshes. - */ - private void watchSourceUpdates() { - try { - long signalTs = latestSignalTs(); - - if (signalTs > lastSignalTs) { - lastSignalTs = signalTs; - requestMatchSoon(); - } - } - catch (RuntimeException e) { - logger.debug("Failed to watch test fix update signals", e); - } - - scheduler.sheduleNamed(TASK_NAME + ".watchUpdates", this::watchSourceUpdates, 5, TimeUnit.MINUTES); - } - /** * Safe refresh wrapper for background tasks. */ @@ -394,13 +365,13 @@ private void refreshJira(String srvCode, List candidates, List match.updatedTs = updatedTs; match.closedTs = closedTs; - cache.put(cacheKey(match), match); + saveMatch(match); stats.jiraSaved++; for (PullRequest pr : linkedPrs) { TestFixMatch linkedPrMatch = githubMatch(candidate, pr); - cache.put(cacheKey(linkedPrMatch), linkedPrMatch); + saveMatch(linkedPrMatch); stats.githubSaved++; } } @@ -433,7 +404,7 @@ private void refreshGithub(String srvCode, List candidates, Li for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { TestFixMatch match = githubMatch(candidate, pr); - cache.put(cacheKey(match), match); + saveMatch(match); stats.githubSaved++; } } @@ -570,133 +541,241 @@ private static void fillCommit(TestFixMatch match, PullRequest pr) { } /** - * @param name Entity name. + * @param refs Refs. */ - private List findRefs(@Nullable String name) { - if (Strings.isNullOrEmpty(name)) - return new ArrayList<>(); + private static List uniqueRefs(List refs) { + Map res = new HashMap<>(); + + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); - return findRefs(name, lookupIndex()); + return new ArrayList<>(res.values()); } /** - * @param name Entity name. - * @param idx Test fix lookup index. + * @param refs Reverse refs. + * @param source Source. */ - private List findRefs(@Nullable String name, TestFixLookupIndex idx) { - if (Strings.isNullOrEmpty(name)) - return new ArrayList<>(); + private TestFixRefUi toUi(TestFixRefs refs, TestFixSource source) { + TestFixRefUi res = new TestFixRefUi(); + + res.entityName = refs.entityName; + res.suiteId = refs.suiteId; + res.suiteName = refs.suiteName; + res.testName = refs.testName; + res.trackedBranch = refs.trackedBranch; + res.currentStatusUrl = refs.currentStatusUrl; + res.sourceType = source.sourceType; + res.text = source.sourceId; + res.url = source.sourceUrl; + res.title = source.title; + res.status = source.status; + res.author = source.author; + res.authorUrl = source.authorUrl; + res.authorAvatarUrl = source.authorAvatarUrl; + res.updatedDate = formatDate(source.updatedTs); + res.closedDate = formatDate(source.closedTs); + res.commitUrl = source.commitUrl; + res.commitText = Strings.isNullOrEmpty(source.commitSha) ? null : shortSha(source.commitSha); - return idx.find(name); + return res; } /** - * @param refs Refs. + * @param status Status. */ - private static List uniqueRefs(List refs) { - Map res = new HashMap<>(); + private static String statusText(@Nullable JiraTicketStatusCode status) { + return JiraTicketStatusCode.text(status); + } - for (TestFixRefUi ref : refs) - res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); + /** + * @param sha SHA. + */ + private static String shortSha(String sha) { + return sha.length() <= PullRequest.INCLUDE_SHORT_VER ? sha : sha.substring(0, PullRequest.INCLUDE_SHORT_VER); + } - return new ArrayList<>(res.values()); + /** + * @param match Match. + */ + private void saveMatch(TestFixMatch match) { + if (!isPlausibleMatch(match)) + return; + + String sourceId = sourceKey(match); + + sourceCache.put(sourceId, source(match)); + + for (String key : lookupKeys(match)) { + TestFixRefs refs = lookupCache.get(key); + + if (refs == null) + refs = refs(match); + + if (!refs.sourceIds.contains(sourceId)) + refs.sourceIds.add(sourceId); + + lookupCache.put(key, refs); + } } /** * @param match Match. */ - private TestFixRefUi toUi(TestFixMatch match) { - TestFixRefUi res = new TestFixRefUi(); + private static TestFixSource source(TestFixMatch match) { + TestFixSource res = new TestFixSource(); - res.entityName = match.entityName; - res.suiteId = match.suiteId; - res.suiteName = match.suiteName; - res.testName = match.testName; - res.trackedBranch = match.trackedBranch; - res.currentStatusUrl = match.currentStatusUrl; res.sourceType = match.sourceType; - res.text = match.sourceId; - res.url = match.sourceUrl; + res.sourceId = match.sourceId; + res.sourceUrl = match.sourceUrl; res.title = match.title; res.status = match.status; res.author = match.author; res.authorUrl = match.authorUrl; res.authorAvatarUrl = match.authorAvatarUrl; - res.updatedDate = formatDate(match.updatedTs); - res.closedDate = formatDate(match.closedTs); + res.updatedTs = match.updatedTs; + res.closedTs = match.closedTs; + res.commitSha = match.commitSha; res.commitUrl = match.commitUrl; - res.commitText = Strings.isNullOrEmpty(match.commitSha) ? null : shortSha(match.commitSha); return res; } /** - * @param status Status. + * @param match Match. */ - private static String statusText(@Nullable JiraTicketStatusCode status) { - return JiraTicketStatusCode.text(status); + private static TestFixRefs refs(TestFixMatch match) { + TestFixRefs res = new TestFixRefs(); + + res.entityName = match.entityName; + res.suiteId = match.suiteId; + res.suiteName = match.suiteName; + res.testName = match.testName; + res.testNameId = match.testNameId; + res.trackedBranch = match.trackedBranch; + res.currentStatusUrl = match.currentStatusUrl; + + return res; } /** - * @param sha SHA. + * @param match Match. */ - private static String shortSha(String sha) { - return sha.length() <= PullRequest.INCLUDE_SHORT_VER ? sha : sha.substring(0, PullRequest.INCLUDE_SHORT_VER); + private static String sourceKey(TestFixMatch match) { + return normalize(match.sourceType) + ":" + normalize(match.sourceId); } /** * @param match Match. */ - private static String cacheKey(TestFixMatch match) { - return match.sourceType + ":" + match.sourceId + ":" + normalize(match.trackedBranch) + ":" + - normalize(match.suiteId) + ":" + normalize(match.entityName); + private static Set lookupKeys(TestFixMatch match) { + Set res = new LinkedHashSet<>(); + + addLookupKey(res, TestFixLookupIndex.suiteLookupKey(match.suiteId)); + addLookupKey(res, TestFixLookupIndex.testLookupKey(match.suiteId, match.testNameId)); + + return res; } - /** */ - private void invalidateLookupIndex() { - lookupIndex = null; + /** + * @param res Target keys. + * @param key Lookup key. + */ + private static void addLookupKey(Set res, String key) { + if (!Strings.isNullOrEmpty(key)) + res.add(key); } - /** */ - private TestFixLookupIndex lookupIndex() { - TestFixLookupIndex idx = lookupIndex; + /** + * @param suites Suites UI. + */ + private TestFixLookupIndex lookupIndex(Collection suites) { + Set keys = new LinkedHashSet<>(); - if (idx != null) - return idx; + for (ShortSuiteUi suite : suites) + collectLookupKeys(suite, keys); - ensureCache(); + return lookupIndex(keys); + } - synchronized (this) { - idx = lookupIndex; + /** + * @param lookupKeys Requested lookup keys. + */ + private TestFixLookupIndex lookupIndex(Set lookupKeys) { + ensureCache(); - if (idx == null) { - idx = buildLookupIndex(); - lookupIndex = idx; - } + TestFixLookupIndex idx = new TestFixLookupIndex(); + if (lookupKeys.isEmpty()) return idx; + + Map mappings = lookupCache.getAll(lookupKeys); + Map sources = sources(sourceIds(mappings.values())); + + for (Map.Entry entry : mappings.entrySet()) + idx.add(entry.getKey(), refsFor(entry.getValue(), sources)); + + idx.finish(); + + return idx; + } + + /** + * @param suite Suite UI. + * @param keys Target keys. + */ + private static void collectLookupKeys(ShortSuiteUi suite, Set keys) { + if (suite == null) + return; + + addLookupKey(keys, TestFixLookupIndex.suiteLookupKey(suite.suiteId)); + + for (ShortTestFailureUi failure : suite.testFailures()) { + addLookupKey(keys, TestFixLookupIndex.testLookupKey(suite.suiteId, failure.testNameId)); } } - /** */ - private TestFixLookupIndex buildLookupIndex() { - TestFixLookupIndex idx = new TestFixLookupIndex(); + /** + * @param mappings Reverse mappings. + */ + private static Set sourceIds(Collection mappings) { + Set res = new LinkedHashSet<>(); - StreamSupport.stream(cache.spliterator(), false) - .map(Cache.Entry::getValue) - .filter(TestFixesService::isPlausibleMatch) - .sorted(Comparator.comparingLong(TestFixesService::sortTs).reversed()) - .forEach(match -> { - TestFixRefUi ref = toUi(match); + for (TestFixRefs mapping : mappings) { + if (mapping != null && mapping.sourceIds != null) + res.addAll(mapping.sourceIds); + } - idx.add(match.entityName, ref); - idx.add(match.testName, ref); - idx.add(match.suiteName, ref); - }); + return res; + } - idx.finish(); + /** + * @param sourceIds Source ids. + */ + private Map sources(Set sourceIds) { + return sourceIds.isEmpty() ? Collections.emptyMap() : sourceCache.getAll(sourceIds); + } - return idx; + /** + * @param refs Reverse refs. + * @param sources Source fixes by id. + */ + private List refsFor(TestFixRefs refs, Map sources) { + List res = new ArrayList<>(); + + if (refs == null || refs.sourceIds == null) + return res; + + for (String sourceId : refs.sourceIds) { + TestFixSource source = sources.get(sourceId); + + if (source != null) + res.add(toUi(refs, source)); + } + + return res.stream() + .sorted(Comparator.comparingLong((TestFixRefUi ref) -> sortTs(ref)).reversed()) + .collect(Collectors.toList()); } /** @@ -719,6 +798,33 @@ private static long sortTs(TestFixMatch match) { return 0; } + /** + * @param ref UI ref. + */ + private static long sortTs(TestFixRefUi ref) { + long closed = dateScore(ref.closedDate); + + if (closed > 0) + return closed; + + return dateScore(ref.updatedDate); + } + + /** + * @param date ISO local date. + */ + private static long dateScore(@Nullable String date) { + if (Strings.isNullOrEmpty(date)) + return 0; + + try { + return LocalDate.parse(date).toEpochDay(); + } + catch (DateTimeParseException ignored) { + return 0; + } + } + /** * @param date Date string. */ @@ -848,7 +954,7 @@ private List candidates(ITrackedBranch branch, ITeamcityIgnite String currentStatusUrl = tc.host() + "buildConfiguration/" + suiteId + "?branch=" + org.apache.ignite.tcbot.common.util.UrlUtil.escape(ITcServerConfig.DEFAULT_TRACKED_BRANCH_NAME); - res.add(new TestFixCandidate(branch.name(), suiteId, suiteName, fullTestName, shortTestName, + res.add(new TestFixCandidate(branch.name(), suiteId, suiteName, testNameId, fullTestName, shortTestName, currentStatusUrl)); } @@ -1095,38 +1201,19 @@ private static boolean containsStandaloneName(String src, @Nullable String token * Initializes cache. */ private void ensureCache() { - if (cache != null) + if (sourceCache != null) return; synchronized (this) { - if (cache == null) { - CacheConfiguration cacheCfg = CacheConfigs.getCache8PartsConfig(CACHE_NAME); - - cacheCfg.setIndexedTypes(String.class, TestFixMatch.class); - - cache = igniteProvider.get().getOrCreateCache(cacheCfg); - signalCache = igniteProvider.get().getOrCreateCache( - CacheConfigs.getCache8PartsConfig(SOURCE_UPDATES_CACHE_NAME)); + if (sourceCache == null) { + sourceCache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig(SOURCE_CACHE_NAME)); + lookupCache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig(REFS_CACHE_NAME)); } } } - /** - * @return Latest source update signal timestamp. - */ - private long latestSignalTs() { - ensureCache(); - - long res = 0; - - for (Cache.Entry entry : signalCache) { - if (entry.getValue() != null) - res = Math.max(res, entry.getValue()); - } - - return res; - } - /** * @param processId Optional user-visible process id. * @param stage Current refresh stage. @@ -1213,6 +1300,9 @@ static class TestFixCandidate { /** Suite display name. */ private final String suiteName; + /** Compacted full test name id. */ + private final Integer testNameId; + /** Full test name. */ private final String fullTestName; @@ -1226,14 +1316,16 @@ static class TestFixCandidate { * @param trackedBranch Tracked branch. * @param suiteId Suite id. * @param suiteName Suite display name. + * @param testNameId Compacted full test name id. * @param fullTestName Full test name. * @param shortTestName Short test name. */ - TestFixCandidate(String trackedBranch, String suiteId, @Nullable String suiteName, String fullTestName, - @Nullable String shortTestName, String currentStatusUrl) { + TestFixCandidate(String trackedBranch, String suiteId, @Nullable String suiteName, Integer testNameId, + String fullTestName, @Nullable String shortTestName, String currentStatusUrl) { this.trackedBranch = trackedBranch; this.suiteId = suiteId; this.suiteName = suiteName; + this.testNameId = testNameId; this.fullTestName = fullTestName; this.shortTestName = shortTestName; this.currentStatusUrl = currentStatusUrl; @@ -1265,6 +1357,7 @@ private TestFixMatch newMatch() { match.suiteId = suiteId; match.suiteName = suiteName; match.testName = Strings.isNullOrEmpty(shortTestName) ? fullTestName : shortTestName; + match.testNameId = testNameId; match.currentStatusUrl = currentStatusUrl; return match; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java index 14e75c880..637f7e975 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java @@ -31,6 +31,9 @@ import static org.apache.ignite.tcbot.engine.ui.DsSuiteUi.buildWebLinkToBuild; public class ShortSuiteUi extends DsHistoryStatUi { + /** Suite build type id. */ + @Nullable public String suiteId; + /** Suite Name */ public String name; @@ -71,6 +74,7 @@ public ShortSuiteUi initFrom(@Nonnull MultBuildRunCtx suite, ITeamcityIgnited tcIgnited, IStringCompactor compactor, IRunHistory baseBranchHist) { + suiteId = suite.suiteId(); name = suite.suiteName(); result = suite.getResult(); webToBuild = buildWebLinkToBuild(tcIgnited, suite); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java index b3c0a14ea..1f9064e78 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java @@ -34,6 +34,12 @@ public class ShortTestFailureUi { /** Test full Name */ public String name; + /** Compacted test name id. */ + @Nullable public Integer testNameId; + + /** Suite build type id. */ + @Nullable public String suiteId; + /** suite (in code) short name */ @Nullable public String suiteName; @@ -56,6 +62,8 @@ public boolean isPossibleBlocker() { public ShortTestFailureUi initFrom(@Nonnull TestCompactedMult failure, ITeamcityIgnited tcIgn, Integer baseBranchId) { + testNameId = failure.testName(); + suiteId = failure.suiteId(); name = failure.getName(); String[] split = Strings.nullToEmpty(name).split("\\:"); diff --git a/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java index 7fe4697b3..89c838f45 100644 --- a/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java +++ b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java @@ -169,7 +169,7 @@ private static List candidates() { * @param suiteId Run configuration id. */ private static TestFixesService.TestFixCandidate candidate(String suiteId) { - return new TestFixesService.TestFixCandidate("master", suiteId, SUITE, FULL_TEST, SHORT_TEST, + return new TestFixesService.TestFixCandidate("master", suiteId, SUITE, 1, FULL_TEST, SHORT_TEST, "https://ci.example/buildConfiguration/" + suiteId + "?branch=%3Cdefault%3E"); } } diff --git a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java index 2bff1ee1d..e315f6de4 100644 --- a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java @@ -301,22 +301,9 @@ protected String runActualizePrs(String srvId, boolean fullReindex) { if (fullReindex) refreshOutdatedPrs(srvId, actualPrs); - if (cntSaved > 0) - signalSourceUpdate("github:" + srvId); - return "Entries saved " + cntSaved + " PRs checked " + totalChecked; } - /** - * @param key Source key. - */ - private void signalSourceUpdate(String key) { - IgniteCache cache = igniteProvider.get().getOrCreateCache( - CacheConfigs.getCache8PartsConfig("testFixSourceUpdates")); - - cache.put(key, System.currentTimeMillis()); - } - /** * Loads full public GitHub profiles for PR authors when GitHub /pulls returned compact users without email. * diff --git a/tcbot-integration-tests/src/integrationTest/resources/branches.json b/tcbot-integration-tests/src/integrationTest/resources/branches.json index 92b7bb1c9..00ef3794f 100644 --- a/tcbot-integration-tests/src/integrationTest/resources/branches.json +++ b/tcbot-integration-tests/src/integrationTest/resources/branches.json @@ -1,7 +1,14 @@ { "primaryServerCode": "apache", "botAdminGroups": ["IGNITE_COMMITTER"], - "resettableCaches": ["testFixMatches", "testFixMatchesV2", "testFixSourceUpdates"], + "resettableCaches": [ + "testFixRefsByTest", + "testFixSourcesById", + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ], "confidence": 0.995, "tcServers": [ { diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java index 23282f321..6db8bee57 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java @@ -49,9 +49,6 @@ public class JiraTicketSync { /** Logger. */ private static final Logger logger = LoggerFactory.getLogger(JiraTicketSync.class); - /** Source update signal cache name. Also used by test-fix matching. */ - private static final String SOURCE_UPDATES_CACHE_NAME = "testFixSourceUpdates"; - /** Test-fix JIRA sync state cache name. */ private static final String TEST_FIX_SYNC_STATE_CACHE_NAME = "jiraTestFixSyncState"; @@ -151,25 +148,12 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { int saved = stats.saved + recent.saved + labeled.saved; - if (saved > 0) - signalSourceUpdate("jira:" + srvCode); - return "Jira tickets saved " + saved + " from " + (stats.processed + recent.processed + labeled.processed) + " checked for service " + srvCode + " (recentDays=" + recentDays + ", labelSync=" + labelSync + ", duplicatesSkipped=" + (stats.duplicatesSkipped + recent.duplicatesSkipped + labeled.duplicatesSkipped) + ")"; } - /** - * @param key Source key. - */ - private void signalSourceUpdate(String key) { - IgniteCache cache = igniteProvider.get().getOrCreateCache( - CacheConfigs.getCache8PartsConfig(SOURCE_UPDATES_CACHE_NAME)); - - cache.put(key, System.currentTimeMillis()); - } - /** * @param srvCode Server code. * @param cfg JIRA config. From 7b9f1b6186014a18d959764a08a8cee10dab55b7 Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 22:41:54 +0300 Subject: [PATCH 5/6] Keep test fix branch focused --- .run/TC Bot Server WAR.run.xml | 11 --- .run/TC Bot Server.run.xml | 11 --- ...ngFlowTest.java => TestFixesFlowTest.java} | 87 +------------------ 3 files changed, 2 insertions(+), 107 deletions(-) delete mode 100644 .run/TC Bot Server WAR.run.xml delete mode 100644 .run/TC Bot Server.run.xml rename tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/{TestFixesAndCancelledOrderingFlowTest.java => TestFixesFlowTest.java} (64%) diff --git a/.run/TC Bot Server WAR.run.xml b/.run/TC Bot Server WAR.run.xml deleted file mode 100644 index 9c286685d..000000000 --- a/.run/TC Bot Server WAR.run.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - diff --git a/.run/TC Bot Server.run.xml b/.run/TC Bot Server.run.xml deleted file mode 100644 index 76962d53f..000000000 --- a/.run/TC Bot Server.run.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - diff --git a/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java similarity index 64% rename from tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java rename to tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java index b6881dd35..a062d784e 100644 --- a/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesAndCancelledOrderingFlowTest.java +++ b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java @@ -17,8 +17,6 @@ package org.apache.ignite.tcbot.integration; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.regex.Pattern; import org.junit.Before; @@ -30,9 +28,9 @@ import static org.junit.Assert.assertTrue; /** - * Black-box coverage for test-fix matching and cancelled suite ordering. + * Black-box coverage for test-fix matching. */ -public class TestFixesAndCancelledOrderingFlowTest { +public class TestFixesFlowTest { /** */ private static final Pattern FINISHED_PROCESS = Pattern.compile("\"finished\"\\s*:\\s*[1-9][0-9]*"); @@ -91,41 +89,6 @@ public void testFixRefreshReturnsOnlyOpenAndResolvedJiraTasksForMatchedTests() t assertTrue(rows, rows.contains("\"closedDate\":\"2026-05-09\"")); } - /** */ - @Test - public void cancelledSuitesAreAfterBuildFailureInPrReportAndVisaComment() throws Exception { - String token = env.login(); - - runTestOnlyAction(token, "refresh-jira", 710000011L); - runTestOnlyAction(token, "refresh-github", 710000012L); - runPrBuildRefsRefresh(token, 710000013L); - - IntegrationTestEnvironment.HttpResponse prResults = request("GET", env.botUrl - + "/rest/pr/results?serverId=apache" - + "&suiteId=" + enc("IgniteTests24Java17_RunAll") - + "&branchForTc=" + enc("pull/12007/head") - + "&action=" + enc("Latest"), - "Token " + token, null, null); - - assertEquals(prResults.body, 200, prResults.status); - assertBefore(prResults.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Cache1"); - assertBefore(prResults.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Sql"); - assertTrue(prResults.body, prResults.body.contains("CANCELLED")); - - String status = commentBuildAnalysis(token, 710000014L, "pull/12007/head", "IGNITE-20007", 12007); - - assertTrue(status, status.contains("JIRA ticket commented: IGNITE-20007")); - assertTrue(status, status.contains("GitHub PR commented: PR #12007")); - - IntegrationTestEnvironment.HttpResponse comments = request("GET", - env.githubUrl + "/repos/apache/ignite/issues/12007/comments", "Bearer github-test-token", null, null); - - assertEquals(comments.body, 200, comments.status); - assertBefore(comments.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Cache1"); - assertBefore(comments.body, "IgniteTests24Java17_Build", "IgniteTests24Java17_Sql"); - assertTrue(comments.body, comments.body.contains("CANCELLED")); - } - /** */ private static void createIssue(String key, String summary, String status, String description, String resolutionDate) throws Exception { @@ -188,38 +151,6 @@ private static String runTestOnlyAction(String token, String action, long proces return waitForProcess(processId, token); } - /** */ - private static String runPrBuildRefsRefresh(String token, long processId) throws Exception { - IntegrationTestEnvironment.HttpResponse start = request("POST", env.botUrl - + "/rest/pr/actualizeBuildRefs?serverId=apache&processId=" + processId, - "Token " + token, null, null); - - assertEquals(start.body, 200, start.status); - assertTrue(start.body, start.body.contains("TeamCity build refs refresh queued")); - - return waitForProcess(processId, token); - } - - /** */ - private static String commentBuildAnalysis(String token, long processId, String branch, String ticket, int prNum) - throws Exception { - IntegrationTestEnvironment.HttpResponse start = request("GET", env.botUrl - + "/rest/build/commentBuildAnalysis?serverId=apache" - + "&branchName=" + enc(branch) - + "&suiteId=" + enc("IgniteTests24Java17_RunAll") - + "&ticketId=" + enc(ticket) - + "&comment=" + enc("JIRA,GITHUB") - + "&prNum=" + prNum - + "&commentOnlyIfNoBlockers=false" - + "&processId=" + processId, - "Token " + token, null, null); - - assertEquals(start.body, 200, start.status); - assertTrue(start.body, start.body.contains("Comment process started")); - - return waitForProcess(processId, token); - } - /** */ private static String waitForProcess(long processId, String token) throws Exception { long deadline = System.nanoTime() + Duration.ofSeconds(90).toNanos(); @@ -239,16 +170,6 @@ private static String waitForProcess(long processId, String token) throws Except throw new IllegalStateException("Process did not finish: " + processId + ", last status: " + lastBody); } - /** */ - private static void assertBefore(String text, String first, String second) { - int firstIdx = text.indexOf(first); - int secondIdx = text.indexOf(second); - - assertTrue("Expected to find " + first + " in: " + text, firstIdx >= 0); - assertTrue("Expected to find " + second + " in: " + text, secondIdx >= 0); - assertTrue("Expected " + first + " before " + second + " in: " + text, firstIdx < secondIdx); - } - /** */ private static int count(String text, String needle) { int res = 0; @@ -267,8 +188,4 @@ private static int count(String text, String needle) { return res; } - /** */ - private static String enc(String val) { - return URLEncoder.encode(val, StandardCharsets.UTF_8); - } } From bc3e26c3a157e832fc8d13872907cdb81b0a6e70 Mon Sep 17 00:00:00 2001 From: Dmitriy Pavlov Date: Sun, 10 May 2026 23:17:51 +0300 Subject: [PATCH 6/6] Address test fix review feedback --- .../src/main/webapp/js/prs-1.3.js | 5 -- .../engine/testfixes/TestFixesService.java | 54 ++++++++++++++----- .../tracked/TrackedBranchChainsProcessor.java | 48 ++++++++++++++++- 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js b/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js index 68bb98788..09bddd1e6 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js +++ b/ignite-tc-helper-web/src/main/webapp/js/prs-1.3.js @@ -411,11 +411,6 @@ function claimGithubAuthorHtml(srvId, row) { let explicitlyConfigured = explicitlyConfiguredGithubLoginsByServer.get(srvId); - if (isDefinedAndFilled(explicitlyConfigured) && explicitlyConfigured.has(String(row.prAuthor).toLowerCase())) - return ""; - - let explicitlyConfigured = explicitlyConfiguredGithubLoginsByServer.get(srvId); - if (isDefinedAndFilled(explicitlyConfigured) && explicitlyConfigured.has(String(row.prAuthor).toLowerCase())) return ""; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java index 989c8342e..b624dde83 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java @@ -263,6 +263,7 @@ public String refresh(@Nullable Long processId) { ensureCache(); RefreshStats stats = new RefreshStats(); + RefreshBuffer buffer = new RefreshBuffer(); publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); Map> candidatesByServer = candidatesByServer(); @@ -281,10 +282,12 @@ public String refresh(@Nullable Long processId) { publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + srvCode, stats); - refreshJira(srvCode, srvCandidates, recentPrs, stats, processId); - refreshGithub(srvCode, srvCandidates, recentPrs, stats, processId); + refreshJira(buffer, srvCode, srvCandidates, recentPrs, stats, processId); + refreshGithub(buffer, srvCode, srvCandidates, recentPrs, stats, processId); } + replaceCaches(buffer); + return stats.finishText(); } @@ -325,8 +328,8 @@ private void safeRefresh(@Nullable Long processId) { * @param srvCode Server code. * @param recentPrs Recent GitHub pull requests. */ - private void refreshJira(String srvCode, List candidates, List recentPrs, - RefreshStats stats, @Nullable Long processId) { + private void refreshJira(RefreshBuffer buffer, String srvCode, List candidates, + List recentPrs, RefreshStats stats, @Nullable Long processId) { try { IJiraIgnited jira = jiraProvider.server(srvCode); IJiraServerConfig jiraCfg = jira.config(); @@ -365,13 +368,13 @@ private void refreshJira(String srvCode, List candidates, List match.updatedTs = updatedTs; match.closedTs = closedTs; - saveMatch(match); + saveMatch(buffer, match); stats.jiraSaved++; for (PullRequest pr : linkedPrs) { TestFixMatch linkedPrMatch = githubMatch(candidate, pr); - saveMatch(linkedPrMatch); + saveMatch(buffer, linkedPrMatch); stats.githubSaved++; } } @@ -389,8 +392,8 @@ private void refreshJira(String srvCode, List candidates, List * @param srvCode Server code. * @param recentPrs Recent GitHub pull requests. */ - private void refreshGithub(String srvCode, List candidates, List recentPrs, - RefreshStats stats, @Nullable Long processId) { + private void refreshGithub(RefreshBuffer buffer, String srvCode, List candidates, + List recentPrs, RefreshStats stats, @Nullable Long processId) { try { publishRefreshStatus(processId, "checking GitHub pull requests for " + srvCode, stats); for (PullRequest pr : recentPrs) { @@ -404,7 +407,7 @@ private void refreshGithub(String srvCode, List candidates, Li for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { TestFixMatch match = githubMatch(candidate, pr); - saveMatch(match); + saveMatch(buffer, match); stats.githubSaved++; } } @@ -598,16 +601,16 @@ private static String shortSha(String sha) { /** * @param match Match. */ - private void saveMatch(TestFixMatch match) { + private void saveMatch(RefreshBuffer buffer, TestFixMatch match) { if (!isPlausibleMatch(match)) return; String sourceId = sourceKey(match); - sourceCache.put(sourceId, source(match)); + buffer.sources.put(sourceId, source(match)); for (String key : lookupKeys(match)) { - TestFixRefs refs = lookupCache.get(key); + TestFixRefs refs = buffer.refs.get(key); if (refs == null) refs = refs(match); @@ -615,10 +618,26 @@ private void saveMatch(TestFixMatch match) { if (!refs.sourceIds.contains(sourceId)) refs.sourceIds.add(sourceId); - lookupCache.put(key, refs); + buffer.refs.put(key, refs); } } + /** + * Replaces persistent match caches with the latest successful refresh snapshot. + * + * @param buffer Newly collected mappings. + */ + private void replaceCaches(RefreshBuffer buffer) { + lookupCache.removeAll(); + sourceCache.removeAll(); + + if (!buffer.refs.isEmpty()) + lookupCache.putAll(buffer.refs); + + if (!buffer.sources.isEmpty()) + sourceCache.putAll(buffer.sources); + } + /** * @param match Match. */ @@ -1223,6 +1242,15 @@ private void publishRefreshStatus(@Nullable Long processId, String stage, Refres processMonitor.status(processId, "Test fix mapping: " + stage + ". " + stats.progressText()); } + /** In-memory refresh result applied to persistent caches after refresh completion. */ + private static class RefreshBuffer { + /** Source fixes by id. */ + private final Map sources = new HashMap<>(); + + /** Lookup refs by suite/test lookup key. */ + private final Map refs = new HashMap<>(); + } + /** Test fix refresh counters. */ private static class RefreshStats { /** Candidate test names from tracked suite histories. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index e3dcc6444..c200ea2be 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -56,6 +56,7 @@ import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; @@ -384,7 +385,6 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { List accessibleChains = tracked.chainsStream() .filter(chainTracked -> tcIgnitedProv.hasAccess(chainTracked.serverCode(), creds)) - .filter(chainTracked -> Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainTracked.tcSuiteId())) .collect(Collectors.toList()); for (ITrackedChain chainTracked : accessibleChains) { @@ -449,10 +449,13 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests, tagSelected, displayMode, maxDurationSec, requireParamVal, showMuted, showIgnored); + filterSuites(chainStatus, suiteId); chainStatus.suites.forEach(testFixesService::decorate); uiInitNanos += System.nanoTime() - stepStart; - res.addChainOnServer(chainStatus); + if (Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainStatus.suiteId) || !chainStatus.suites.isEmpty()) + res.addChainOnServer(chainStatus); + chainTotalNanos += System.nanoTime() - chainStart; } @@ -477,6 +480,47 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { return res; } + /** + * Keeps a requested child suite inside the loaded root tracked chain. + * + * @param chainStatus Chain UI. + * @param suiteId Optional requested suite id. + */ + private static void filterSuites(DsChainUi chainStatus, @Nullable String suiteId) { + if (chainStatus == null || Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainStatus.suiteId)) + return; + + chainStatus.suites = chainStatus.suites.stream() + .filter(suite -> suiteId.equals(suite.suiteId)) + .collect(Collectors.toList()); + + chainStatus.failedTests = sum(chainStatus.suites.stream().map(suite -> suite.failedTests) + .collect(Collectors.toList())); + chainStatus.totalTests = sum(chainStatus.suites.stream().map(suite -> suite.totalTests) + .collect(Collectors.toList())); + chainStatus.trustedTests = sum(chainStatus.suites.stream().map(suite -> suite.trustedTests) + .collect(Collectors.toList())); + chainStatus.totalBlockers = chainStatus.suites.stream().mapToInt(DsSuiteUi::totalBlockers).sum(); + } + + /** + * @param vals Values. + */ + private static Integer sum(Collection vals) { + int res = 0; + boolean found = false; + + for (Integer val : vals) { + if (val == null) + continue; + + res += val; + found = true; + } + + return found ? res : null; + } + /** * @param nanos Nanoseconds. */