diff --git a/conf/branches.json b/conf/branches.json index efbc5da47..67a48fe0b 100644 --- a/conf/branches.json +++ b/conf/branches.json @@ -5,7 +5,14 @@ // Apache Ignite test admins are exposed by TeamCity REST as key="IGNITE_COMMITER". "botAdminGroups": ["IGNITE_COMMITER"], // Ignite cache names admins may reset from monitoring UI. - "resettableCaches": ["testFixMatches", "testFixSourceUpdates"], + "resettableCaches": [ + "testFixRefsByTest", + "testFixSourcesById", + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ], "confidence": 0.995, "cleanerConfig": { "numOfItemsToDel": 100000, diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java index eb775bf12..a129641e7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java @@ -632,6 +632,7 @@ protected String checkFailuresEx(String brachName) { false, null, null, + null, DisplayMode.None, null, -1, false, false); @@ -645,6 +646,7 @@ protected String checkFailuresEx(String brachName) { false, null, null, + null, DisplayMode.OnlyFailures, null, -1, false, false); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java index 6a6937c56..6c061de9c 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java @@ -16,6 +16,10 @@ */ package org.apache.ignite.ci.web.rest.monitoring; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import java.io.BufferedReader; import java.io.File; @@ -31,7 +35,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -40,9 +46,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.security.RolesAllowed; +import javax.cache.Cache; import javax.servlet.ServletContext; import javax.ws.rs.BadRequestException; import javax.ws.rs.ClientErrorException; +import javax.ws.rs.ForbiddenException; import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.NotFoundException; @@ -110,6 +118,44 @@ public class MonitoringService { /** Max summary length. */ private static final int SUMMARY_LIMIT = 240; + /** Default number of cache entries to preview. */ + private static final int DFLT_CACHE_PEEK_LIMIT = 5; + + /** Hard cache preview cap. */ + private static final int MAX_CACHE_PEEK_LIMIT = 20; + + /** System property with comma-separated exact cache names allowed for raw preview. */ + private static final String CACHE_PEEK_ALLOWED_CACHES = "tcbot.monitoring.cachePeek.allowedCaches"; + + /** Built-in exact cache names allowed for raw preview. */ + private static final Set DFLT_CACHE_PEEK_ALLOWED_CACHES = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList( + "botDetectedDefects", + "botDetectedIssues", + "buildLogCheckResult", + "buildsConditions", + "compactVisasHistoryCacheV2", + "gitHubBranch", + "gitHubPr", + "jiraTestFixSyncState", + "mutedIssues", + "newTestsCache", + "teamcityBuildRef", + "teamcityBuildStartTime", + "teamcityBuildTypeRef", + "teamcityChange", + "teamcityFatBuild", + "teamcityFatBuildType", + "teamcityMute", + "teamcitySuiteHistory", + "testFixRefsByTest", + "testFixSourcesById" + ))); + + /** JSON mapper for raw cache entry values. */ + private static final ObjectMapper CACHE_PEEK_MAPPER = new ObjectMapper() + .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + /** Log timestamp format. */ private static final DateTimeFormatter LOG_TS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); @@ -602,6 +648,110 @@ public List getCacheStat() { return res; } + @GET + @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) + @Produces(MediaType.TEXT_PLAIN) + @Path("cachePeek") + public String cachePeek(@QueryParam("name") String name, @QueryParam("limit") Integer limit) { + if (Strings.isNullOrEmpty(name)) + throw new BadRequestException("Cache name is required"); + + ensureCanPeekCache(name); + + Ignite ignite = instance(Ignite.class); + IgniteCache cache = ignite.cache(name); + + if (cache == null) + throw new NotFoundException("Cache not found: " + name); + + int actualLimit = normalizeCachePeekLimit(limit); + StringBuilder res = new StringBuilder(); + int shown = 0; + boolean truncated = false; + + res.append("Cache: ").append(name).append('\n'); + res.append("Size: not calculated by cachePeek").append('\n'); + res.append("Limit: ").append(actualLimit).append("\n\n"); + + for (Cache.Entry entry : cache) { + if (shown >= actualLimit) { + truncated = true; + + break; + } + + shown++; + + res.append("Entry #").append(shown).append('\n'); + res.append("Key class: ").append(className(entry.getKey())).append('\n'); + res.append(toJsonNode(entry.getKey()).toPrettyString()).append('\n'); + res.append("Value class: ").append(className(entry.getValue())).append('\n'); + res.append(toJsonNode(entry.getValue()).toPrettyString()).append("\n\n"); + } + + res.append("Entries shown: ").append(shown).append('\n'); + res.append("Truncated: ").append(truncated).append('\n'); + + return res.toString(); + } + + /** + * @param limit Requested limit. + */ + private static int normalizeCachePeekLimit(Integer limit) { + if (limit == null || limit <= 0) + return DFLT_CACHE_PEEK_LIMIT; + + return Math.min(limit, MAX_CACHE_PEEK_LIMIT); + } + + /** + * @param obj Object. + */ + private static String className(Object obj) { + return obj == null ? "null" : obj.getClass().getName(); + } + + /** + * @param name Cache name. + */ + private void ensureCanPeekCache(String name) { + if (cachePeekAllowedCaches().contains(name)) + return; + + throw new ForbiddenException("Cache peek is not allowed for cache: " + name + + ". Add the exact cache name to " + CACHE_PEEK_ALLOWED_CACHES + " to enable it."); + } + + /** + * @return Exact cache names allowed for raw preview. + */ + private static Set cachePeekAllowedCaches() { + Set res = new HashSet<>(DFLT_CACHE_PEEK_ALLOWED_CACHES); + + Arrays.stream(Strings.nullToEmpty(System.getProperty(CACHE_PEEK_ALLOWED_CACHES)).split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .forEach(res::add); + + return res; + } + + /** + * @param obj Object. + */ + private static JsonNode toJsonNode(Object obj) { + try { + return CACHE_PEEK_MAPPER.valueToTree(obj); + } + catch (RuntimeException e) { + return CACHE_PEEK_MAPPER.createObjectNode() + .put("serializationError", e.getClass().getSimpleName() + ": " + e.getMessage()) + .put("class", className(obj)) + .put("toString", String.valueOf(obj)); + } + } + @POST @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) @Path("resetCache") diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java new file mode 100644 index 000000000..04536f006 --- /dev/null +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ci.web.rest.testfixes; + +import java.util.List; +import javax.annotation.security.RolesAllowed; +import javax.servlet.ServletContext; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import org.apache.ignite.ci.web.CtxListener; +import org.apache.ignite.ci.web.auth.AuthenticationFilter; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; + +/** + * Test fix history REST service. + */ +@Path("testFixes") +@Produces(MediaType.APPLICATION_JSON) +public class TestFixesRestService { + /** Servlet Context. */ + @Context private ServletContext ctx; + + /** + * @param limit Max rows. + */ + @GET + @Path("recent") + public List recent(@QueryParam("limit") Integer limit) { + int actualLimit = limit == null ? 0 : limit; + + return CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class).recent(actualLimit); + } + + /** + * Schedules an immediate refresh and returns currently cached rows. + * + * @param limit Max rows. + * @param processId Optional user-visible process id. + */ + @GET + @RolesAllowed(AuthenticationFilter.ADMIN_ROLE) + @Path("refresh") + public List refresh(@QueryParam("limit") Integer limit, @QueryParam("processId") Long processId) { + TestFixesService svc = CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class); + + svc.requestMatchNow(processId); + + return recent(limit); + } +} diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java index 571abaa02..2d1feafc8 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java @@ -124,6 +124,7 @@ public DsSummaryUi getTestFailsResultsNoSync( @Nullable @QueryParam("trustedTests") Boolean trustedTests, @Nullable @QueryParam("tagSelected") String tagSelected, @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected, + @Nullable @QueryParam("suiteId") String suiteId, @Nullable @QueryParam("displayMode") String displayMode, @Nullable @QueryParam("sortOption") String sortOption, @Nullable @QueryParam("count") Integer mergeCnt, @@ -131,7 +132,7 @@ public DsSummaryUi getTestFailsResultsNoSync( @Nullable @QueryParam("muted") Boolean showMuted, @Nullable @QueryParam("ignored") Boolean showIgnored) { return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected, - SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); + suiteId, SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); } @GET @@ -143,6 +144,7 @@ public DsSummaryUi getTestFailsNoCache( @Nullable @QueryParam("trustedTests") Boolean trustedTests, @Nullable @QueryParam("tagSelected") String tagSelected, @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected, + @Nullable @QueryParam("suiteId") String suiteId, @Nullable @QueryParam("displayMode") String displayMode, @Nullable @QueryParam("sortOption") String sortOption, @Nullable @QueryParam("count") Integer mergeCnt, @@ -150,7 +152,8 @@ public DsSummaryUi getTestFailsNoCache( @Nullable @QueryParam("muted") Boolean showMuted, @Nullable @QueryParam("ignored") Boolean showIgnored) { return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected, - SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored); + suiteId, SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, + showIgnored); } @NotNull private DsSummaryUi latestBuildResults( @@ -159,6 +162,7 @@ public DsSummaryUi getTestFailsNoCache( @Nullable Boolean trustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nonnull SyncMode mode, @Nullable String displayMode, @Nullable String sortOption, @@ -185,6 +189,7 @@ public DsSummaryUi getTestFailsNoCache( Boolean.TRUE.equals(trustedTests), tagSelected, tagForHistSelected, + suiteId, DisplayMode.parseStringValue(displayMode), SortOption.parseStringValue(sortOption), maxDurationSec, diff --git a/ignite-tc-helper-web/src/main/webapp/current.html b/ignite-tc-helper-web/src/main/webapp/current.html index 07ca05899..11384f510 100644 --- a/ignite-tc-helper-web/src/main/webapp/current.html +++ b/ignite-tc-helper-web/src/main/webapp/current.html @@ -204,7 +204,11 @@ let tagForHistSelected = gVue.$data.tagForHistSelected; if (tagForHistSelected != null && tagForHistSelected !== "") - curReqParms += "&tagForHistSelected=" + tagForHistSelected; + curReqParms += "&tagForHistSelected=" + encodeURIComponent(tagForHistSelected); + + let suiteId = findGetParameter("suiteId"); + if (suiteId != null && suiteId !== "") + curReqParms += "&suiteId=" + encodeURIComponent(suiteId); let runTime = gVue.$data.showTestLongerThan; if (runTime != null && runTime > 0) diff --git a/ignite-tc-helper-web/src/main/webapp/guard.html b/ignite-tc-helper-web/src/main/webapp/guard.html index cdb9dbdb1..430d84a24 100644 --- a/ignite-tc-helper-web/src/main/webapp/guard.html +++ b/ignite-tc-helper-web/src/main/webapp/guard.html @@ -4,7 +4,6 @@ - @@ -19,7 +18,6 @@ .guard-page { margin: 22px 18px 34px 18px; - max-width: 1320px; padding: 0; } diff --git a/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js b/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js index 9de453ca6..d05365991 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js +++ b/ignite-tc-helper-web/src/main/webapp/js/common-1.7.js @@ -732,6 +732,7 @@ function showMenu(menuData) { res += "Compare builds"; res += "Issues history"; res += "Visas history"; + res += "Test fixes history"; res += "Muted tests"; res += "Muted issues"; res += "Board"; diff --git a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js index 5c4a6ad15..55c3a99a2 100644 --- a/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js +++ b/ignite-tc-helper-web/src/main/webapp/js/testfails-2.3.js @@ -1801,6 +1801,8 @@ function showSuiteData(suite, settings, prNum) { res += "title='Open AI prompt with TeamCity context for this suite'>[AI Prompt]"; } + res += testFixRefsHtml(suite.fixRefs); + if(isDefinedAndFilled(suite.tags)) { for (let i = 0; i < suite.tags.length; i++) { const tag = suite.tags[i]; @@ -2068,6 +2070,8 @@ function showTestFailData(testFail, isFailureShown, settings) { res += "title='Open AI prompt with TeamCity context for this test'>[AI Prompt]"; } + res += testFixRefsHtml(testFail.fixRefs); + var histContent = ""; //see class TestHistory @@ -2153,6 +2157,34 @@ function showTestFailData(testFail, isFailureShown, settings) { return res; } +function testFixRefsHtml(refs) { + if (!isDefinedAndFilled(refs) || refs.length === 0) + return ""; + + var res = ""; + + for (var i = 0; i < refs.length; i++) { + var ref = refs[i]; + var text = ref.sourceType === "github" ? "GH" : "JIRA"; + var title = "Matched fix"; + + if (isDefinedAndFilled(ref.title)) + title += ": " + ref.title; + + if (isDefinedAndFilled(ref.status)) + title += " [" + ref.status + "]"; + + if (isDefinedAndFilled(ref.closedDate)) + title += " closed " + ref.closedDate; + + res += " " + escapeHtml(text) + ""; + } + + return res; +} + function drawLatestRuns(latestRuns) { if(isDefinedAndFilled(findGetParameter("reportMode"))) return ""; diff --git a/ignite-tc-helper-web/src/main/webapp/monitoring.html b/ignite-tc-helper-web/src/main/webapp/monitoring.html index 68c5bdb1a..6695c6047 100644 --- a/ignite-tc-helper-web/src/main/webapp/monitoring.html +++ b/ignite-tc-helper-web/src/main/webapp/monitoring.html @@ -17,6 +17,7 @@ + + + + + + + + +
+ +
+ Show latest + + matched fixes + +
+ + + + + + + + + + + +
Fix / commitResolved testsAuthorStatusDates
+ +
+ + + + diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java index 492654ffc..4321c2db7 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/TrackedBranchProcessorTest.java @@ -121,7 +121,7 @@ public void testTrackedBranchChainsProcessor() { false, 1, mock, SyncMode.RELOAD_QUEUED, - false, null, null, DisplayMode.OnlyFailures, null, + false, null, null, null, DisplayMode.OnlyFailures, null, -1, false, false); Gson gson = new GsonBuilder().setPrettyPrinting().create(); diff --git a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java index 49167a01e..87232a0dd 100644 --- a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java +++ b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/app/guice/GuiceTcBotApplicationContext.java @@ -38,6 +38,7 @@ import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.conf.NotificationsConfig; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.notify.ISlackSender; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.apache.ignite.tcservice.http.TeamcityRecorder; @@ -91,6 +92,7 @@ private void startBackgroundServicesWhenReady(Future igniteFuture) { getInstance(BuildObserver.class); getInstance(UserAdminRefreshService.class).start(); + getInstance(TestFixesService.class).start(); ready.set(true); } catch (Exception e) { diff --git a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java index b3d1abeea..7b495dcfd 100644 --- a/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java +++ b/tcbot-app-guice/src/main/java/org/apache/ignite/tcbot/engine/TcBotEngineModule.java @@ -29,6 +29,7 @@ import org.apache.ignite.tcbot.engine.issue.IssuesStorage; import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch; import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor; import org.apache.ignite.tcbot.engine.user.IUserStorage; @@ -55,6 +56,7 @@ public class TcBotEngineModule extends AbstractModule { bind(MutedIssuesDao.class).in(Scopes.SINGLETON); bind(NewTestsStorage.class).in(Scopes.SINGLETON); bind(BotProcessMonitor.class).in(Scopes.SINGLETON); + bind(TestFixesService.class).in(Scopes.SINGLETON); install(new TcBotCommonModule()); } diff --git a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java index bf1fd2982..916f8c999 100644 --- a/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java +++ b/tcbot-common/src/main/java/org/apache/ignite/tcbot/common/conf/IJiraServerConfig.java @@ -60,6 +60,20 @@ public interface IJiraServerConfig { */ @Nullable String decodedHttpAuthToken(); + /** + * JIRA label used to explicitly mark test-fix tickets. + */ + default String testFixesLabel() { + return "MakeTeamcityGreenAgain"; + } + + /** + * Number of recent days to scan for test-fix tickets. Default matches base branch run-history horizon. + */ + default int testFixesLookbackDays() { + return org.apache.ignite.tcbot.common.TcBotConst.HISTORY_MAX_DAYS; + } + /** * @return {@code True} if JIRA authorization token is available. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java index 02a99fb46..f36add13a 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/BuildChainProcessor.java @@ -242,8 +242,11 @@ else if (sortOption == SortOption.SuiteDuration) { }; } - if (function != null) - contexts.sort(Comparator.comparing(function).reversed()); + if (function != null) { + contexts.sort(Comparator + .comparing(MultBuildRunCtx::onlyCancelledBuilds) + .thenComparing(Comparator.comparing(function).reversed())); + } fullChainRunCtx.addAllSuites(contexts); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java index 199bdeaf0..1da0f822d 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/chain/TestCompactedMult.java @@ -60,6 +60,10 @@ public static void resetCached() { @Nullable public Integer testName() { return occurrences.isEmpty() ? null : occurrences.iterator().next().testName(); } + + @Nullable public String suiteId() { + return ctx == null ? null : ctx.suiteId(); + } public String getName() { return occurrences.isEmpty() ? "" : occurrences.iterator().next().testName(compactor); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java index 8bbb26b9a..413b6c8f6 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java @@ -42,7 +42,15 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier { String DEFAULT_BOT_ADMIN_GROUP = "IGNITE_COMMITER"; /** Caches safe to reset from monitoring UI by default. */ - List DEFAULT_RESETTABLE_CACHES = List.of("testFixMatches", "testFixSourceUpdates"); + List DEFAULT_RESETTABLE_CACHES = List.of( + "testFixRefsByTest", + "testFixSourcesById", + // Old test-fix cache names may still be restored from persistence after cache layout changes. + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ); /** */ String primaryServerCode(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java index 72d94e322..20f3ddf3b 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/JiraServerConfig.java @@ -37,6 +37,12 @@ public class JiraServerConfig implements IJiraServerConfig { /** JIRA URL to build links to tickets. */ public static final String JIRA_URL = "jira.url"; + /** Explicit JIRA label for test-fix tickets. */ + public static final String JIRA_TEST_FIXES_LABEL = "jira.test_fixes_label"; + + /** Recent days window for test-fix ticket matching. */ + public static final String JIRA_TEST_FIXES_LOOKBACK_DAYS = "jira.test_fixes_lookback_days"; + /** Prefix for JIRA ticket names. */ @Deprecated public static final String JIRA_TICKET_TEMPLATE = "jira.ticket_template"; @@ -79,6 +85,12 @@ public class JiraServerConfig implements IJiraServerConfig { **/ private JiraApiVersion apiVersion = JiraApiVersion.defaultApiVersion(); + /** Explicit JIRA label for test-fix tickets. */ + private String testFixesLabel; + + /** Recent days window for test-fix ticket matching. */ + @Nullable private Integer testFixesLookbackDays; + public JiraServerConfig() { } @@ -139,6 +151,32 @@ public JiraServerConfig code(String code) { return Strings.emptyToNull(branchNumPrefix); } + /** {@inheritDoc} */ + @Override public String testFixesLabel() { + if (!Strings.isNullOrEmpty(testFixesLabel)) + return testFixesLabel; + + return props != null + ? props.getProperty(JIRA_TEST_FIXES_LABEL, IJiraServerConfig.super.testFixesLabel()) + : IJiraServerConfig.super.testFixesLabel(); + } + + /** {@inheritDoc} */ + @Override public int testFixesLookbackDays() { + if (testFixesLookbackDays != null) + return testFixesLookbackDays; + + if (props == null) + return IJiraServerConfig.super.testFixesLookbackDays(); + + String val = props.getProperty(JIRA_TEST_FIXES_LOOKBACK_DAYS); + + if (Strings.isNullOrEmpty(val)) + return IJiraServerConfig.super.testFixesLookbackDays(); + + return Integer.parseInt(val); + } + /** {@inheritDoc} */ @Nullable @Override diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java index 54c4de5bd..ab1e21387 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/pr/PrChainsProcessor.java @@ -61,6 +61,7 @@ import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; @@ -135,6 +136,9 @@ private static class Action { /** User-visible process monitor. */ @Inject private BotProcessMonitor processMonitor; + /** Test fix matcher. */ + @Inject private TestFixesService testFixesService; + /** * @param creds Credentials. * @param srvCodeOrAlias Server code or alias. @@ -217,6 +221,7 @@ else if (Action.CHAIN.equals(act)) //fail rate reference is always default (master) chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false, null, null, -1, null, false, false); // don't need for PR + chainStatus.suites.forEach(testFixesService::decorate); chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor, newTestsStorage); initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited); } @@ -656,7 +661,7 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, Predicate filter = suite -> suite.isFailed() || suite.hasTestToReport(tcIgnited, baseBranchId, false, false); - return fullChainRunCtx + List suites = fullChainRunCtx .filteredChildSuites(filter) .map((ctx) -> { IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId, null); @@ -674,15 +679,21 @@ private List findBlockerFailures(FullChainRunCtx fullChainRunCtx, // test failure based blockers and/or blocker found by suite results if (!failures.isEmpty() || !Strings.isNullOrEmpty(suiteComment)) { - return new ShortSuiteUi() + ShortSuiteUi suite = new ShortSuiteUi() .testShortFailures(failures) .initFrom(ctx, tcIgnited, compactor, statInBaseBranch); + + return suite; } return null; }) .filter(Objects::nonNull) .collect(Collectors.toList()); + + testFixesService.decorate(suites); + + return suites; } /** diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java new file mode 100644 index 000000000..c74d69733 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixLookupIndex.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Response-local lookup for decorating suites and test failures with known fix refs. + */ +class TestFixLookupIndex { + /** Suite-level lookup entity. */ + private static final String SUITE_ENTITY = "suite"; + + /** Max refs shown for a suite/test. */ + private static final int MAX_REFS = 5; + + /** Matches by suite id and compacted test id lookup key. */ + private final Map> refsByKey = new HashMap<>(); + + /** */ + public void add(String lookupKey, List refs) { + if (Strings.isNullOrEmpty(lookupKey) || refs == null || refs.isEmpty()) + return; + + refsByKey.computeIfAbsent(lookupKey, unused -> new ArrayList<>()).addAll(refs); + } + + /** */ + public void finish() { + for (Map.Entry> entry : refsByKey.entrySet()) + entry.setValue(limit(uniqueRefs(entry.getValue()))); + } + + /** */ + public List findSuite(@Nullable String suiteId) { + return find(suiteLookupKey(suiteId)); + } + + /** */ + public List findTest(@Nullable String suiteId, @Nullable Integer testNameId) { + return find(testLookupKey(suiteId, testNameId)); + } + + /** */ + static String suiteLookupKey(@Nullable String suiteId) { + return lookupKey(suiteId, SUITE_ENTITY); + } + + /** */ + static String testLookupKey(@Nullable String suiteId, @Nullable Integer testNameId) { + if (testNameId == null) + return ""; + + return lookupKey(suiteId, "test:" + testNameId); + } + + /** */ + private List find(String lookupKey) { + if (Strings.isNullOrEmpty(lookupKey)) + return new ArrayList<>(); + + List found = refsByKey.get(lookupKey); + + return found == null ? new ArrayList<>() : new ArrayList<>(found); + } + + /** */ + private static String lookupKey(@Nullable String suiteId, String entityKey) { + if (Strings.isNullOrEmpty(suiteId)) + return ""; + + return suiteId + "::" + entityKey; + } + + /** */ + private static List uniqueRefs(List refs) { + Map res = new LinkedHashMap<>(); + + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); + + return new ArrayList<>(res.values()); + } + + /** */ + private static List limit(List refs) { + if (refs.size() <= MAX_REFS) + return refs; + + return new ArrayList<>(refs.subList(0, MAX_REFS)); + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java new file mode 100644 index 000000000..56410579b --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixMatch.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Persisted detected relation between a test/suite name and a fix source. + */ +@Persisted +public class TestFixMatch implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Matched test or suite name. */ + public String entityName; + + /** Matched suite name, when known. */ + @Nullable public String suiteName; + + /** Matched suite id, when known. */ + @Nullable public String suiteId; + + /** Matched test name, when known. */ + @Nullable public String testName; + + /** Compacted test name id, when known. */ + @Nullable public Integer testNameId; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current master TeamCity suite status URL. */ + @Nullable public String currentStatusUrl; + + /** Source type: jira or github. */ + public String sourceType; + + /** Source id. */ + public String sourceId; + + /** Source URL. */ + public String sourceUrl; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated timestamp. */ + @Nullable public Long updatedTs; + + /** Closed/resolved timestamp. */ + @Nullable public Long closedTs; + + /** Commit SHA, when known. */ + @Nullable public String commitSha; + + /** Commit URL, when known. */ + @Nullable public String commitUrl; + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java new file mode 100644 index 000000000..13dffc955 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefUi.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; + +/** + * Link to a ticket or PR mentioning a test/suite fix. + */ +@SuppressWarnings("PublicField") +public class TestFixRefUi { + /** Matched test or suite name. */ + public String entityName; + + /** Suite id. */ + @Nullable public String suiteId; + + /** Suite display name. */ + @Nullable public String suiteName; + + /** Test display name. */ + @Nullable public String testName; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current tracked branch status URL. */ + @Nullable public String currentStatusUrl; + + /** Source type: jira or github. */ + public String sourceType; + + /** Short source text, for example IGNITE-12345 or PR #123. */ + public String text; + + /** Source URL. */ + public String url; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated date. */ + @Nullable public String updatedDate; + + /** Closed/resolved date. */ + @Nullable public String closedDate; + + /** Closing or best-known fix commit URL. */ + @Nullable public String commitUrl; + + /** Closing or best-known fix commit text. */ + @Nullable public String commitText; +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java new file mode 100644 index 000000000..07d12caea --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixRefs.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Reverse lookup from a suite/test spelling to source fix ids. + */ +@Persisted +public class TestFixRefs implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Matched test or suite name. */ + public String entityName; + + /** Matched suite name, when known. */ + @Nullable public String suiteName; + + /** Matched suite id, when known. */ + @Nullable public String suiteId; + + /** Matched test name, when known. */ + @Nullable public String testName; + + /** Compacted test name id, when known. */ + @Nullable public Integer testNameId; + + /** Tracked branch used to resolve this match. */ + @Nullable public String trackedBranch; + + /** Current master TeamCity suite status URL. */ + @Nullable public String currentStatusUrl; + + /** Synthetic source ids. */ + public List sourceIds = new ArrayList<>(); + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java new file mode 100644 index 000000000..44cdc561e --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixSource.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import javax.annotation.Nullable; +import org.apache.ignite.tcbot.persistence.IVersionedEntity; +import org.apache.ignite.tcbot.persistence.Persisted; + +/** + * Persisted fix source data keyed by synthetic source id. + */ +@Persisted +public class TestFixSource implements IVersionedEntity { + /** Entity version. */ + private static final int LATEST_VERSION = 1; + + /** Entity version. */ + @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; + + /** Source type: jira or github. */ + public String sourceType; + + /** Source id. */ + public String sourceId; + + /** Source URL. */ + public String sourceUrl; + + /** Source title. */ + @Nullable public String title; + + /** Source status. */ + @Nullable public String status; + + /** Author name/login. */ + @Nullable public String author; + + /** Author URL. */ + @Nullable public String authorUrl; + + /** Author avatar URL. */ + @Nullable public String authorAvatarUrl; + + /** Updated timestamp. */ + @Nullable public Long updatedTs; + + /** Closed/resolved timestamp. */ + @Nullable public Long closedTs; + + /** Commit SHA, when known. */ + @Nullable public String commitSha; + + /** Commit URL, when known. */ + @Nullable public String commitUrl; + + /** {@inheritDoc} */ + @Override public int version() { + return _ver == null ? -1 : _ver; + } + + /** {@inheritDoc} */ + @Override public int latestVersion() { + return LATEST_VERSION; + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java new file mode 100644 index 000000000..b624dde83 --- /dev/null +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesService.java @@ -0,0 +1,1400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import com.google.common.base.Strings; +import java.time.Instant; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import javax.annotation.Nullable; +import javax.cache.Cache; +import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.ci.github.GitHubBranch; +import org.apache.ignite.ci.github.GitHubUser; +import org.apache.ignite.ci.github.PullRequest; +import org.apache.ignite.ci.teamcity.ignited.buildtype.BuildTypeCompacted; +import org.apache.ignite.ci.teamcity.ignited.buildtype.SnapshotDependencyCompacted; +import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted; +import org.apache.ignite.githubignited.IGitHubConnIgnited; +import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider; +import org.apache.ignite.jiraignited.IJiraIgnited; +import org.apache.ignite.jiraignited.IJiraIgnitedProvider; +import org.apache.ignite.jiraservice.JiraTicketStatusCode; +import org.apache.ignite.jiraservice.Ticket; +import org.apache.ignite.tcbot.common.conf.ITcServerConfig; +import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; +import org.apache.ignite.tcbot.common.interceptor.AutoProfiling; +import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; +import org.apache.ignite.tcbot.engine.conf.ITrackedBranch; +import org.apache.ignite.tcbot.engine.conf.ITrackedChain; +import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.ui.ShortSuiteUi; +import org.apache.ignite.tcbot.engine.ui.ShortTestFailureUi; +import org.apache.ignite.tcbot.persistence.IStringCompactor; +import org.apache.ignite.tcbot.persistence.CacheConfigs; +import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; +import org.apache.ignite.tcbot.persistence.scheduler.MaintenanceActionRegistry; +import org.apache.ignite.tcignited.ITeamcityIgnited; +import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; +import org.apache.ignite.tcignited.history.ISuiteRunHistory; +import org.apache.ignite.tcservice.model.conf.BuildType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch; + +/** + * Matches JIRA tickets and GitHub PRs mentioning tests or suites that are being fixed. + */ +public class TestFixesService { + /** Logger. */ + private static final Logger logger = LoggerFactory.getLogger(TestFixesService.class); + + /** Reverse lookup cache name. */ + private static final String REFS_CACHE_NAME = "testFixRefsByTest"; + + /** Source fix cache name. */ + private static final String SOURCE_CACHE_NAME = "testFixSourcesById"; + + /** Background scheduled task name. */ + private static final String TASK_NAME = TestFixesService.class.getSimpleName() + ".refresh"; + + /** Ignite provider. */ + @Inject private Provider igniteProvider; + + /** Scheduler. */ + @Inject private IScheduler scheduler; + + /** Admin maintenance actions. */ + @Inject private MaintenanceActionRegistry maintenanceActions; + + /** Config. */ + @Inject private ITcBotConfig cfg; + + /** JIRA provider. */ + @Inject private IJiraIgnitedProvider jiraProvider; + + /** GitHub provider. */ + @Inject private IGitHubConnIgnitedProvider ghProvider; + + /** TeamCity provider. */ + @Inject private ITeamcityIgnitedProvider tcProvider; + + /** User-visible process monitor. */ + @Inject private BotProcessMonitor processMonitor; + + /** String compactor. */ + @Inject private IStringCompactor compactor; + + /** Source fix cache keyed by synthetic source id. */ + private volatile IgniteCache sourceCache; + + /** Reverse lookup cache keyed by suite/test spelling. */ + private volatile IgniteCache lookupCache; + + /** + * Starts rare background refresh. + */ + public void start() { + maintenanceActions.register(TASK_NAME, "Refresh JIRA/GitHub test-fix mapping", this::refresh); + + scheduler.invokeLater(this::refreshAndReschedule, 2, TimeUnit.MINUTES); + } + + /** + * Schedules earlier matching after data update signal. + */ + public void requestMatchSoon() { + scheduler.sheduleNamed(TASK_NAME + ".soon", this::safeRefresh, 3, TimeUnit.MINUTES); + } + + /** + * Schedules an immediate manual refresh. + * + * @param processId Optional user-visible process id. + * @return {@code true} if refresh was accepted. + */ + public boolean requestMatchNow(@Nullable Long processId) { + processMonitor.start(processId, "testFixesRefresh", "Test fix mapping refresh queued."); + + boolean accepted = scheduler.runNamedNow(TASK_NAME + ".manual", () -> safeRefresh(processId), processId); + + if (!accepted) + processMonitor.fail(processId, "Test fix mapping refresh is already queued or running."); + + return accepted; + } + + /** + * @param suite Suite UI. + */ + public void decorate(ShortSuiteUi suite) { + if (suite == null) + return; + + decorate(Collections.singletonList(suite)); + } + + /** + * @param suites Suites UI. + */ + public void decorate(Collection suites) { + if (suites == null || suites.isEmpty()) + return; + + TestFixLookupIndex idx = lookupIndex(suites); + + for (ShortSuiteUi suite : suites) + decorate(suite, idx); + } + + /** + * @param suite Suite UI. + * @param idx Test fix lookup index. + */ + private void decorate(ShortSuiteUi suite, TestFixLookupIndex idx) { + if (suite == null) + return; + + suite.fixRefs = idx.findSuite(suite.suiteId); + + for (ShortTestFailureUi failure : suite.testFailures()) + decorate(failure, suite.suiteId, idx); + } + + /** + * @param failure Test failure UI. + */ + public void decorate(ShortTestFailureUi failure) { + if (failure == null) + return; + + Set keys = new LinkedHashSet<>(); + + addLookupKey(keys, TestFixLookupIndex.testLookupKey(failure.suiteId, failure.testNameId)); + + decorate(failure, failure.suiteId, lookupIndex(keys)); + } + + /** + * @param failure Test failure UI. + * @param suiteId Suite id. + * @param idx Test fix lookup index. + */ + private void decorate(ShortTestFailureUi failure, @Nullable String suiteId, TestFixLookupIndex idx) { + if (failure == null) + return; + + failure.fixRefs = idx.findTest(suiteId, failure.testNameId); + } + + /** + * @param limit Max rows. + */ + public List recent(int limit) { + ensureCache(); + + List mappings = StreamSupport.stream(lookupCache.spliterator(), false) + .map(Cache.Entry::getValue) + .collect(Collectors.toList()); + + Map sources = sources(sourceIds(mappings)); + + java.util.stream.Stream stream = mappings.stream() + .flatMap(mapping -> refsFor(mapping, sources).stream()) + .sorted(Comparator.comparingLong((TestFixRefUi ref) -> sortTs(ref)).reversed()); + + if (limit > 0) + stream = stream.limit(limit); + + return uniqueRefs(stream.collect(Collectors.toList())); + } + + /** + * Refreshes match cache. + */ + @AutoProfiling + public String refresh() { + return refresh(null); + } + + /** + * Refreshes match cache. + * + * @param processId Optional user-visible process id. + */ + @AutoProfiling + public String refresh(@Nullable Long processId) { + ensureCache(); + + RefreshStats stats = new RefreshStats(); + RefreshBuffer buffer = new RefreshBuffer(); + + publishRefreshStatus(processId, "collecting test names from tracked suite history", stats); + Map> candidatesByServer = candidatesByServer(); + stats.collectCandidateStats(candidatesByServer); + publishRefreshStatus(processId, "collected test names from tracked suite history", stats); + + for (String srvCode : serverCodes()) { + List srvCandidates = candidatesByServer.get(srvCode); + + if (srvCandidates == null || srvCandidates.isEmpty()) + continue; + + publishRefreshStatus(processId, "loading recent GitHub pull requests for " + srvCode, stats); + List recentPrs = recentPullRequests(srvCode); + stats.githubLoaded += recentPrs.size(); + publishRefreshStatus(processId, "loaded " + recentPrs.size() + " recent GitHub pull requests for " + + srvCode, stats); + + refreshJira(buffer, srvCode, srvCandidates, recentPrs, stats, processId); + refreshGithub(buffer, srvCode, srvCandidates, recentPrs, stats, processId); + } + + replaceCaches(buffer); + + return stats.finishText(); + } + + /** + * Refreshes and schedules next rare run. + */ + private void refreshAndReschedule() { + safeRefresh(); + scheduler.sheduleNamed(TASK_NAME, this::refreshAndReschedule, 6, TimeUnit.HOURS); + } + + /** + * Safe refresh wrapper for background tasks. + */ + private void safeRefresh() { + safeRefresh(null); + } + + /** + * Safe refresh wrapper for background tasks. + * + * @param processId Optional user-visible process id. + */ + private void safeRefresh(@Nullable Long processId) { + try { + String res = refresh(processId); + + processMonitor.finish(processId, res); + logger.info(res); + } + catch (RuntimeException e) { + processMonitor.fail(processId, e); + logger.warn("Failed to refresh test fix matches", e); + } + } + + /** + * @param srvCode Server code. + * @param recentPrs Recent GitHub pull requests. + */ + private void refreshJira(RefreshBuffer buffer, String srvCode, List candidates, + List recentPrs, RefreshStats stats, @Nullable Long processId) { + try { + IJiraIgnited jira = jiraProvider.server(srvCode); + IJiraServerConfig jiraCfg = jira.config(); + String label = jiraCfg.testFixesLabel(); + long cutoffTs = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(jiraCfg.testFixesLookbackDays()); + + publishRefreshStatus(processId, "checking JIRA tickets for " + srvCode, stats); + for (Ticket ticket : jira.getTickets()) { + if (ticket == null || ticket.fields == null) + continue; + + stats.jiraChecked++; + + if (stats.jiraChecked % 100 == 0) + publishRefreshStatus(processId, "checked JIRA tickets for " + srvCode, stats); + + Long updatedTs = parseDate(ticket.fields.updated()); + Long closedTs = parseDate(ticket.fields.resolutionDate()); + boolean labeled = ticket.fields.labels().stream().anyMatch(label::equalsIgnoreCase); + + if (!labeled && updatedTs != null && updatedTs < cutoffTs) + continue; + + String text = Strings.nullToEmpty(ticket.fields.summary()) + "\n" + + Strings.nullToEmpty(ticket.fields.description()); + + List linkedPrs = relatedPullRequests(ticket, recentPrs); + + for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { + TestFixMatch match = candidate.newMatch(); + match.sourceType = "jira"; + match.sourceId = ticket.key; + match.sourceUrl = jira.generateTicketUrl(ticket.key); + match.title = ticket.fields.summary(); + match.status = statusText(ticket.status()); + match.updatedTs = updatedTs; + match.closedTs = closedTs; + + saveMatch(buffer, match); + stats.jiraSaved++; + + for (PullRequest pr : linkedPrs) { + TestFixMatch linkedPrMatch = githubMatch(candidate, pr); + + saveMatch(buffer, linkedPrMatch); + stats.githubSaved++; + } + } + } + + publishRefreshStatus(processId, "checked JIRA tickets for " + srvCode, stats); + } + catch (RuntimeException e) { + logger.debug("Skipping JIRA test fix matching for " + srvCode, e); + publishRefreshStatus(processId, "skipped JIRA tickets for " + srvCode + ": " + e.getMessage(), stats); + } + } + + /** + * @param srvCode Server code. + * @param recentPrs Recent GitHub pull requests. + */ + private void refreshGithub(RefreshBuffer buffer, String srvCode, List candidates, + List recentPrs, RefreshStats stats, @Nullable Long processId) { + try { + publishRefreshStatus(processId, "checking GitHub pull requests for " + srvCode, stats); + for (PullRequest pr : recentPrs) { + stats.githubChecked++; + + if (stats.githubChecked % 25 == 0) + publishRefreshStatus(processId, "checked GitHub pull requests for " + srvCode, stats); + + String text = Strings.nullToEmpty(pr.getTitle()) + "\n" + Strings.nullToEmpty(pr.getBody()); + + for (TestFixCandidate candidate : matchingCandidates(text, candidates)) { + TestFixMatch match = githubMatch(candidate, pr); + + saveMatch(buffer, match); + stats.githubSaved++; + } + } + + publishRefreshStatus(processId, "checked GitHub pull requests for " + srvCode, stats); + } + catch (RuntimeException e) { + logger.debug("Skipping GitHub test fix matching for " + srvCode, e); + publishRefreshStatus(processId, "skipped GitHub pull requests for " + srvCode + ": " + e.getMessage(), + stats); + } + } + + /** + * @param srvCode Server code. + */ + private List recentPullRequests(String srvCode) { + try { + IJiraServerConfig jiraCfg = cfg.getJiraConfig(srvCode); + IGitHubConnIgnited gh = ghProvider.server(srvCode); + + return gh.getRecentPullRequests(jiraCfg.testFixesLookbackDays()); + } + catch (RuntimeException e) { + logger.debug("Skipping GitHub PR loading for test fix matching for " + srvCode, e); + + return new ArrayList<>(); + } + } + + /** + * @param candidate Candidate. + * @param pr Pull request. + */ + private static TestFixMatch githubMatch(TestFixCandidate candidate, PullRequest pr) { + TestFixMatch match = candidate.newMatch(); + + match.sourceType = "github"; + match.sourceId = "PR #" + pr.getNumber(); + match.sourceUrl = pr.htmlUrl(); + match.title = pr.getTitle(); + match.status = pr.getState(); + match.updatedTs = parseDate(pr.getTimeUpdate()); + match.closedTs = parseDate(pr.mergedAt()); + + fillGithubAuthor(match, pr.gitHubUser()); + fillCommit(match, pr); + + return match; + } + + /** + * @param ticket JIRA ticket. + * @param prs Recent pull requests. + */ + private static List relatedPullRequests(Ticket ticket, List prs) { + if (ticket == null || Strings.isNullOrEmpty(ticket.key) || prs.isEmpty()) + return new ArrayList<>(); + + String ticketKey = ticket.key.toUpperCase(Locale.ROOT); + + return prs.stream() + .filter(pr -> mentionsTicket(pr, ticketKey)) + .collect(Collectors.toList()); + } + + /** + * @param pr Pull request. + * @param ticketKey Uppercase JIRA ticket key. + */ + private static boolean mentionsTicket(PullRequest pr, String ticketKey) { + GitHubBranch head = pr.head(); + + return containsIgnoreCase(pr.getTitle(), ticketKey) || + containsIgnoreCase(pr.getBody(), ticketKey) || + (head != null && containsIgnoreCase(head.ref(), ticketKey)); + } + + /** + * @param text Text. + * @param token Uppercase token. + */ + private static boolean containsIgnoreCase(@Nullable String text, String token) { + return !Strings.isNullOrEmpty(text) && text.toUpperCase(Locale.ROOT).contains(token); + } + + /** + * @param match Match. + * @param user GitHub user. + */ + private static void fillGithubAuthor(TestFixMatch match, @Nullable GitHubUser user) { + if (user == null) + return; + + match.author = user.login(); + match.authorUrl = Strings.isNullOrEmpty(user.login()) ? null : "https://github.com/" + user.login(); + match.authorAvatarUrl = user.avatarUrl(); + } + + /** + * @param match Match. + * @param pr Pull request. + */ + private static void fillCommit(TestFixMatch match, PullRequest pr) { + String sha = pr.mergeCommitSha(); + + if (Strings.isNullOrEmpty(sha)) { + GitHubBranch head = pr.head(); + + if (head == null || Strings.isNullOrEmpty(head.sha())) + return; + + sha = head.sha(); + } + + match.commitSha = sha; + match.commitUrl = commitUrl(pr.htmlUrl(), sha); + } + + /** + * @param prUrl Pull request URL. + * @param sha Commit SHA. + */ + @Nullable private static String commitUrl(@Nullable String prUrl, String sha) { + if (Strings.isNullOrEmpty(prUrl)) + return null; + + int pullIdx = prUrl.indexOf("/pull/"); + + if (pullIdx < 0) + return null; + + return prUrl.substring(0, pullIdx) + "/commit/" + sha; + } + + /** + * @param refs Refs. + */ + private static List uniqueRefs(List refs) { + Map res = new HashMap<>(); + + for (TestFixRefUi ref : refs) + res.putIfAbsent(ref.sourceType + ":" + ref.text, ref); + + return new ArrayList<>(res.values()); + } + + /** + * @param refs Reverse refs. + * @param source Source. + */ + private TestFixRefUi toUi(TestFixRefs refs, TestFixSource source) { + TestFixRefUi res = new TestFixRefUi(); + + res.entityName = refs.entityName; + res.suiteId = refs.suiteId; + res.suiteName = refs.suiteName; + res.testName = refs.testName; + res.trackedBranch = refs.trackedBranch; + res.currentStatusUrl = refs.currentStatusUrl; + res.sourceType = source.sourceType; + res.text = source.sourceId; + res.url = source.sourceUrl; + res.title = source.title; + res.status = source.status; + res.author = source.author; + res.authorUrl = source.authorUrl; + res.authorAvatarUrl = source.authorAvatarUrl; + res.updatedDate = formatDate(source.updatedTs); + res.closedDate = formatDate(source.closedTs); + res.commitUrl = source.commitUrl; + res.commitText = Strings.isNullOrEmpty(source.commitSha) ? null : shortSha(source.commitSha); + + return res; + } + + /** + * @param status Status. + */ + private static String statusText(@Nullable JiraTicketStatusCode status) { + return JiraTicketStatusCode.text(status); + } + + /** + * @param sha SHA. + */ + private static String shortSha(String sha) { + return sha.length() <= PullRequest.INCLUDE_SHORT_VER ? sha : sha.substring(0, PullRequest.INCLUDE_SHORT_VER); + } + + /** + * @param match Match. + */ + private void saveMatch(RefreshBuffer buffer, TestFixMatch match) { + if (!isPlausibleMatch(match)) + return; + + String sourceId = sourceKey(match); + + buffer.sources.put(sourceId, source(match)); + + for (String key : lookupKeys(match)) { + TestFixRefs refs = buffer.refs.get(key); + + if (refs == null) + refs = refs(match); + + if (!refs.sourceIds.contains(sourceId)) + refs.sourceIds.add(sourceId); + + buffer.refs.put(key, refs); + } + } + + /** + * Replaces persistent match caches with the latest successful refresh snapshot. + * + * @param buffer Newly collected mappings. + */ + private void replaceCaches(RefreshBuffer buffer) { + lookupCache.removeAll(); + sourceCache.removeAll(); + + if (!buffer.refs.isEmpty()) + lookupCache.putAll(buffer.refs); + + if (!buffer.sources.isEmpty()) + sourceCache.putAll(buffer.sources); + } + + /** + * @param match Match. + */ + private static TestFixSource source(TestFixMatch match) { + TestFixSource res = new TestFixSource(); + + res.sourceType = match.sourceType; + res.sourceId = match.sourceId; + res.sourceUrl = match.sourceUrl; + res.title = match.title; + res.status = match.status; + res.author = match.author; + res.authorUrl = match.authorUrl; + res.authorAvatarUrl = match.authorAvatarUrl; + res.updatedTs = match.updatedTs; + res.closedTs = match.closedTs; + res.commitSha = match.commitSha; + res.commitUrl = match.commitUrl; + + return res; + } + + /** + * @param match Match. + */ + private static TestFixRefs refs(TestFixMatch match) { + TestFixRefs res = new TestFixRefs(); + + res.entityName = match.entityName; + res.suiteId = match.suiteId; + res.suiteName = match.suiteName; + res.testName = match.testName; + res.testNameId = match.testNameId; + res.trackedBranch = match.trackedBranch; + res.currentStatusUrl = match.currentStatusUrl; + + return res; + } + + /** + * @param match Match. + */ + private static String sourceKey(TestFixMatch match) { + return normalize(match.sourceType) + ":" + normalize(match.sourceId); + } + + /** + * @param match Match. + */ + private static Set lookupKeys(TestFixMatch match) { + Set res = new LinkedHashSet<>(); + + addLookupKey(res, TestFixLookupIndex.suiteLookupKey(match.suiteId)); + addLookupKey(res, TestFixLookupIndex.testLookupKey(match.suiteId, match.testNameId)); + + return res; + } + + /** + * @param res Target keys. + * @param key Lookup key. + */ + private static void addLookupKey(Set res, String key) { + if (!Strings.isNullOrEmpty(key)) + res.add(key); + } + + /** + * @param suites Suites UI. + */ + private TestFixLookupIndex lookupIndex(Collection suites) { + Set keys = new LinkedHashSet<>(); + + for (ShortSuiteUi suite : suites) + collectLookupKeys(suite, keys); + + return lookupIndex(keys); + } + + /** + * @param lookupKeys Requested lookup keys. + */ + private TestFixLookupIndex lookupIndex(Set lookupKeys) { + ensureCache(); + + TestFixLookupIndex idx = new TestFixLookupIndex(); + + if (lookupKeys.isEmpty()) + return idx; + + Map mappings = lookupCache.getAll(lookupKeys); + Map sources = sources(sourceIds(mappings.values())); + + for (Map.Entry entry : mappings.entrySet()) + idx.add(entry.getKey(), refsFor(entry.getValue(), sources)); + + idx.finish(); + + return idx; + } + + /** + * @param suite Suite UI. + * @param keys Target keys. + */ + private static void collectLookupKeys(ShortSuiteUi suite, Set keys) { + if (suite == null) + return; + + addLookupKey(keys, TestFixLookupIndex.suiteLookupKey(suite.suiteId)); + + for (ShortTestFailureUi failure : suite.testFailures()) { + addLookupKey(keys, TestFixLookupIndex.testLookupKey(suite.suiteId, failure.testNameId)); + } + } + + /** + * @param mappings Reverse mappings. + */ + private static Set sourceIds(Collection mappings) { + Set res = new LinkedHashSet<>(); + + for (TestFixRefs mapping : mappings) { + if (mapping != null && mapping.sourceIds != null) + res.addAll(mapping.sourceIds); + } + + return res; + } + + /** + * @param sourceIds Source ids. + */ + private Map sources(Set sourceIds) { + return sourceIds.isEmpty() ? Collections.emptyMap() : sourceCache.getAll(sourceIds); + } + + /** + * @param refs Reverse refs. + * @param sources Source fixes by id. + */ + private List refsFor(TestFixRefs refs, Map sources) { + List res = new ArrayList<>(); + + if (refs == null || refs.sourceIds == null) + return res; + + for (String sourceId : refs.sourceIds) { + TestFixSource source = sources.get(sourceId); + + if (source != null) + res.add(toUi(refs, source)); + } + + return res.stream() + .sorted(Comparator.comparingLong((TestFixRefUi ref) -> sortTs(ref)).reversed()) + .collect(Collectors.toList()); + } + + /** + * @param value Value. + */ + private static String normalize(@Nullable String value) { + return Strings.nullToEmpty(value).trim().toLowerCase(Locale.ROOT); + } + + /** + * @param match Match. + */ + private static long sortTs(TestFixMatch match) { + if (match.closedTs != null) + return match.closedTs; + + if (match.updatedTs != null) + return match.updatedTs; + + return 0; + } + + /** + * @param ref UI ref. + */ + private static long sortTs(TestFixRefUi ref) { + long closed = dateScore(ref.closedDate); + + if (closed > 0) + return closed; + + return dateScore(ref.updatedDate); + } + + /** + * @param date ISO local date. + */ + private static long dateScore(@Nullable String date) { + if (Strings.isNullOrEmpty(date)) + return 0; + + try { + return LocalDate.parse(date).toEpochDay(); + } + catch (DateTimeParseException ignored) { + return 0; + } + } + + /** + * @param date Date string. + */ + @Nullable private static Long parseDate(@Nullable String date) { + if (Strings.isNullOrEmpty(date)) + return null; + + try { + return Instant.parse(date).toEpochMilli(); + } + catch (DateTimeParseException ignored) { + // Try JIRA offset format below. + } + + try { + return OffsetDateTime.parse(date, DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")) + .toInstant().toEpochMilli(); + } + catch (DateTimeParseException ignored) { + return null; + } + } + + /** + * @param ts Timestamp. + */ + @Nullable private static String formatDate(@Nullable Long ts) { + if (ts == null) + return null; + + return DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.ofEpochMilli(ts).atZone(java.time.ZoneOffset.UTC)); + } + + /** + * @return Server codes. + */ + private Collection serverCodes() { + Set res = new HashSet<>(cfg.getServerIds()); + + if (!Strings.isNullOrEmpty(cfg.primaryServerCode())) + res.add(cfg.primaryServerCode()); + + return res; + } + + /** + * @return Match candidates grouped by TeamCity/GitHub/JIRA server code. + */ + private Map> candidatesByServer() { + Map> res = new HashMap<>(); + + cfg.getTrackedBranches().branchesStream().forEach(branch -> collectCandidates(branch, res)); + + return res; + } + + /** + * @param branch Tracked branch. + * @param res Target map. + */ + private void collectCandidates(ITrackedBranch branch, Map> res) { + branch.chainsStream().forEach(chain -> { + try { + res.computeIfAbsent(chain.serverCode(), k -> new ArrayList<>()) + .addAll(candidates(branch, chain)); + } + catch (RuntimeException e) { + logger.debug("Failed to collect test fix candidates [branch={}, server={}, suite={}]", + branch.name(), chain.serverCode(), chain.tcSuiteId(), e); + } + }); + } + + /** + * @param branch Tracked branch. + * @param chain Tracked chain. + */ + private List candidates(ITrackedBranch branch, ITrackedChain chain) { + ITeamcityIgnited tc = tcProvider.server(chain.serverCode(), null); + String baseBranch = chain.tcBaseBranch().orElse(chain.tcBranch()); + Integer baseBranchId = compactor.getStringIdIfPresent(normalizeBranch(baseBranch)); + + if (baseBranchId == null) + return new ArrayList<>(); + + List res = new ArrayList<>(); + + for (String suiteId : suiteIdsForHistory(tc, chain.tcSuiteId(), baseBranch)) + res.addAll(candidates(branch, tc, baseBranchId, suiteId)); + + return res; + } + + /** + * @param branch Tracked branch. + * @param tc TeamCity storage. + * @param baseBranchId Compacted base branch id. + * @param suiteId TeamCity build configuration id. + */ + private List candidates(ITrackedBranch branch, ITeamcityIgnited tc, Integer baseBranchId, + String suiteId) { + Integer compactedSuiteId = compactor.getStringIdIfPresent(suiteId); + + if (compactedSuiteId == null) + return new ArrayList<>(); + + ISuiteRunHistory hist = tc.getSuiteRunHist(compactedSuiteId, baseBranchId); + + if (hist == null) + return new ArrayList<>(); + + List res = new ArrayList<>(); + + for (Integer testNameId : hist.testNames()) { + String fullTestName = compactor.getStringFromId(testNameId); + + if (Strings.isNullOrEmpty(fullTestName)) + continue; + + String shortTestName = ShortTestFailureUi.extractTest(fullTestName); + + if (!isPlausibleTestName(shortTestName)) + continue; + + String suiteName = testSuiteName(fullTestName, shortTestName); + + String currentStatusUrl = tc.host() + "buildConfiguration/" + suiteId + "?branch=" + + org.apache.ignite.tcbot.common.util.UrlUtil.escape(ITcServerConfig.DEFAULT_TRACKED_BRANCH_NAME); + + res.add(new TestFixCandidate(branch.name(), suiteId, suiteName, testNameId, fullTestName, shortTestName, + currentStatusUrl)); + } + + return res; + } + + /** + * @param tc TeamCity storage. + * @param rootSuiteId Root tracked chain build configuration id. + * @param baseBranch Base branch in TeamCity history. + */ + private List suiteIdsForHistory(ITeamcityIgnited tc, String rootSuiteId, String baseBranch) { + LinkedHashSet res = new LinkedHashSet<>(); + + collectSuiteIds(tc, rootSuiteId, res, new HashSet<>()); + collectSuiteIdsFromRecentBuilds(tc, rootSuiteId, baseBranch, res); + + return new ArrayList<>(res); + } + + /** + * @param tc TeamCity storage. + * @param suiteId TeamCity build configuration id. + * @param res Collected suite ids. + * @param visited Visited build configurations. + */ + private void collectSuiteIds(ITeamcityIgnited tc, @Nullable String suiteId, LinkedHashSet res, + Set visited) { + if (Strings.isNullOrEmpty(suiteId) || !visited.add(suiteId)) + return; + + res.add(suiteId); + + BuildTypeCompacted buildType = tc.getBuildType(suiteId); + + if (buildType == null) + return; + + for (SnapshotDependencyCompacted dep : buildType.snapshotDependencies()) { + BuildType depBuildType = dep.toSnapshotDependency(compactor).bt(); + + if (depBuildType != null) + collectSuiteIds(tc, depBuildType.getId(), res, visited); + } + } + + /** + * @param tc TeamCity storage. + * @param rootSuiteId Root tracked chain build configuration id. + * @param baseBranch Base branch in TeamCity history. + * @param res Collected suite ids. + */ + private void collectSuiteIdsFromRecentBuilds(ITeamcityIgnited tc, String rootSuiteId, String baseBranch, + LinkedHashSet res) { + try { + for (Integer buildId : tc.getLastNBuildsFromHistory(rootSuiteId, baseBranch, 3)) + collectSuiteIdsFromBuild(tc, buildId, res, new HashSet<>()); + } + catch (RuntimeException e) { + logger.debug("Failed to collect test fix suite ids from recent builds [suite={}, branch={}]", + rootSuiteId, baseBranch, e); + } + } + + /** + * @param tc TeamCity storage. + * @param buildId Build id. + * @param res Collected suite ids. + * @param visitedBuilds Visited build ids. + */ + private void collectSuiteIdsFromBuild(ITeamcityIgnited tc, @Nullable Integer buildId, LinkedHashSet res, + Set visitedBuilds) { + if (buildId == null || !visitedBuilds.add(buildId)) + return; + + FatBuildCompacted build = tc.getFatBuild(buildId); + + if (build == null || build.isFakeStub()) + return; + + String suiteId = build.buildTypeId(compactor); + + if (!Strings.isNullOrEmpty(suiteId)) + res.add(suiteId); + + for (int depId : build.snapshotDependencies()) + collectSuiteIdsFromBuild(tc, depId, res, visitedBuilds); + } + + /** + * @param fullTestName Full TeamCity test name. + * @param shortTestName Short test name in Class.method form. + */ + @Nullable private static String testSuiteName(String fullTestName, @Nullable String shortTestName) { + if (!Strings.isNullOrEmpty(shortTestName)) { + int dot = shortTestName.lastIndexOf('.'); + + if (dot > 0) + return shortTestName.substring(0, dot); + } + + return ShortTestFailureUi.extractSuite(fullTestName); + } + + /** + * @param text Source title/body/description. + * @param candidates Candidates from tracked branch histories. + */ + static Collection matchingCandidates(String text, List candidates) { + String rawSrc = normalize(text); + String src = mentionKey(rawSrc); + Map res = new LinkedHashMap<>(); + + for (TestFixCandidate candidate : candidates) { + if (candidate.matchesQualified(src)) + res.putIfAbsent(candidate.key(), candidate); + } + + if (!res.isEmpty()) + return res.values(); + + for (TestFixCandidate candidate : candidates) { + if (candidate.matchesUnqualified(src, rawSrc)) + res.putIfAbsent(candidate.key(), candidate); + } + + return res.values(); + } + + /** + * @param match Persisted match. + */ + private static boolean isPlausibleMatch(TestFixMatch match) { + return match != null && isPlausibleTestName(match.testName) && !Strings.isNullOrEmpty(match.suiteId); + } + + /** + * @param testName Short test name in Class.method form. + */ + static boolean isPlausibleTestName(@Nullable String testName) { + if (Strings.isNullOrEmpty(testName)) + return false; + + return testName.matches("[A-Za-z_$][A-Za-z0-9_$]*[.][A-Za-z_$][A-Za-z0-9_$]*(?:\\[[^\\]]+\\])?"); + } + + /** + * @param value Source text or a candidate alias. + */ + private static String mentionKey(@Nullable String value) { + String normalized = normalize(value); + StringBuilder res = new StringBuilder(normalized.length()); + boolean lastSeparator = true; + + for (int i = 0; i < normalized.length(); i++) { + char ch = normalized.charAt(i); + + if (Character.isLetterOrDigit(ch)) { + res.append(ch); + lastSeparator = false; + } + else if (!lastSeparator) { + res.append('.'); + lastSeparator = true; + } + } + + int len = res.length(); + + if (len > 0 && res.charAt(len - 1) == '.') + res.deleteCharAt(len - 1); + + return res.toString(); + } + + /** + * @param src Canonical source text. + * @param token Source token. + */ + private static boolean containsMention(String src, @Nullable String token) { + if (Strings.isNullOrEmpty(token)) + return false; + + String needle = mentionKey(token); + + if (Strings.isNullOrEmpty(needle)) + return false; + + int start = 0; + + while (start <= src.length() - needle.length()) { + int idx = src.indexOf(needle, start); + + if (idx < 0) + return false; + + int end = idx + needle.length(); + boolean leftBoundary = idx == 0 || src.charAt(idx - 1) == '.'; + boolean rightBoundary = end == src.length() || src.charAt(end) == '.'; + + if (leftBoundary && rightBoundary) + return true; + + start = idx + 1; + } + + return false; + } + + /** + * @param src Normalized source text. + * @param token Standalone name token. + */ + private static boolean containsStandaloneName(String src, @Nullable String token) { + String needle = normalize(token); + + if (Strings.isNullOrEmpty(needle)) + return false; + + int start = 0; + + while (start <= src.length() - needle.length()) { + int idx = src.indexOf(needle, start); + + if (idx < 0) + return false; + + int end = idx + needle.length(); + boolean leftBoundary = idx == 0 || !Character.isLetterOrDigit(src.charAt(idx - 1)); + boolean rightBoundary = end == src.length() || !Character.isLetterOrDigit(src.charAt(end)); + boolean followedByMethod = end + 1 < src.length() && (src.charAt(end) == '.' || src.charAt(end) == '#') && + Character.isLetterOrDigit(src.charAt(end + 1)); + + if (leftBoundary && rightBoundary && !followedByMethod) + return true; + + start = idx + 1; + } + + return false; + } + + /** + * Initializes cache. + */ + private void ensureCache() { + if (sourceCache != null) + return; + + synchronized (this) { + if (sourceCache == null) { + sourceCache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig(SOURCE_CACHE_NAME)); + lookupCache = igniteProvider.get().getOrCreateCache( + CacheConfigs.getCache8PartsConfig(REFS_CACHE_NAME)); + } + } + } + + /** + * @param processId Optional user-visible process id. + * @param stage Current refresh stage. + * @param stats Refresh stats. + */ + private void publishRefreshStatus(@Nullable Long processId, String stage, RefreshStats stats) { + processMonitor.status(processId, "Test fix mapping: " + stage + ". " + stats.progressText()); + } + + /** In-memory refresh result applied to persistent caches after refresh completion. */ + private static class RefreshBuffer { + /** Source fixes by id. */ + private final Map sources = new HashMap<>(); + + /** Lookup refs by suite/test lookup key. */ + private final Map refs = new HashMap<>(); + } + + /** Test fix refresh counters. */ + private static class RefreshStats { + /** Candidate test names from tracked suite histories. */ + private int candidates; + + /** Unique suites with history candidates. */ + private int suites; + + /** Unique full test names with history candidates. */ + private int tests; + + /** Recent GitHub PRs loaded from cache. */ + private int githubLoaded; + + /** GitHub PRs checked against candidates. */ + private int githubChecked; + + /** JIRA tickets checked against candidates. */ + private int jiraChecked; + + /** Saved JIRA matches. */ + private int jiraSaved; + + /** Saved GitHub matches. */ + private int githubSaved; + + /** + * @param candidatesByServer Candidates by server. + */ + private void collectCandidateStats(Map> candidatesByServer) { + Set suiteIds = new HashSet<>(); + Set testNames = new HashSet<>(); + + for (Map.Entry> entry : candidatesByServer.entrySet()) { + for (TestFixCandidate candidate : entry.getValue()) { + candidates++; + suiteIds.add(entry.getKey() + ":" + candidate.suiteId); + testNames.add(candidate.fullTestName); + } + } + + suites = suiteIds.size(); + tests = testNames.size(); + } + + /** */ + private int saved() { + return jiraSaved + githubSaved; + } + + /** */ + private String progressText() { + return "Analyzed " + tests + " unique test names (" + candidates + " candidate rows) in " + suites + + " suites. Checked sources: GitHub " + githubChecked + "/" + githubLoaded + " PRs, JIRA " + + jiraChecked + " tickets. Matches saved: " + saved() + " (JIRA " + jiraSaved + ", GitHub " + + githubSaved + ")."; + } + + /** */ + private String finishText() { + return "Test fix matches refreshed: " + saved() + " (tests=" + tests + ", candidates=" + candidates + + ", suites=" + suites + ", githubChecked=" + githubChecked + "/" + githubLoaded + + ", jiraChecked=" + jiraChecked + ", jira=" + jiraSaved + ", github=" + githubSaved + ")"; + } + } + + /** Test fix candidate resolved from tracked branch suite history. */ + static class TestFixCandidate { + /** Tracked branch. */ + private final String trackedBranch; + + /** Suite id. */ + private final String suiteId; + + /** Suite display name. */ + private final String suiteName; + + /** Compacted full test name id. */ + private final Integer testNameId; + + /** Full test name. */ + private final String fullTestName; + + /** Short test name. */ + private final String shortTestName; + + /** Current master TeamCity suite status URL. */ + private final String currentStatusUrl; + + /** + * @param trackedBranch Tracked branch. + * @param suiteId Suite id. + * @param suiteName Suite display name. + * @param testNameId Compacted full test name id. + * @param fullTestName Full test name. + * @param shortTestName Short test name. + */ + TestFixCandidate(String trackedBranch, String suiteId, @Nullable String suiteName, Integer testNameId, + String fullTestName, @Nullable String shortTestName, String currentStatusUrl) { + this.trackedBranch = trackedBranch; + this.suiteId = suiteId; + this.suiteName = suiteName; + this.testNameId = testNameId; + this.fullTestName = fullTestName; + this.shortTestName = shortTestName; + this.currentStatusUrl = currentStatusUrl; + } + + /** + * @param src Canonical source text. + */ + private boolean matchesQualified(String src) { + return containsMention(src, suiteId + "." + shortTestName) || + containsMention(src, suiteId + "." + fullTestName); + } + + /** + * @param src Canonical source text. + * @param rawSrc Normalized source text. + */ + private boolean matchesUnqualified(String src, String rawSrc) { + return containsMention(src, fullTestName) || + containsMention(src, shortTestName); + } + + /** */ + private TestFixMatch newMatch() { + TestFixMatch match = new TestFixMatch(); + + match.entityName = Strings.isNullOrEmpty(shortTestName) ? fullTestName : shortTestName; + match.trackedBranch = trackedBranch; + match.suiteId = suiteId; + match.suiteName = suiteName; + match.testName = Strings.isNullOrEmpty(shortTestName) ? fullTestName : shortTestName; + match.testNameId = testNameId; + match.currentStatusUrl = currentStatusUrl; + + return match; + } + + /** */ + private String key() { + return normalize(suiteId) + ":" + normalize(fullTestName); + } + + } +} diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java index 13baae23d..9a85bfe5b 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/IDetailedStatusForTrackedBranch.java @@ -37,6 +37,7 @@ public interface IDetailedStatusForTrackedBranch { * @param calcTrustedTests Calculate trusted tests count. * @param tagSelected Selected tag based filter. If null or empty all data is returned. * @param tagForHistSelected Selected tag for filtering history (applicable to reruns and history stripe). + * @param suiteId Suite id to include. If null or empty all suites are returned. * @param displayMode Suites and tests display mode. Default - failures only. * @param sortOption Sort mode * @param maxDurationSec Show test as failed if duration is greater than provided seconds count. @@ -52,6 +53,7 @@ public DsSummaryUi getTrackedBranchTestFailures( boolean calcTrustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nullable DisplayMode displayMode, @Nullable SortOption sortOption, int maxDurationSec, diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java index a63688387..c200ea2be 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/tracked/TrackedBranchChainsProcessor.java @@ -53,8 +53,10 @@ import org.apache.ignite.tcbot.engine.conf.ITrackedChain; import org.apache.ignite.tcbot.engine.pool.TcUpdatePool; import org.apache.ignite.tcbot.engine.process.BotProcessMonitor; +import org.apache.ignite.tcbot.engine.testfixes.TestFixesService; import org.apache.ignite.tcbot.engine.ui.DsChainUi; import org.apache.ignite.tcbot.engine.ui.DsSummaryUi; +import org.apache.ignite.tcbot.engine.ui.DsSuiteUi; import org.apache.ignite.tcbot.engine.ui.GuardBranchStatusUi; import org.apache.ignite.tcbot.engine.ui.LrTestsFullSummaryUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; @@ -112,6 +114,9 @@ public class TrackedBranchChainsProcessor implements IDetailedStatusForTrackedBr /** User-visible process monitor. */ @Inject private BotProcessMonitor processMonitor; + /** Test fix matcher. */ + @Inject private TestFixesService testFixesService; + /** * @param branch Branch. * @param buildResMergeCnt Build results merge count. @@ -356,6 +361,7 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { boolean calcTrustedTests, @Nullable String tagSelected, @Nullable String tagForHistSelected, + @Nullable String suiteId, @Nullable DisplayMode displayMode, @Nullable SortOption sortOption, int maxDurationSec, @@ -443,9 +449,13 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests, tagSelected, displayMode, maxDurationSec, requireParamVal, showMuted, showIgnored); + filterSuites(chainStatus, suiteId); + chainStatus.suites.forEach(testFixesService::decorate); uiInitNanos += System.nanoTime() - stepStart; - res.addChainOnServer(chainStatus); + if (Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainStatus.suiteId) || !chainStatus.suites.isEmpty()) + res.addChainOnServer(chainStatus); + chainTotalNanos += System.nanoTime() - chainStart; } @@ -470,6 +480,47 @@ private void promptStatus(@Nullable Long processId, long reqId, String text) { return res; } + /** + * Keeps a requested child suite inside the loaded root tracked chain. + * + * @param chainStatus Chain UI. + * @param suiteId Optional requested suite id. + */ + private static void filterSuites(DsChainUi chainStatus, @Nullable String suiteId) { + if (chainStatus == null || Strings.isNullOrEmpty(suiteId) || suiteId.equals(chainStatus.suiteId)) + return; + + chainStatus.suites = chainStatus.suites.stream() + .filter(suite -> suiteId.equals(suite.suiteId)) + .collect(Collectors.toList()); + + chainStatus.failedTests = sum(chainStatus.suites.stream().map(suite -> suite.failedTests) + .collect(Collectors.toList())); + chainStatus.totalTests = sum(chainStatus.suites.stream().map(suite -> suite.totalTests) + .collect(Collectors.toList())); + chainStatus.trustedTests = sum(chainStatus.suites.stream().map(suite -> suite.trustedTests) + .collect(Collectors.toList())); + chainStatus.totalBlockers = chainStatus.suites.stream().mapToInt(DsSuiteUi::totalBlockers).sum(); + } + + /** + * @param vals Values. + */ + private static Integer sum(Collection vals) { + int res = 0; + boolean found = false; + + for (Integer val : vals) { + if (val == null) + continue; + + res += val; + found = true; + } + + return found ? res : null; + } + /** * @param nanos Nanoseconds. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java index 8f042a742..637f7e975 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortSuiteUi.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; import org.apache.ignite.tcbot.persistence.IStringCompactor; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistory; @@ -30,6 +31,9 @@ import static org.apache.ignite.tcbot.engine.ui.DsSuiteUi.buildWebLinkToBuild; public class ShortSuiteUi extends DsHistoryStatUi { + /** Suite build type id. */ + @Nullable public String suiteId; + /** Suite Name */ public String name; @@ -44,6 +48,9 @@ public class ShortSuiteUi extends DsHistoryStatUi { public List testShortFailures = new ArrayList<>(); + /** Matched tickets/PRs that mention fixing this suite. */ + public List fixRefs = new ArrayList<>(); + /** Web Href. to suite particular run */ public String webToBuild = ""; @@ -67,6 +74,7 @@ public ShortSuiteUi initFrom(@Nonnull MultBuildRunCtx suite, ITeamcityIgnited tcIgnited, IStringCompactor compactor, IRunHistory baseBranchHist) { + suiteId = suite.suiteId(); name = suite.suiteName(); result = suite.getResult(); webToBuild = buildWebLinkToBuild(tcIgnited, suite); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java index a1a8560eb..1f9064e78 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/ui/ShortTestFailureUi.java @@ -17,11 +17,14 @@ package org.apache.ignite.tcbot.engine.ui; import com.google.common.base.Strings; +import java.util.ArrayList; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.apache.ignite.tcbot.engine.chain.TestCompactedMult; +import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi; import org.apache.ignite.tcignited.ITeamcityIgnited; import org.apache.ignite.tcignited.history.IRunHistory; @@ -31,6 +34,12 @@ public class ShortTestFailureUi { /** Test full Name */ public String name; + /** Compacted test name id. */ + @Nullable public Integer testNameId; + + /** Suite build type id. */ + @Nullable public String suiteId; + /** suite (in code) short name */ @Nullable public String suiteName; @@ -41,6 +50,9 @@ public class ShortTestFailureUi { /** Blocker comment: indicates test seems to be introduced failure. */ @Nullable public String blockerComment; + /** Matched tickets/PRs that mention fixing this test. */ + public List fixRefs = new ArrayList<>(); + /** * */ @@ -50,6 +62,8 @@ public boolean isPossibleBlocker() { public ShortTestFailureUi initFrom(@Nonnull TestCompactedMult failure, ITeamcityIgnited tcIgn, Integer baseBranchId) { + testNameId = failure.testName(); + suiteId = failure.suiteId(); name = failure.getName(); String[] split = Strings.nullToEmpty(name).split("\\:"); diff --git a/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java new file mode 100644 index 000000000..89c838f45 --- /dev/null +++ b/tcbot-engine/src/test/java/org/apache/ignite/tcbot/engine/testfixes/TestFixesServiceTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.engine.testfixes; + +import java.util.Collection; +import java.util.List; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Tests for test-fix source text matching. + */ +public class TestFixesServiceTest { + /** Test suite/class name. */ + private static final String SUITE = "SqlPlanHistoryIntegrationTest"; + + /** Test method name. */ + private static final String TEST = "cacheQuery"; + + /** Short test name. */ + private static final String SHORT_TEST = SUITE + "." + TEST; + + /** Full TeamCity test name. */ + private static final String FULL_TEST = + "org.apache.ignite.internal.processors.query.calcite.integration." + SHORT_TEST; + + /** Cache run configuration. */ + private static final TestFixesService.TestFixCandidate CACHE8 = candidate("IgniteTests24Java8_Cache8"); + + /** Queries run configuration. */ + private static final TestFixesService.TestFixCandidate QUERIES1 = candidate("IgniteTests24Java8_Queries1"); + + /** */ + @Test + public void jiraTextMatchesSuiteDotTestInAllRunConfigurations() { + Collection matches = jiraMatches("Fix " + SHORT_TEST + " flakiness"); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void jiraTextMatchesSuiteHashTestInAllRunConfigurations() { + Collection matches = jiraMatches("Fix " + SUITE + "#" + TEST); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void jiraTextMatchesQualifiedRunConfigurationOnly() { + Collection matches = jiraMatches( + "Fix IgniteTests24Java8_Cache8::" + SUITE + "#" + TEST); + + assertEquals(1, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertFalse(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesSuiteDotTestInAllRunConfigurations() { + Collection matches = prMatches( + "IGNITE-1 Fix test", + "This PR fixes " + SHORT_TEST + " in master."); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesSuiteHashTestInAllRunConfigurations() { + Collection matches = prMatches( + "IGNITE-1 Fix " + SUITE + "#" + TEST, + "No comments from code review should be needed."); + + assertEquals(2, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertTrue(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void prTextMatchesQualifiedRunConfigurationOnly() { + Collection matches = prMatches( + "IGNITE-1 Fix IgniteTests24Java8_Cache8::" + SUITE + "#" + TEST, + "The same test exists in another suite, but this PR names the run configuration."); + + assertEquals(1, matches.size()); + assertTrue(matches.contains(CACHE8)); + assertFalse(matches.contains(QUERIES1)); + } + + /** */ + @Test + public void partialWordsDoNotMatch() { + Collection matches = jiraMatches( + "Fix Not" + SHORT_TEST + " and " + SHORT_TEST + "Extra"); + + assertTrue(matches.isEmpty()); + } + + /** */ + @Test + public void suiteNameAloneDoesNotMatch() { + Collection matches = jiraMatches( + "Fix " + SUITE + " instability in master"); + + assertTrue(matches.isEmpty()); + } + + /** */ + @Test + public void garbageTeamcityNamesAreNotPlausibleTests() { + assertFalse(TestFixesService.isPlausibleTestName("0.0\\\\, Culture=neutral\\\\, PublicKeyToken=123\\\\]\\\\]\")")); + assertFalse(TestFixesService.isPlausibleTestName("Configuration.ConfigurationException: x)")); + assertFalse(TestFixesService.isPlausibleTestName("Data.PropertyCollection)")); + assertFalse(TestFixesService.isPlausibleTestName("Log.CustomLoggerTest+CustomEnum)")); + + assertTrue(TestFixesService.isPlausibleTestName("OpenCensusSqlNativeTracingTest.testNextPageRequestFailure")); + assertTrue(TestFixesService.isPlausibleTestName("SqlPlanHistoryIntegrationTest.cacheQuery")); + } + + /** + * @param text JIRA summary plus description. + */ + private static Collection jiraMatches(String text) { + return TestFixesService.matchingCandidates(text, candidates()); + } + + /** + * @param title PR title. + * @param body PR body. + */ + private static Collection prMatches(String title, String body) { + return TestFixesService.matchingCandidates(title + "\n" + body, candidates()); + } + + /** */ + private static List candidates() { + return List.of(CACHE8, QUERIES1); + } + + /** + * @param suiteId Run configuration id. + */ + private static TestFixesService.TestFixCandidate candidate(String suiteId) { + return new TestFixesService.TestFixCandidate("master", suiteId, SUITE, 1, FULL_TEST, SHORT_TEST, + "https://ci.example/buildConfiguration/" + suiteId + "?branch=%3Cdefault%3E"); + } +} diff --git a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java index 117dd11f8..e315f6de4 100644 --- a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/GitHubConnIgnitedImpl.java @@ -18,6 +18,8 @@ import java.io.FileNotFoundException; import java.io.UncheckedIOException; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -154,6 +156,12 @@ public void init(IGitHubConnection conn) { return runActualizePrs(srvCode, false); } + /** {@inheritDoc} */ + @AutoProfiling + @Override public List getRecentPullRequests(int lookbackDays) { + return loadRecentPullRequests(Math.max(1, lookbackDays)); + } + /** {@inheritDoc} */ @AutoProfiling @Override public List getBranches() { @@ -198,6 +206,55 @@ private void actualizePrs() { 5, TimeUnit.MINUTES); } + /** + * @param lookbackDays Lookback window. + */ + private List loadRecentPullRequests(int lookbackDays) { + long cutoffMs = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(lookbackDays); + AtomicReference outLinkNext = new AtomicReference<>(); + List res = new java.util.ArrayList<>(); + String nextPageUrl = null; + + do { + List page = conn.getRecentPullRequestsPage(nextPageUrl, outLinkNext); + + enrichPullRequestAuthors(page); + savePrsChunk(page); + + for (PullRequest pr : page) { + Long updatedTs = parseGithubTime(pr.getTimeUpdate()); + + if (updatedTs == null || updatedTs >= cutoffMs) + res.add(pr); + } + + boolean reachedOldPage = page.stream() + .map(PullRequest::getTimeUpdate) + .map(GitHubConnIgnitedImpl::parseGithubTime) + .anyMatch(ts -> ts != null && ts < cutoffMs); + + nextPageUrl = reachedOldPage ? null : outLinkNext.get(); + } + while (nextPageUrl != null); + + return res; + } + + /** + * @param time GitHub ISO time. + */ + @Nullable private static Long parseGithubTime(@Nullable String time) { + if (time == null || time.isEmpty()) + return null; + + try { + return Instant.parse(time).toEpochMilli(); + } + catch (DateTimeParseException e) { + return null; + } + } + /** * */ diff --git a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java index 6327b8311..f3205e6ad 100644 --- a/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java +++ b/tcbot-github-ignited/src/main/java/org/apache/ignite/githubignited/IGitHubConnIgnited.java @@ -53,6 +53,14 @@ public interface IGitHubConnIgnited { */ public String refreshPullRequests(); + /** + * Loads recently updated pull requests, including closed/merged PRs. + * + * @param lookbackDays Number of recent days. + * @return Recent pull requests. + */ + public List getRecentPullRequests(int lookbackDays); + /** * Reloads branches from GitHub immediately. * diff --git a/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java index a04928165..8757ada19 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java +++ b/tcbot-github/src/main/java/org/apache/ignite/ci/github/PullRequest.java @@ -39,7 +39,7 @@ public class PullRequest implements IVersionedEntity { public static final int INCLUDE_SHORT_VER = 7; /** Entitiy current (latest) version. */ - private static final int LATEST_VERSION = 7; + private static final int LATEST_VERSION = 10; /** Entity version. */ @SuppressWarnings("FieldCanBeLocal") private Integer _ver = LATEST_VERSION; @@ -53,10 +53,15 @@ public class PullRequest implements IVersionedEntity { /** Pull Request title. */ private String title; + /** Pull Request body. */ + private String body; + @SerializedName("html_url") private String htmlUrl; @SerializedName("updated_at") private String updatedAt; + @SerializedName("merged_at") private String mergedAt; + /** Pull Request statuses URL. */ @SerializedName("statuses_url") private String statusesUrl; @@ -66,6 +71,8 @@ public class PullRequest implements IVersionedEntity { @SerializedName("base") private GitHubBranch base; + @SerializedName("merge_commit_sha") private String mergeCommitSha; + /** * @return Pull Request time update. */ @@ -73,6 +80,13 @@ public String getTimeUpdate() { return updatedAt; } + /** + * @return Pull Request merge time. + */ + @Nullable public String mergedAt() { + return mergedAt; + } + /** * @return Pull Request number. */ @@ -94,6 +108,13 @@ public String getTitle() { return title; } + /** + * @return Pull Request body. + */ + public String getBody() { + return body; + } + /** * @return Url to get PR statuses. */ @@ -129,6 +150,13 @@ public GitHubBranch head() { return head; } + /** + * @return Merge commit SHA. + */ + @Nullable public String mergeCommitSha() { + return mergeCommitSha; + } + /** {@inheritDoc} */ @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -152,17 +180,21 @@ public GitHubBranch head() { Objects.equals(_ver, req._ver) && Objects.equals(state, req.state) && Objects.equals(title, req.title) && + Objects.equals(body, req.body) && Objects.equals(htmlUrl, req.htmlUrl) && Objects.equals(updatedAt, req.updatedAt) && + Objects.equals(mergedAt, req.mergedAt) && Objects.equals(statusesUrl, req.statusesUrl) && Objects.equals(gitHubUser, req.gitHubUser) && Objects.equals(head, req.head) && - Objects.equals(base, req.base); + Objects.equals(base, req.base) && + Objects.equals(mergeCommitSha, req.mergeCommitSha); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(_ver, num, state, title, htmlUrl, updatedAt, statusesUrl, gitHubUser, head, base); + return Objects.hash(_ver, num, state, title, body, htmlUrl, updatedAt, mergedAt, statusesUrl, gitHubUser, head, base, + mergeCommitSha); } /** {@inheritDoc} */ diff --git a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java index ab0496d8c..a8055a4ab 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/GitHubConnectionImpl.java @@ -273,6 +273,26 @@ class GitHubConnectionImpl implements IGitHubConnection { return readOnePage(outLinkNext, url, rspHeaders, tok); } + /** {@inheritDoc} */ + @AutoProfiling + @Override public List getRecentPullRequestsPage(@Nullable String fullUrl, + @Nullable AtomicReference outLinkNext) { + String gitApiUrl = getApiUrlMandatory(); + + String url = fullUrl != null ? fullUrl : gitApiUrl + "pulls?state=all&sort=updated&direction=desc&per_page=100"; + + HashMap rspHeaders = new HashMap<>(); + if (outLinkNext != null) { + outLinkNext.set(null); + rspHeaders.put("Link", null); + } + + TypeToken> tok = new TypeToken>() { + }; + + return readOnePage(outLinkNext, url, rspHeaders, tok); + } + @Nonnull public String getApiUrlMandatory() { String gitApiUrl = config().gitApiUrl(); diff --git a/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java index cbd81dc08..34c16fd29 100644 --- a/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java +++ b/tcbot-github/src/main/java/org/apache/ignite/githubservice/IGitHubConnection.java @@ -50,6 +50,13 @@ public interface IGitHubConnection { */ public List getPullRequestsPage(@Nullable String fullUrl, @Nullable AtomicReference outLinkNext); + /** + * @param fullUrl Full url - null for first page, not null for next page. + * @param outLinkNext Out link for return next page full url. + */ + public List getRecentPullRequestsPage(@Nullable String fullUrl, + @Nullable AtomicReference outLinkNext); + /** * @param prNum Pull request number. * @return Pull request issue comments. diff --git a/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java new file mode 100644 index 000000000..a062d784e --- /dev/null +++ b/tcbot-integration-tests/src/integrationTest/java/org/apache/ignite/tcbot/integration/TestFixesFlowTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tcbot.integration; + +import java.time.Duration; +import java.util.regex.Pattern; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.ignite.tcbot.integration.IntegrationTestEnvironment.request; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Black-box coverage for test-fix matching. + */ +public class TestFixesFlowTest { + /** */ + private static final Pattern FINISHED_PROCESS = Pattern.compile("\"finished\"\\s*:\\s*[1-9][0-9]*"); + + /** */ + private static IntegrationTestEnvironment env; + + /** */ + @BeforeClass + public static void startSuite() throws Exception { + env = IntegrationTestEnvironment.get(); + } + + /** */ + @Before + public void resetEmulators() throws Exception { + env.resetEmulators(); + } + + /** */ + @Test + public void testFixRefreshReturnsOnlyOpenAndResolvedJiraTasksForMatchedTests() throws Exception { + String token = env.login(); + + createIssue("IGNITE-20101", + "Fix SqlRetryTest.testRetryOnTopologyChange flakiness", + "Open", + "The fix is being prepared for SqlRetryTest.testRetryOnTopologyChange.", + null); + createIssue("IGNITE-20102", + "Fix CacheRebalanceTest.testHistoricalRebalance instability", + "Resolved", + "Resolved test CacheRebalanceTest.testHistoricalRebalance after master history check.", + "2026-05-09T12:00:00.000+0000"); + + runTestOnlyAction(token, "refresh-jira", 710000001L); + runTestOnlyAction(token, "refresh-teamcity-builds", 710000002L); + + IntegrationTestEnvironment.HttpResponse master = request("GET", + env.botUrl + "/rest/tracked/results?branch=master", "Token " + token, null, null); + + assertEquals(master.body, 200, master.status); + assertTrue(master.body, master.body.contains("SqlRetryTest.testRetryOnTopologyChange")); + assertTrue(master.body, master.body.contains("CacheRebalanceTest.testHistoricalRebalance")); + + startTestFixRefresh(token, 710000003L); + waitForProcess(710000003L, token); + + String rows = waitForTestFixRows(token, "IGNITE-20101", "IGNITE-20102"); + + assertEquals(rows, 1, count(rows, "\"text\":\"IGNITE-20101\"")); + assertEquals(rows, 1, count(rows, "\"text\":\"IGNITE-20102\"")); + assertEquals(rows, 2, count(rows, "\"sourceType\":\"jira\"")); + assertEquals(rows, 0, count(rows, "\"sourceType\":\"github\"")); + assertTrue(rows, rows.contains("\"status\":\"Open\"")); + assertTrue(rows, rows.contains("\"status\":\"Resolved\"")); + assertTrue(rows, rows.contains("\"closedDate\":\"2026-05-09\"")); + } + + /** */ + private static void createIssue(String key, String summary, String status, String description, + String resolutionDate) throws Exception { + String body = "{\"key\":\"" + key + "\"," + + "\"summary\":\"" + summary + "\"," + + "\"status\":\"" + status + "\"," + + "\"description\":\"" + description + "\"," + + "\"updated\":\"2026-05-10T10:00:00.000+0000\"" + + (resolutionDate == null ? "" : ",\"resolutiondate\":\"" + resolutionDate + "\"") + + "}"; + + IntegrationTestEnvironment.HttpResponse issue = request("POST", env.jiraUrl + "/__test__/jira/create-issue", + null, "application/json", body); + + assertEquals(issue.body, 201, issue.status); + } + + /** */ + private static void startTestFixRefresh(String token, long processId) throws Exception { + IntegrationTestEnvironment.HttpResponse refresh = request("GET", env.botUrl + + "/rest/testFixes/refresh?limit=0&processId=" + processId, + "Token " + token, null, null); + + assertEquals(refresh.body, 200, refresh.status); + } + + /** */ + private static String waitForTestFixRows(String token, String... expected) throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + String lastBody = ""; + + while (System.nanoTime() < deadline) { + IntegrationTestEnvironment.HttpResponse response = request("GET", env.botUrl + + "/rest/testFixes/recent?limit=0", "Token " + token, null, null); + lastBody = response.status + " " + response.body; + + boolean found = response.status == 200; + + for (String item : expected) + found &= response.body.contains(item); + + if (found) + return response.body; + + Thread.sleep(500); + } + + throw new IllegalStateException("Test fix rows were not found, last response: " + lastBody); + } + + /** */ + private static String runTestOnlyAction(String token, String action, long processId) throws Exception { + IntegrationTestEnvironment.HttpResponse start = request("POST", env.botUrl + + "/rest/__test__/bot/" + action + "?serverId=apache&processId=" + processId, + "Token " + token, null, null); + + assertEquals(start.body, 200, start.status); + assertTrue(start.body, start.body.contains("\"queued\":true")); + + return waitForProcess(processId, token); + } + + /** */ + private static String waitForProcess(long processId, String token) throws Exception { + long deadline = System.nanoTime() + Duration.ofSeconds(90).toNanos(); + String url = env.botUrl + "/rest/process/status?id=" + processId; + String lastBody = ""; + + while (System.nanoTime() < deadline) { + IntegrationTestEnvironment.HttpResponse response = request("GET", url, "Token " + token, null, null); + lastBody = response.status + " " + response.body; + + if (response.status == 200 && FINISHED_PROCESS.matcher(response.body).find()) + return response.body; + + Thread.sleep(250); + } + + throw new IllegalStateException("Process did not finish: " + processId + ", last status: " + lastBody); + } + + /** */ + private static int count(String text, String needle) { + int res = 0; + int start = 0; + + while (start < text.length()) { + int idx = text.indexOf(needle, start); + + if (idx < 0) + return res; + + res++; + start = idx + needle.length(); + } + + return res; + } + +} diff --git a/tcbot-integration-tests/src/integrationTest/resources/branches.json b/tcbot-integration-tests/src/integrationTest/resources/branches.json index 93ea9aaf0..00ef3794f 100644 --- a/tcbot-integration-tests/src/integrationTest/resources/branches.json +++ b/tcbot-integration-tests/src/integrationTest/resources/branches.json @@ -1,7 +1,14 @@ { "primaryServerCode": "apache", "botAdminGroups": ["IGNITE_COMMITTER"], - "resettableCaches": ["testFixMatches", "testFixSourceUpdates"], + "resettableCaches": [ + "testFixRefsByTest", + "testFixSourcesById", + "testFixMatches", + "testFixMatchesV2", + "testFixSourcesV2", + "testFixSourceUpdates" + ], "confidence": 0.995, "tcServers": [ { diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java index 797174019..dad0fa66e 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java @@ -18,6 +18,8 @@ package org.apache.ignite.ci.jira.ignited; import java.util.Objects; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nullable; import org.apache.ignite.ci.tcbot.common.StringFieldCompacted; import org.apache.ignite.jiraservice.Status; @@ -56,6 +58,15 @@ public class TicketCompacted { /** Internal JIRA id of ticket status, see {@link org.apache.ignite.jiraservice.JiraTicketStatusCode}. */ public int statusCodeId; + /** Labels joined by new lines, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted labels = new StringFieldCompacted(); + + /** Last updated date as returned by JIRA, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted updated = new StringFieldCompacted(); + + /** Resolution date as returned by JIRA, nullable because of older entry versions. */ + @Nullable private StringFieldCompacted resolutiondate = new StringFieldCompacted(); + /** * @param ticket Jira ticket. * @param comp Compactor. @@ -68,6 +79,9 @@ public TicketCompacted(Ticket ticket, IStringCompactor comp, String projectCode) summary.setValue(ticket.fields.summary()); customfield_11050.setValue(ticket.fields.igniteLink()/*customfield_11050*/); description.setValue(ticket.fields.description()); + labels.setValue(String.join("\n", ticket.fields.labels())); + updated.setValue(ticket.fields.updated()); + resolutiondate.setValue(ticket.fields.resolutionDate()); } /** @@ -85,12 +99,37 @@ public Ticket toTicket(IStringCompactor comp, String projectCode) { fields.summary = summary != null ? summary.getValue() : null; fields.customfield_11050 = customfield_11050 != null ? customfield_11050.getValue() : null; fields.description = description != null ? description.getValue() : null; + fields.labels = splitLines(labels); + fields.updated = updated != null ? updated.getValue() : null; + fields.resolutiondate = resolutiondate != null ? resolutiondate.getValue() : null; ticket.fields = fields; return ticket; } + /** + * @param val Compacted string field. + */ + private static List splitLines(@Nullable StringFieldCompacted val) { + List res = new ArrayList<>(); + + if (val == null) + return res; + + String text = val.getValue(); + + if (text == null || text.isEmpty()) + return res; + + for (String label : text.split("\\n")) { + if (!label.isEmpty()) + res.add(label); + } + + return res; + } + /** {@inheritDoc} */ @Override public boolean equals(Object o) { if (this == o) @@ -106,11 +145,15 @@ public Ticket toTicket(IStringCompactor comp, String projectCode) { statusCodeId == compacted.statusCodeId && Objects.equals(summary, compacted.summary) && Objects.equals(customfield_11050, compacted.customfield_11050) && - Objects.equals(description, compacted.description); + Objects.equals(description, compacted.description) && + Objects.equals(labels, compacted.labels) && + Objects.equals(updated, compacted.updated) && + Objects.equals(resolutiondate, compacted.resolutiondate); } /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(id, igniteId, statusCodeId, summary, customfield_11050, description); + return Objects.hash(id, igniteId, statusCodeId, summary, customfield_11050, description, labels, updated, + resolutiondate); } } diff --git a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java index b4edb515a..6db8bee57 100644 --- a/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java +++ b/tcbot-jira-ignited/src/main/java/org/apache/ignite/jiraignited/JiraTicketSync.java @@ -17,10 +17,16 @@ package org.apache.ignite.jiraignited; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.inject.Inject; +import javax.inject.Provider; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.IJiraIntegration; @@ -29,6 +35,7 @@ import org.apache.ignite.jiraservice.Ticket; import org.apache.ignite.tcbot.common.conf.IJiraServerConfig; import org.apache.ignite.tcbot.common.interceptor.MonitoredTask; +import org.apache.ignite.tcbot.persistence.CacheConfigs; import org.apache.ignite.tcbot.persistence.scheduler.IScheduler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +49,21 @@ public class JiraTicketSync { /** Logger. */ private static final Logger logger = LoggerFactory.getLogger(JiraTicketSync.class); + /** Test-fix JIRA sync state cache name. */ + private static final String TEST_FIX_SYNC_STATE_CACHE_NAME = "jiraTestFixSyncState"; + + /** Labeled test-fix catch-up period. */ + private static final long TEST_FIX_LABEL_SYNC_PERIOD_MS = TimeUnit.DAYS.toMillis(1); + + /** Max age for labeled test-fix catch-up. */ + private static final int TEST_FIX_LABEL_LOOKBACK_DAYS = 365; + /** Scheduler. */ @Inject private IScheduler scheduler; + /** Ignite provider. */ + @Inject private Provider igniteProvider; + /** Mute DAO. */ @Inject private JiraTicketDao jiraDao; @@ -98,11 +117,132 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { IJiraServerConfig cfg = jira.config(); String projectCode = cfg.projectCodeForVisa(); - String baseUrl = cfg.getApiVersion().searchUrl() + escape("project=" + projectCode + " order by updated DESC") - + "&" + - "fields=" + reqFields + - "&maxResults=100"; + String baseUrl = searchUrl(cfg, "project=" + projectCode + " order by updated DESC", reqFields); + Set processedKeys = new HashSet<>(); + + SyncStats stats = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, baseUrl, fullResync, processedKeys); + SyncStats recent = new SyncStats(); + SyncStats labeled = new SyncStats(); + int recentDays = 0; + boolean labelSync = false; + + if (fullResync) + markTestFixJiraSyncComplete(srvCode); + else { + recentDays = recentLookbackDays(srvCode, cfg); + recent = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND updated >= -" + recentDays + + "d order by updated DESC", reqFields), true, processedKeys); + markTestFixRecentJiraSyncComplete(srvCode); + + labelSync = shouldSyncTestFixLabels(srvCode); + + if (labelSync) { + labeled = loadQuery(jira, cfg, srvIdMaskHigh, projectCode, + searchUrl(cfg, "project=" + projectCode + " AND labels = " + cfg.testFixesLabel() + + " AND updated >= -" + TEST_FIX_LABEL_LOOKBACK_DAYS + "d order by updated DESC", reqFields), + true, processedKeys); + markTestFixLabelJiraSyncComplete(srvCode); + } + } + + int saved = stats.saved + recent.saved + labeled.saved; + + return "Jira tickets saved " + saved + " from " + + (stats.processed + recent.processed + labeled.processed) + " checked for service " + srvCode + + " (recentDays=" + recentDays + ", labelSync=" + labelSync + ", duplicatesSkipped=" + + (stats.duplicatesSkipped + recent.duplicatesSkipped + labeled.duplicatesSkipped) + ")"; + } + + /** + * @param srvCode Server code. + * @param cfg JIRA config. + */ + private int recentLookbackDays(String srvCode, IJiraServerConfig cfg) { + Long lastSyncTs = testFixSyncStateCache().get(testFixRecentSyncKey(srvCode)); + int maxDays = Math.max(1, cfg.testFixesLookbackDays()); + + if (lastSyncTs == null) + return maxDays; + + long elapsedMs = Math.max(0, System.currentTimeMillis() - lastSyncTs); + int elapsedDays = (int)(elapsedMs / TimeUnit.DAYS.toMillis(1)) + 1; + + return Math.min(maxDays, Math.max(1, elapsedDays)); + } + + /** + * @param srvCode Server code. + */ + private boolean shouldSyncTestFixLabels(String srvCode) { + Long lastSyncTs = testFixSyncStateCache().get(testFixLabelSyncKey(srvCode)); + + return lastSyncTs == null || System.currentTimeMillis() - lastSyncTs >= TEST_FIX_LABEL_SYNC_PERIOD_MS; + } + + /** + * @param srvCode Server code. + */ + private void markTestFixJiraSyncComplete(String srvCode) { + long now = System.currentTimeMillis(); + IgniteCache cache = testFixSyncStateCache(); + + cache.put(testFixRecentSyncKey(srvCode), now); + cache.put(testFixLabelSyncKey(srvCode), now); + } + + /** + * @param srvCode Server code. + */ + private void markTestFixRecentJiraSyncComplete(String srvCode) { + testFixSyncStateCache().put(testFixRecentSyncKey(srvCode), System.currentTimeMillis()); + } + /** + * @param srvCode Server code. + */ + private void markTestFixLabelJiraSyncComplete(String srvCode) { + testFixSyncStateCache().put(testFixLabelSyncKey(srvCode), System.currentTimeMillis()); + } + + /** */ + private IgniteCache testFixSyncStateCache() { + return igniteProvider.get().getOrCreateCache(CacheConfigs.getCache8PartsConfig(TEST_FIX_SYNC_STATE_CACHE_NAME)); + } + + /** + * @param srvCode Server code. + */ + private static String testFixRecentSyncKey(String srvCode) { + return "recent:" + srvCode; + } + + /** + * @param srvCode Server code. + */ + private static String testFixLabelSyncKey(String srvCode) { + return "label:" + srvCode; + } + + /** + * @param cfg JIRA config. + * @param jql JQL query. + * @param reqFields Requested fields. + */ + private String searchUrl(IJiraServerConfig cfg, String jql, String reqFields) { + return cfg.getApiVersion().searchUrl() + escape(jql) + "&fields=" + reqFields + "&maxResults=100"; + } + + /** + * @param jira JIRA facade. + * @param cfg JIRA config. + * @param srvIdMaskHigh Server id mask high. + * @param projectCode Project code. + * @param baseUrl Search URL. + * @param fullResync Full resync flag. + */ + private SyncStats loadQuery(IJiraIntegration jira, IJiraServerConfig cfg, int srvIdMaskHigh, String projectCode, + String baseUrl, boolean fullResync, Set processedKeys) { String url = baseUrl; logger.info("Requesting JIRA tickets using URL " + url + ("\n" + cfg.restApiUrl() + url)); @@ -110,12 +250,14 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { Collection page = tickets.issues(); if (F.isEmpty(page)) - return "Something went wrong - no tickets found. Check jira availability: " + - "[project=" + projectCode + ", url=" + url + "]"; + return new SyncStats(); - int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); + SyncStats res = new SyncStats(); + Collection uniquePage = filterProcessedTickets(srvIdMaskHigh, page, projectCode, processedKeys, res); + int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, uniquePage, projectCode); - int ticketsProcessed = page.size(); + res.saved += ticketsSaved; + res.processed += uniquePage.size(); if (ticketsSaved != 0 || fullResync) { while (tickets.hasNextPage()) { @@ -129,16 +271,52 @@ protected String actualizeJiraTickets(String srvCode, boolean fullResync) { if (F.isEmpty(page)) break; - int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, projectCode); + uniquePage = filterProcessedTickets(srvIdMaskHigh, page, projectCode, processedKeys, res); + int savedNow = jiraDao.saveChunk(srvIdMaskHigh, uniquePage, projectCode); - ticketsSaved += savedNow; - ticketsProcessed += page.size(); + res.saved += savedNow; + res.processed += uniquePage.size(); if (savedNow == 0 && !fullResync) break; // find not updated chunk and exit } } - return "Jira tickets saved " + ticketsSaved + " from " + ticketsProcessed + " checked for service " + srvCode; + return res; + } + + /** + * @param srvIdMaskHigh Server id mask high. + * @param page Tickets page. + * @param projectCode Project code. + * @param processedKeys Already processed cache keys. + * @param stats Sync stats. + */ + private static Collection filterProcessedTickets(int srvIdMaskHigh, Collection page, + String projectCode, Set processedKeys, SyncStats stats) { + Collection res = new ArrayList<>(); + + for (Ticket ticket : page) { + long key = JiraTicketDao.ticketToCacheKey(srvIdMaskHigh, ticket.keyWithoutProject(projectCode)); + + if (processedKeys.add(key)) + res.add(ticket); + else + stats.duplicatesSkipped++; + } + + return res; + } + + /** Sync stats. */ + private static class SyncStats { + /** Saved count. */ + private int saved; + + /** Processed count. */ + private int processed; + + /** Duplicate tickets skipped before save. */ + private int duplicatesSkipped; } } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java index 9e0666929..3916633d9 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/IFields.java @@ -24,7 +24,10 @@ public interface IFields { "status", "summary", "customfield_11050", - "description" + "description", + "labels", + "updated", + "resolutiondate" ); /** Ticket status. */ @@ -38,4 +41,13 @@ public interface IFields { /** Description. */ String description(); + + /** Labels. */ + List labels(); + + /** Last updated date as returned by JIRA. */ + String updated(); + + /** Resolution date as returned by JIRA. */ + String resolutionDate(); } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java index 616b9474f..a1ef8d4fa 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v2/Fields.java @@ -18,6 +18,8 @@ package org.apache.ignite.jiraservice.v2; import com.google.common.base.MoreObjects; +import java.util.Collections; +import java.util.List; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.Status; @@ -37,6 +39,15 @@ public class Fields implements IFields { /** Description. */ public String description; + /** Labels. */ + public List labels; + + /** Last updated date. */ + public String updated; + + /** Resolution date. */ + public String resolutiondate; + /** {@inheritDoc} */ @Override public Status status() { return status; @@ -57,12 +68,28 @@ public class Fields implements IFields { return description; } + /** {@inheritDoc} */ + @Override public List labels() { + return labels == null ? Collections.emptyList() : labels; + } + + /** {@inheritDoc} */ + @Override public String updated() { + return updated; + } + + /** {@inheritDoc} */ + @Override public String resolutionDate() { + return resolutiondate; + } + /** {@inheritDoc} */ @Override public String toString() { return MoreObjects.toStringHelper(this) .add("status", status) .add("summary", summary) .add("customfield_11050", customfield_11050) + .add("updated", updated) .toString(); } } diff --git a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java index ad221f93d..876907795 100644 --- a/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java +++ b/tcbot-jira/src/main/java/org/apache/ignite/jiraservice/v3/FieldsV3.java @@ -18,6 +18,7 @@ package org.apache.ignite.jiraservice.v3; import com.google.common.base.MoreObjects; +import java.util.Collections; import java.util.List; import org.apache.ignite.jiraservice.IFields; import org.apache.ignite.jiraservice.Status; @@ -39,6 +40,15 @@ public class FieldsV3 implements IFields { /** Description. */ private Description description; + /** Labels. */ + private List labels; + + /** Last updated date. */ + private String updated; + + /** Resolution date. */ + private String resolutiondate; + /** {@inheritDoc} */ @Override public Status status() { return status; @@ -65,11 +75,27 @@ public class FieldsV3 implements IFields { return sb.toString(); } + /** {@inheritDoc} */ + @Override public List labels() { + return labels == null ? Collections.emptyList() : labels; + } + + /** {@inheritDoc} */ + @Override public String updated() { + return updated; + } + + /** {@inheritDoc} */ + @Override public String resolutionDate() { + return resolutiondate; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("status", status) .add("summary", summary) .add("customfield_11050", customfield_11050) + .add("updated", updated) .toString(); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java index 1fae99acd..120325991 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/ISuiteRunHistory.java @@ -16,6 +16,7 @@ */ package org.apache.ignite.tcignited.history; +import java.util.Set; import java.util.Map; import javax.annotation.Nullable; @@ -23,5 +24,10 @@ public interface ISuiteRunHistory { IRunHistory self(); @Nullable IRunHistory getTestRunHist(int testName); + /** + * @return Test name ids present in this suite history. + */ + Set testNames(); + ISuiteRunHistory filter(Map requireParameters); } diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java index 51b40a6e0..d59c40455 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/SuiteHistory.java @@ -101,6 +101,11 @@ private void addSuiteInvocation(SuiteInvocation suiteInv, Map testNames() { + return testsInvStatues.keySet(); + } + /** {@inheritDoc} */ @Override public ISuiteRunHistory filter(Map requireParameters) { RunHistCompacted suitesFiltered = suiteHist.filterSuiteInvByParms(requireParameters);