Ignite-28655: test failure with JIRA investigation/fix match, try 2

conf/branches.json

@@ -5,7 +5,14 @@
   // Apache Ignite test admins are exposed by TeamCity REST as key="IGNITE_COMMITER".
   "botAdminGroups": ["IGNITE_COMMITER"],
   // Ignite cache names admins may reset from monitoring UI.
-  "resettableCaches": ["testFixMatches", "testFixSourceUpdates"],
+  "resettableCaches": [
+    "testFixRefsByTest",
+    "testFixSourcesById",
+    "testFixMatches",
+    "testFixMatchesV2",
+    "testFixSourcesV2",
+    "testFixSourceUpdates"
+  ],
   "confidence": 0.995,
   "cleanerConfig": {
     "numOfItemsToDel": 100000,

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java

@@ -632,6 +632,7 @@ protected String checkFailuresEx(String brachName) {
             false,
             null,
             null,
+            null,
             DisplayMode.None,
             null,
             -1, false, false);
@@ -645,6 +646,7 @@ protected String checkFailuresEx(String brachName) {
                 false,
                 null,
                 null,
+                null,
                 DisplayMode.OnlyFailures,
                 null,
                 -1, false, false);

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/monitoring/MonitoringService.java

@@ -16,6 +16,10 @@
  */
 package org.apache.ignite.ci.web.rest.monitoring;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Strings;
 import java.io.BufferedReader;
 import java.io.File;
@@ -31,7 +35,9 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -40,9 +46,11 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.security.RolesAllowed;
+import javax.cache.Cache;
 import javax.servlet.ServletContext;
 import javax.ws.rs.BadRequestException;
 import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.ForbiddenException;
 import javax.ws.rs.FormParam;
 import javax.ws.rs.GET;
 import javax.ws.rs.NotFoundException;
@@ -110,6 +118,44 @@ public class MonitoringService {
     /** Max summary length. */
     private static final int SUMMARY_LIMIT = 240;
 
+    /** Default number of cache entries to preview. */
+    private static final int DFLT_CACHE_PEEK_LIMIT = 5;
+
+    /** Hard cache preview cap. */
+    private static final int MAX_CACHE_PEEK_LIMIT = 20;
+
+    /** System property with comma-separated exact cache names allowed for raw preview. */
+    private static final String CACHE_PEEK_ALLOWED_CACHES = "tcbot.monitoring.cachePeek.allowedCaches";
+
+    /** Built-in exact cache names allowed for raw preview. */
+    private static final Set<String> DFLT_CACHE_PEEK_ALLOWED_CACHES = Collections.unmodifiableSet(new HashSet<>(
+        Arrays.asList(
+            "botDetectedDefects",
+            "botDetectedIssues",
+            "buildLogCheckResult",
+            "buildsConditions",
+            "compactVisasHistoryCacheV2",
+            "gitHubBranch",
+            "gitHubPr",
+            "jiraTestFixSyncState",
+            "mutedIssues",
+            "newTestsCache",
+            "teamcityBuildRef",
+            "teamcityBuildStartTime",
+            "teamcityBuildTypeRef",
+            "teamcityChange",
+            "teamcityFatBuild",
+            "teamcityFatBuildType",
+            "teamcityMute",
+            "teamcitySuiteHistory",
+            "testFixRefsByTest",
+            "testFixSourcesById"
+        )));
+
+    /** JSON mapper for raw cache entry values. */
+    private static final ObjectMapper CACHE_PEEK_MAPPER = new ObjectMapper()
+        .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
     /** Log timestamp format. */
     private static final DateTimeFormatter LOG_TS_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
 
@@ -602,6 +648,110 @@ public List<CacheMetricsUi> getCacheStat() {
         return res;
     }
 
+    @GET
+    @RolesAllowed(AuthenticationFilter.ADMIN_ROLE)
+    @Produces(MediaType.TEXT_PLAIN)
+    @Path("cachePeek")
+    public String cachePeek(@QueryParam("name") String name, @QueryParam("limit") Integer limit) {
+        if (Strings.isNullOrEmpty(name))
+            throw new BadRequestException("Cache name is required");
+
+        ensureCanPeekCache(name);
+
+        Ignite ignite = instance(Ignite.class);
+        IgniteCache<?, ?> cache = ignite.cache(name);
+
+        if (cache == null)
+            throw new NotFoundException("Cache not found: " + name);
+
+        int actualLimit = normalizeCachePeekLimit(limit);
+        StringBuilder res = new StringBuilder();
+        int shown = 0;
+        boolean truncated = false;
+
+        res.append("Cache: ").append(name).append('\n');
+        res.append("Size: not calculated by cachePeek").append('\n');
+        res.append("Limit: ").append(actualLimit).append("\n\n");
+
+        for (Cache.Entry<?, ?> entry : cache) {
+            if (shown >= actualLimit) {
+                truncated = true;
+
+                break;
+            }
+
+            shown++;
+
+            res.append("Entry #").append(shown).append('\n');
+            res.append("Key class: ").append(className(entry.getKey())).append('\n');
+            res.append(toJsonNode(entry.getKey()).toPrettyString()).append('\n');
+            res.append("Value class: ").append(className(entry.getValue())).append('\n');
+            res.append(toJsonNode(entry.getValue()).toPrettyString()).append("\n\n");
+        }
+
+        res.append("Entries shown: ").append(shown).append('\n');
+        res.append("Truncated: ").append(truncated).append('\n');
+
+        return res.toString();
+    }
+
+    /**
+     * @param limit Requested limit.
+     */
+    private static int normalizeCachePeekLimit(Integer limit) {
+        if (limit == null || limit <= 0)
+            return DFLT_CACHE_PEEK_LIMIT;
+
+        return Math.min(limit, MAX_CACHE_PEEK_LIMIT);
+    }
+
+    /**
+     * @param obj Object.
+     */
+    private static String className(Object obj) {
+        return obj == null ? "null" : obj.getClass().getName();
+    }
+
+    /**
+     * @param name Cache name.
+     */
+    private void ensureCanPeekCache(String name) {
+        if (cachePeekAllowedCaches().contains(name))
+            return;
+
+        throw new ForbiddenException("Cache peek is not allowed for cache: " + name +
+            ". Add the exact cache name to " + CACHE_PEEK_ALLOWED_CACHES + " to enable it.");
+    }
+
+    /**
+     * @return Exact cache names allowed for raw preview.
+     */
+    private static Set<String> cachePeekAllowedCaches() {
+        Set<String> res = new HashSet<>(DFLT_CACHE_PEEK_ALLOWED_CACHES);
+
+        Arrays.stream(Strings.nullToEmpty(System.getProperty(CACHE_PEEK_ALLOWED_CACHES)).split(","))
+            .map(String::trim)
+            .filter(s -> !s.isEmpty())
+            .forEach(res::add);
+
+        return res;
+    }
+
+    /**
+     * @param obj Object.
+     */
+    private static JsonNode toJsonNode(Object obj) {
+        try {
+            return CACHE_PEEK_MAPPER.valueToTree(obj);
+        }
+        catch (RuntimeException e) {
+            return CACHE_PEEK_MAPPER.createObjectNode()
+                .put("serializationError", e.getClass().getSimpleName() + ": " + e.getMessage())
+                .put("class", className(obj))
+                .put("toString", String.valueOf(obj));
+        }
+    }
+
     @POST
     @RolesAllowed(AuthenticationFilter.ADMIN_ROLE)
     @Path("resetCache")

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/testfixes/TestFixesRestService.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ci.web.rest.testfixes;
+
+import java.util.List;
+import javax.annotation.security.RolesAllowed;
+import javax.servlet.ServletContext;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import org.apache.ignite.ci.web.CtxListener;
+import org.apache.ignite.ci.web.auth.AuthenticationFilter;
+import org.apache.ignite.tcbot.engine.testfixes.TestFixRefUi;
+import org.apache.ignite.tcbot.engine.testfixes.TestFixesService;
+
+/**
+ * Test fix history REST service.
+ */
+@Path("testFixes")
+@Produces(MediaType.APPLICATION_JSON)
+public class TestFixesRestService {
+    /** Servlet Context. */
+    @Context private ServletContext ctx;
+
+    /**
+     * @param limit Max rows.
+     */
+    @GET
+    @Path("recent")
+    public List<TestFixRefUi> recent(@QueryParam("limit") Integer limit) {
+        int actualLimit = limit == null ? 0 : limit;
+
+        return CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class).recent(actualLimit);
+    }
+
+    /**
+     * Schedules an immediate refresh and returns currently cached rows.
+     *
+     * @param limit Max rows.
+     * @param processId Optional user-visible process id.
+     */
+    @GET
+    @RolesAllowed(AuthenticationFilter.ADMIN_ROLE)
+    @Path("refresh")
+    public List<TestFixRefUi> refresh(@QueryParam("limit") Integer limit, @QueryParam("processId") Long processId) {
+        TestFixesService svc = CtxListener.getApplicationContext(ctx).getInstance(TestFixesService.class);
+
+        svc.requestMatchNow(processId);
+
+        return recent(limit);
+    }
+}

ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/web/rest/tracked/GetTrackedBranchTestResults.java

@@ -124,14 +124,15 @@ public DsSummaryUi getTestFailsResultsNoSync(
         @Nullable @QueryParam("trustedTests") Boolean trustedTests,
         @Nullable @QueryParam("tagSelected") String tagSelected,
         @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected,
+        @Nullable @QueryParam("suiteId") String suiteId,
         @Nullable @QueryParam("displayMode") String displayMode,
         @Nullable @QueryParam("sortOption") String sortOption,
         @Nullable @QueryParam("count") Integer mergeCnt,
         @Nullable @QueryParam("showTestLongerThan") Integer showTestLongerThan,
         @Nullable @QueryParam("muted") Boolean showMuted,
         @Nullable @QueryParam("ignored") Boolean showIgnored) {
         return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected,
-            SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored);
+            suiteId, SyncMode.NONE, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored);
     }
 
     @GET
@@ -143,14 +144,16 @@ public DsSummaryUi getTestFailsNoCache(
         @Nullable @QueryParam("trustedTests") Boolean trustedTests,
         @Nullable @QueryParam("tagSelected") String tagSelected,
         @Nullable @QueryParam("tagForHistSelected") String tagForHistSelected,
+        @Nullable @QueryParam("suiteId") String suiteId,
         @Nullable @QueryParam("displayMode") String displayMode,
         @Nullable @QueryParam("sortOption") String sortOption,
         @Nullable @QueryParam("count") Integer mergeCnt,
         @Nullable @QueryParam("showTestLongerThan") Integer showTestLongerThan,
         @Nullable @QueryParam("muted") Boolean showMuted,
         @Nullable @QueryParam("ignored") Boolean showIgnored) {
         return latestBuildResults(branch, checkAllLogs, trustedTests, tagSelected, tagForHistSelected,
-            SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted, showIgnored);
+            suiteId, SyncMode.RELOAD_QUEUED, displayMode, sortOption, mergeCnt, showTestLongerThan, showMuted,
+            showIgnored);
     }
 
     @NotNull private DsSummaryUi latestBuildResults(
@@ -159,6 +162,7 @@ public DsSummaryUi getTestFailsNoCache(
         @Nullable Boolean trustedTests,
         @Nullable String tagSelected,
         @Nullable String tagForHistSelected,
+        @Nullable String suiteId,
         @Nonnull SyncMode mode,
         @Nullable String displayMode,
         @Nullable String sortOption,
@@ -185,6 +189,7 @@ public DsSummaryUi getTestFailsNoCache(
                 Boolean.TRUE.equals(trustedTests),
                 tagSelected,
                 tagForHistSelected,
+                suiteId,
                 DisplayMode.parseStringValue(displayMode),
                 SortOption.parseStringValue(sortOption),
                 maxDurationSec,

ignite-tc-helper-web/src/main/webapp/current.html

@@ -204,7 +204,11 @@
 
     let tagForHistSelected = gVue.$data.tagForHistSelected;
     if (tagForHistSelected != null && tagForHistSelected !== "")
-        curReqParms += "&tagForHistSelected=" + tagForHistSelected;
+        curReqParms += "&tagForHistSelected=" + encodeURIComponent(tagForHistSelected);
+
+    let suiteId = findGetParameter("suiteId");
+    if (suiteId != null && suiteId !== "")
+        curReqParms += "&suiteId=" + encodeURIComponent(suiteId);
 
     let runTime = gVue.$data.showTestLongerThan;
     if (runTime != null && runTime > 0)

ignite-tc-helper-web/src/main/webapp/guard.html

@@ -4,7 +4,6 @@
     <link rel="icon" href="img/leaf.svg">
 
     <link rel="stylesheet" href="https://code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css">
-
     <link rel="stylesheet" href="css/style-1.5.css">
 
     <script src="https://code.jquery.com/jquery-1.12.4.js"></script>
@@ -19,7 +18,6 @@
 
         .guard-page {
             margin: 22px 18px 34px 18px;
-            max-width: 1320px;
             padding: 0;
         }
 

ignite-tc-helper-web/src/main/webapp/js/common-1.7.js

@@ -732,6 +732,7 @@ function showMenu(menuData) {
         res += "<a href=\"/compare.html\" title='Compare builds tests test'>Compare builds</a>";
         res += "<a href=\"/issues.html\" title='Detected issues list'>Issues history</a>";
         res += "<a href=\"/visas.html\" title='Issued TC Bot Visa history'>Visas history</a>";
+        res += "<a href=\"/testfixes.html\" title='Matched test fixes history'>Test fixes history</a>";
         res += "<a href=\"/mutes.html\" title='Muted tests list'>Muted tests</a>";
         res += "<a href=\"/mutedissues/index.html\" title='Muted issues list'>Muted issues</a>";
         res += "<a href=\"/board/index.html\" title='Board'>Board</a>";