Merge branch '7.11' of github.com:gchq/stroom

Author: stroomdev66

CHANGELOG.md

@@ -13,6 +13,25 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 ~~~
 
 
+## [v7.11.0] - 2026-03-04
+
+* Bug : Change proxy event store to close its appenders on app shutdown.
+
+* Bug : Fix missing docker managed volues for proxy `zip_file_ingest` and `zip_file_ingest_failed` directories.
+
+* Bug : Fix NPE in downstreamHostConfig ctor if there is no hostname, which is a possibility.
+
+* Bug : Fix incorrect path for /datafeed in the ProxyWelcomeServlet response.
+
+* Bug : Move the `/queues` proxy endpoint from the app port to the admin port.
+
+* Bug **#5424** : Change the `receiptCheckMode` `RECEIVE_ALL` to no longer check for the existence of the feed. Add the `receiptCheckMode` `FEED_EXISTENCE` to perform a feed existence check.
+
+* Bug : Fix `subPathTemplate.enabled` defaulting to `false` if subPathTemplate has been set in the yaml, but enabled has not.
+
+* Bug : Fix bug in proxy which will throw an error if subPathTemplate contains repeated vars, e.g. `${year}/${year}-${month}`.
+
+
 ## [v7.11-beta.25] - 2026-02-13
 
 * Bug **#5392** : Fix PlanB segfault.
@@ -2014,7 +2033,8 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 * Issue **#3830** : Add S3 data storage option.
 
 
-[Unreleased]: https://github.com/gchq/stroom/compare/v7.11-beta.25...HEAD
+[Unreleased]: https://github.com/gchq/stroom/compare/v7.11.0...HEAD
+[v7.11.0]: https://github.com/gchq/stroom/compare/v7.11-beta.25...v7.11.0
 [v7.11-beta.25]: https://github.com/gchq/stroom/compare/v7.11-beta.24...v7.11-beta.25
 [v7.11-beta.24]: https://github.com/gchq/stroom/compare/v7.11-beta.23...v7.11-beta.24
 [v7.11-beta.23]: https://github.com/gchq/stroom/compare/v7.11-beta.22...v7.11-beta.23

stroom-core-shared/src/main/java/stroom/receive/rules/shared/ReceiptCheckMode.java

@@ -27,17 +27,34 @@ public enum ReceiptCheckMode {
     /**
      * The feed status (RECEIVE|DROP|REJECT) for the feed of the received data will be checked by calling
      * the downstream stroom/proxy.
+     * If auto content creation is enabled on stroom, then when the check is performed,
+     * the feed will be auto-created (if various conditions for that are met) and the data will then
+     * be accpeted.
      */
     FEED_STATUS(true),
     /**
      * The meta attributes from the headers will be checked against the receipt policy rules to determine
      * whether the data should be accepted for receipt, rejected or silently dropped. ALL downstream
      * stroom-proxy instances in the chain must also use this mode if this mode is set.
+     * <p>
+     * This mode will also check if the feed exists. If the feed does not exist the data will be rejected.
+     * If auto content creation is enabled on stroom, then when the check is performed,
+     * the feed will be auto-created (if various conditions for that are met) and this
+     * filter will then return true.
      */
     RECEIPT_POLICY(true),
     /**
-     * No check is performed. All data is accepted for receipt (subject to other checks
-     * like presence of stream type).
+     * This mode will check if the feed exists. If the feed does not exist the data will be rejected.
+     * If it does exist it will be accepted regardless of the feed status.
+     * <p>
+     * If auto content creation is enabled on stroom, then when the check is performed,
+     * the feed will be auto-created (if various conditions for that are met) and this
+     * filter will then return true.
+     */
+    FEED_EXISTENCE(true),
+    /**
+     * No feed or receipt policy checks are performed.
+     * All data is accepted for receipt (subject to other checks like presence of stream type).
      */
     RECEIVE_ALL(true),
     /**

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/guice/ProxyModule.java

@@ -113,11 +113,11 @@ protected void configure() {
         ServletBinder.create(binder())
                 .bind(DebugServlet.class)
                 .bind(ProxyStatusServlet.class)
-                .bind(ProxyQueueMonitoringServlet.class)
                 .bind(ProxyWelcomeServlet.class)
                 .bind(ReceiveDataServlet.class);
 
         AdminServletBinder.create(binder())
+                .bind(ProxyQueueMonitoringServlet.class)
                 .bind(FilteredHealthCheckServlet.class);
 
         RestResourcesBinder.create(binder())

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/ForwardFileDestinationImpl.java

@@ -42,6 +42,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class ForwardFileDestinationImpl implements ForwardFileDestination {
 
@@ -104,7 +105,10 @@ public ForwardFileDestinationImpl(final Path storeDir,
             final String[] vars = pathCreator.findVars(pathTemplate);
             if (NullSafe.hasItems(vars)) {
                 staticBaseDir = null;
-                varsInTemplate = Set.of(vars);
+                // Do this rather than Set.of() because vars can be repeated in the arr
+                // which caused Set.of() to throw.
+                varsInTemplate = NullSafe.stream(vars)
+                        .collect(Collectors.toSet());
             } else {
                 staticBaseDir = resolveSubPath(pathTemplate);
                 FileUtil.ensureDirExists(staticBaseDir);

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/PathTemplateConfig.java

@@ -35,6 +35,7 @@
 @JsonPropertyOrder(alphabetic = true)
 public class PathTemplateConfig extends AbstractConfig implements IsProxyConfig {
 
+    public static final boolean DEFAULT_ENABLED_SATE = true;
     public static final TemplatingMode DEFAULT_TEMPLATING_MODE = TemplatingMode.REPLACE_UNKNOWN_PARAMS;
     public static final String DATE_AND_FEED_TEMPLATE = "${year}${month}${day}/${feed}";
     public static final PathTemplateConfig DEFAULT = new PathTemplateConfig(
@@ -50,18 +51,18 @@ public class PathTemplateConfig extends AbstractConfig implements IsProxyConfig
     private final TemplatingMode templatingMode;
 
     public PathTemplateConfig(final String pathTemplate) {
-        this(true, pathTemplate, DEFAULT_TEMPLATING_MODE);
+        this(DEFAULT_ENABLED_SATE, pathTemplate, DEFAULT_TEMPLATING_MODE);
     }
 
     public PathTemplateConfig(final String pathTemplate,
                               final TemplatingMode templatingMode) {
-        this(true, pathTemplate, templatingMode);
+        this(DEFAULT_ENABLED_SATE, pathTemplate, templatingMode);
     }
 
-    public PathTemplateConfig(@JsonProperty("enabled") final boolean enabled,
+    public PathTemplateConfig(@JsonProperty("enabled") final Boolean enabled,
                               @JsonProperty("pathTemplate") final String pathTemplate,
                               @JsonProperty("templatingMode") final TemplatingMode templatingMode) {
-        this.enabled = enabled;
+        this.enabled = Objects.requireNonNullElse(enabled, DEFAULT_ENABLED_SATE);
         this.templatingMode = Objects.requireNonNullElse(templatingMode, DEFAULT_TEMPLATING_MODE);
         this.pathTemplate = Objects.requireNonNullElse(pathTemplate, DATE_AND_FEED_TEMPLATE);
     }

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/handler/RemoteFeedStatusService.java

@@ -156,7 +156,7 @@ public GetFeedStatusResponse getFeedStatus(final GetFeedStatusRequestV2 request)
     private FeedStatus getDefaultFeedStatus() {
         final ReceiveDataConfig receiveDataConfig = receiveDataConfigProvider.get();
         return switch (receiveDataConfig.getReceiptCheckMode()) {
-            case FEED_STATUS, RECEIPT_POLICY -> switch (receiveDataConfig.getFallbackReceiveAction()) {
+            case FEED_STATUS, RECEIPT_POLICY, FEED_EXISTENCE -> switch (receiveDataConfig.getFallbackReceiveAction()) {
                 case RECEIVE -> FeedStatus.Receive;
                 case REJECT -> FeedStatus.Reject;
                 case DROP -> FeedStatus.Drop;

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/servlet/ProxyQueueMonitoringServlet.java

@@ -18,8 +18,7 @@
 
 import stroom.proxy.repo.queue.QueueMonitors;
 import stroom.proxy.repo.store.FileStores;
-import stroom.util.shared.IsServlet;
-import stroom.util.shared.ResourcePaths;
+import stroom.util.shared.IsAdminServlet;
 import stroom.util.shared.Unauthenticated;
 
 import jakarta.inject.Inject;
@@ -34,10 +33,9 @@
 import java.util.Set;
 
 @Unauthenticated
-public class ProxyQueueMonitoringServlet extends HttpServlet implements IsServlet {
+public class ProxyQueueMonitoringServlet extends HttpServlet implements IsAdminServlet {
 
-    private static final Set<String> PATH_SPECS = Set.of(
-            ResourcePaths.addLegacyUnauthenticatedServletPrefix("/queues"));
+    private static final Set<String> PATH_SPECS = Set.of("/queues");
 
     private final Provider<QueueMonitors> queueMonitorsProvider;
     private final Provider<FileStores> fileStoresProvider;
@@ -49,28 +47,27 @@ public ProxyQueueMonitoringServlet(final Provider<QueueMonitors> queueMonitorsPr
         this.fileStoresProvider = fileStoresProvider;
     }
 
-
     @Override
     protected void doGet(final HttpServletRequest request, final HttpServletResponse response)
             throws ServletException, IOException {
         final QueueMonitors queueMonitors = queueMonitorsProvider.get();
         final Writer writer = response.getWriter();
         writer.write("<html>\n" +
-                "<head>\n" +
-                "<style>\n" +
-                "body {\n" +
-                "\tfont-family: arial, tahoma, verdana;\n" +
-                "}\n" +
-                "</style>\n" +
-                "</head>\n" +
-                "<body>\n");
+                     "<head>\n" +
+                     "<style>\n" +
+                     "body {\n" +
+                     "\tfont-family: arial, tahoma, verdana;\n" +
+                     "}\n" +
+                     "</style>\n" +
+                     "</head>\n" +
+                     "<body>\n");
         writer.write("<h1>Queues</h1>");
         writer.write(queueMonitors.log());
         writer.write("<h1>File Stores</h1>");
         writer.write(fileStoresProvider.get().log());
 
         writer.write("</body>\n" +
-                "</html>");
+                     "</html>");
         writer.close();
     }
 

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/servlet/ProxyWelcomeServlet.java

@@ -16,6 +16,7 @@
 
 package stroom.proxy.app.servlet;
 
+import stroom.receive.common.ReceiveDataServlet;
 import stroom.util.date.DateUtil;
 import stroom.util.shared.BuildInfo;
 import stroom.util.shared.IsServlet;
@@ -53,34 +54,40 @@ protected void doGet(final HttpServletRequest request, final HttpServletResponse
         final BuildInfo buildInfo = buildInfoProvider.get();
         final Writer writer = response.getWriter();
         writer.write("<html>\n" +
-                "<head>\n" +
-                "<style>\n" +
-                "body {\n" +
-                "\tfont-family: arial, tahoma, verdana;\n" +
-                "}\n" +
-                "</style>\n" +
-                "</head>\n" +
-                "<body>\n" +
-                "<h1>Stroom Proxy " +
-                buildInfo.getBuildVersion() +
-                " built on " +
-                DateUtil.createNormalDateTimeString(buildInfo.getBuildTime()) +
-                "</h1>\n" +
-                "\n" +
-                "<p>Send data to " +
-                getURL(request) +
-                "datafeed</p>\n" +
-                "\n" +
-                "</body>\n" +
-                "</html>");
+                     "<head>\n" +
+                     "<style>\n" +
+                     "body {\n" +
+                     "\tfont-family: arial, tahoma, verdana;\n" +
+                     "}\n" +
+                     "</style>\n" +
+                     "</head>\n" +
+                     "<body>\n" +
+                     "<h1>Stroom Proxy " +
+                     buildInfo.getBuildVersion() +
+                     " built on " +
+                     DateUtil.createNormalDateTimeString(buildInfo.getBuildTime()) +
+                     "</h1>\n" +
+                     "\n" +
+                     "<p>Send data to " +
+                     getDataFeedURL(request) +
+                     "</p>\n" +
+                     "\n" +
+                     "</body>\n" +
+                     "</html>");
         writer.close();
     }
 
-    private String getURL(final HttpServletRequest request) {
+    private String getDataFeedURL(final HttpServletRequest request) {
         String url = request.getRequestURL().toString();
+        // First remove the /ui part
+        url = url.replace(PATH_PART, "");
         if (!url.endsWith("/")) {
             url += "/";
         }
+        // Add the /datafeed part
+        url += ReceiveDataServlet.DATA_FEED_PATH_PART;
+        // Remove any double slashes
+        url = url.replaceAll("(?<!:)//", "/");
         return url;
     }
 

stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/AttributeMapFilterFactory.java

@@ -119,9 +119,9 @@ private List<AttributeMapFilter> getReceiptCheckFilters(final ReceiptCheckMode r
             case RECEIPT_POLICY -> List.of(
                     dataReceiptPolicyAttrMapFilterFactoryProvider.get().create(),
                     feedExistenceAttributeMapFilterProvider.get());
+            case FEED_EXISTENCE -> List.of(feedExistenceAttributeMapFilterProvider.get());
             // Receiving everything
-            case RECEIVE_ALL -> List.of(
-                    feedExistenceAttributeMapFilterProvider.get());
+            case RECEIVE_ALL -> List.of(ReceiveAllAttributeMapFilter.getInstance());
             case REJECT_ALL -> List.of(RejectAllAttributeMapFilter.getInstance());
             case DROP_ALL -> List.of(DropAllAttributeMapFilter.getInstance());
         };

stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/FeedExistenceAttributeMapFilter.java

@@ -31,7 +31,9 @@
 
 /**
  * Checks whether the Feed in the {@link AttributeMap} exists or not.
- * If it does not exist then the filter will return false.
+ * If it does not exist then the filter will throw a {@link StroomStreamException} with
+ * statusCode {@link StroomStatusCode#FEED_IS_NOT_DEFINED}.
+ * <p>
  * If auto content creation is enabled on stroom, then when the check is performed,
  * the feed will be auto-created (if various conditions for that are met) and this
  * filter will then return true.