Addressing live test flakiness

sdk/storage/azure-storage-blob/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "java",
   "TagPrefix": "java/storage/azure-storage-blob",
-  "Tag": "java/storage/azure-storage-blob_e4ae407bc2"
+  "Tag": "java/storage/azure-storage-blob_0c7e10b175"
 }

sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java

@@ -22,6 +22,7 @@
 import com.azure.core.http.rest.Response;
 import com.azure.core.test.TestMode;
 import com.azure.core.test.TestProxyTestBase;
+import com.azure.core.test.http.MockHttpResponse;
 import com.azure.core.test.models.CustomMatcher;
 import com.azure.core.test.models.TestProxySanitizer;
 import com.azure.core.test.models.TestProxySanitizerType;
@@ -37,7 +38,6 @@
 import com.azure.identity.ChainedTokenCredentialBuilder;
 import com.azure.identity.DefaultAzureCredentialBuilder;
 import com.azure.identity.EnvironmentCredentialBuilder;
-import com.azure.json.JsonProviders;
 import com.azure.json.JsonSerializable;
 import com.azure.json.JsonWriter;
 import com.azure.storage.blob.models.BlobContainerItem;
@@ -68,7 +68,6 @@
 import reactor.core.publisher.Mono;
 import reactor.test.StepVerifier;
 
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -879,18 +878,18 @@ protected String generateShareName() {
         return generateResourceName(entityNo++);
     }
 
+    private HttpPipeline dataPlanePipeline;
+    private HttpHeaders genericHeaders;
+
     protected String createFileAndDirectoryWithoutFileShareDependency(byte[] data, String shareName)
         throws IOException {
         String accountName = ENVIRONMENT.getPrimaryAccount().getName();
         //authenticate
         BearerTokenAuthenticationPolicy credentialPolicyDataPlane = new BearerTokenAuthenticationPolicy(
             getTokenCredential(ENVIRONMENT.getTestMode()), Constants.STORAGE_SCOPE);
 
-        //create share through management plane
-        createFileShareWithoutDependency(shareName);
-
         //setup headers that will be used in every request
-        HttpHeaders genericHeaders = new HttpHeaders().set(X_MS_VERSION, "2025-07-05")
+        genericHeaders = new HttpHeaders().set(X_MS_VERSION, "2025-07-05")
             .set(HttpHeaderName.ACCEPT, "application/xml")
             .set(HttpHeaderName.HOST, accountName + ".file.core.windows.net")
             .set(HttpHeaderName.CONTENT_LENGTH, "0")
@@ -904,8 +903,15 @@ protected String createFileAndDirectoryWithoutFileShareDependency(byte[] data, S
         policies.add(new RequestIdPolicy());
 
         // create data plane pipeline
-        HttpPipeline dataPlanePipeline
-            = new HttpPipelineBuilder().policies(policies.toArray(new HttpPipelinePolicy[0])).build();
+        dataPlanePipeline = new HttpPipelineBuilder().policies(policies.toArray(new HttpPipelinePolicy[0])).build();
+
+        // create share through data plane pipeline
+        String shareUrl = String.format("https://%s.file.core.windows.net/%s?restype=share", accountName, shareName);
+
+        HttpResponse shareCreateResponse
+            = dataPlanePipeline.send(new HttpRequest(HttpMethod.PUT, new URL(shareUrl), genericHeaders)).block();
+        assertNotNull(shareCreateResponse);
+        assertEquals(201, shareCreateResponse.getStatusCode());
 
         // create directory
         String directoryName = generateBlobName();
@@ -945,48 +951,14 @@ protected String createFileAndDirectoryWithoutFileShareDependency(byte[] data, S
         return fileUrl;
     }
 
-    protected void createFileShareWithoutDependency(String shareName) throws IOException {
-        String shareID = getFileShareID(shareName);
-        Body shareBody = new Body();
-        shareBody.setId(shareID);
-        shareBody.setName(shareName);
-        shareBody.setType("Microsoft.Storage/storageAccounts/fileServices/shares");
-
-        ByteArrayOutputStream shareJson = new ByteArrayOutputStream();
-        try (JsonWriter jsonWriter = JsonProviders.createWriter(shareJson)) {
-            shareBody.toJson(jsonWriter);
-        }
-        HttpResponse response
-            = getManagementPlanePipeline().send(new HttpRequest(HttpMethod.PUT, new URL(getFileShareUri(shareID)),
-                new HttpHeaders(), Flux.just(ByteBuffer.wrap(shareJson.toByteArray())))).block();
-        assertNotNull(response);
-        assertEquals(201, response.getStatusCode());
-    }
-
     protected void deleteFileShareWithoutDependency(String shareName) throws IOException {
-        String shareID = getFileShareID(shareName);
-        HttpResponse response = getManagementPlanePipeline()
-            .send(new HttpRequest(HttpMethod.DELETE, new URL(getFileShareUri(shareID)), new HttpHeaders()))
-            .block();
-        assertNotNull(response);
-        assertEquals(200, response.getStatusCode());
-    }
-
-    protected HttpPipeline getManagementPlanePipeline() {
-        BearerTokenAuthenticationPolicy credentialPolicyManagementPlane = new BearerTokenAuthenticationPolicy(
-            getTokenCredential(ENVIRONMENT.getTestMode()), "https://management.azure.com/.default");
-        return new HttpPipelineBuilder().policies(credentialPolicyManagementPlane).build();
-    }
+        String shareUrl = String.format("https://%s.file.core.windows.net/%s?restype=share",
+            ENVIRONMENT.getPrimaryAccount().getName(), shareName);
 
-    protected String getFileShareID(String shareName) {
-        return String.format(
-            "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/"
-                + "%s/fileServices/default/shares/%s",
-            SUBSCRIPTION_ID, RESOURCE_GROUP_NAME, ENVIRONMENT.getPrimaryAccount().getName(), shareName);
-    }
-
-    protected String getFileShareUri(String fileShareID) {
-        return "https://management.azure.com" + fileShareID + "?api-version=2024-01-01";
+        HttpResponse shareDeleteResponse
+            = dataPlanePipeline.send(new HttpRequest(HttpMethod.DELETE, new URL(shareUrl), genericHeaders)).block();
+        assertNotNull(shareDeleteResponse);
+        assertEquals(202, shareDeleteResponse.getStatusCode());
     }
 
     public static final class Body implements JsonSerializable<Body> {
@@ -1073,7 +1045,7 @@ public JsonWriter toJson(JsonWriter jsonWriter) throws IOException {
         }
     }
 
-    //todo: change the copy of this method in StorageCommonTestUtils to take in TestMode instead of interception manager
+    //todo isbr: change the copy of this method in StorageCommonTestUtils to take in TestMode instead of interception manager
     protected static TokenCredential getTokenCredential(TestMode testMode) {
         if (testMode == TestMode.RECORD) {
             return new DefaultAzureCredentialBuilder().build();
@@ -1109,4 +1081,172 @@ protected static TokenCredential getTokenCredential(TestMode testMode) {
             return new MockTokenCredential();
         }
     }
+
+    protected static final class ListBlobsWithTimeoutTestClient implements HttpClient {
+        private final boolean isAsync;
+
+        ListBlobsWithTimeoutTestClient(Boolean isAsync) {
+            this.isAsync = isAsync;
+        }
+
+        private HttpResponse response(HttpRequest request, String xml) {
+            HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/xml");
+            return new MockHttpResponse(request, 200, headers, xml.getBytes(StandardCharsets.UTF_8));
+        }
+
+        private String buildFirstRequest(Boolean useDelimiter) {
+            String delimiterString = useDelimiter ? "<Delimiter>/</Delimiter>" : "";
+
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\" ContainerName=\"foo\">"
+                + "<MaxResults>3</MaxResults>" + delimiterString + "<Blobs>" + "<Blob>" + "<Name>blob1</Name>"
+                + "</Blob>" + "<Blob>" + "<Name>blob2</Name>" + "</Blob>" + "<Blob>" + "<Name>blob3</Name>" + "</Blob>"
+                + "</Blobs>" + "<NextMarker>MARKER--</NextMarker>" + "</EnumerationResults>";
+        }
+
+        private String buildSecondRequest(Boolean useDelimiter) {
+            String delimiterString = useDelimiter ? "<Delimiter>/</Delimiter>" : "";
+
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\" ContainerName=\"foo\">"
+                + "<Marker>MARKER--</Marker>" + "<MaxResults>3</MaxResults>" + delimiterString + "<Blobs>" + "<Blob>"
+                + "<Name>blob4</Name>" + "</Blob>" + "<Blob>" + "<Name>blob5</Name>" + "</Blob>" + "</Blobs>"
+                + "<NextMarker/>" + "</EnumerationResults>";
+        }
+
+        @Override
+        public Mono<HttpResponse> send(HttpRequest request) {
+            String url = request.getUrl().toString();
+            HttpResponse response;
+            int delay = isAsync ? 4 : 8;
+
+            if (url.contains("?restype=container&comp=list&maxresults=")) {
+                // flat first request
+                response = response(request, buildFirstRequest(false));
+            } else if (url.contains("?restype=container&comp=list&marker=")) {
+                // flat second request
+                response = response(request, buildSecondRequest(false));
+            } else if (url.contains("?restype=container&comp=list&delimiter=/&maxresults=")) {
+                // hierarchy first request
+                response = response(request, buildFirstRequest(true));
+            } else if (url.contains("?restype=container&comp=list&delimiter=/&marker=")) {
+                // hierarchy second request
+                response = response(request, buildSecondRequest(true));
+            } else {
+                // fallback
+                return Mono.just(new MockHttpResponse(request, 404));
+            }
+
+            return Mono.delay(Duration.ofSeconds(delay)).then(Mono.just(response));
+        }
+    }
+
+    protected static final class FindBlobsWithTimeoutClient implements HttpClient {
+        private final boolean isAsync;
+
+        FindBlobsWithTimeoutClient(Boolean isAsync) {
+            this.isAsync = isAsync;
+        }
+
+        private HttpResponse response(HttpRequest request, String xml) {
+            HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/xml");
+            return new MockHttpResponse(request, 200, headers, xml.getBytes(StandardCharsets.UTF_8));
+        }
+
+        private String buildFirstRequest() {
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\">"
+                + "<Where>&quot;dummyKey&quot;=&apos;dummyValue&apos;</Where>" + "<MaxResults>3</MaxResults>"
+                + "<Blobs>" + "<Blob>" + "<Name>blob1</Name>" + "<ContainerName>foo</ContainerName>" + "<Tags>"
+                + "<TagSet>" + "<Tag>" + "<Key>dummyKey</Key>" + "<Value>dummyValue</Value>" + "</Tag>" + "</TagSet>"
+                + "</Tags>" + "</Blob>" + "<Blob>" + "<Name>blob2</Name>" + "<ContainerName>foo</ContainerName>"
+                + "<Tags>" + "<TagSet>" + "<Tag>" + "<Key>dummyKey</Key>" + "<Value>dummyValue</Value>" + "</Tag>"
+                + "</TagSet>" + "</Tags>" + "</Blob>" + "<Blob>" + "<Name>blob3</Name>"
+                + "<ContainerName>foo</ContainerName>" + "<Tags>" + "<TagSet>" + "<Tag>" + "<Key>dummyKey</Key>"
+                + "<Value>dummyValue</Value>" + "</Tag>" + "</TagSet>" + "</Tags>" + "</Blob>" + "</Blobs>"
+                + "<NextMarker>MARKER-</NextMarker>" + "</EnumerationResults>";
+        }
+
+        private String buildSecondRequest() {
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\">"
+                + "<Marker>MARKER-</Marker>" + "<Where>&quot;dummyKey&quot;=&apos;dummyValue&apos;</Where>"
+                + "<MaxResults>3</MaxResults>" + "<Blobs>" + "<Blob>" + "<Name>blob4</Name>"
+                + "<ContainerName>foo</ContainerName>" + "<Tags>" + "<TagSet>" + "<Tag>" + "<Key>dummyKey</Key>"
+                + "<Value>dummyValue</Value>" + "</Tag>" + "</TagSet>" + "</Tags>" + "</Blob>" + "<Blob>"
+                + "<Name>blob5</Name>" + "<ContainerName>foo</ContainerName>" + "<Tags>" + "<TagSet>" + "<Tag>"
+                + "<Key>dummyKey</Key>" + "<Value>dummyValue</Value>" + "</Tag>" + "</TagSet>" + "</Tags>" + "</Blob>"
+                + "</Blobs>" + "<NextMarker/>" + "</EnumerationResults>";
+        }
+
+        @Override
+        public Mono<HttpResponse> send(HttpRequest request) {
+            String url = request.getUrl().toString();
+            HttpResponse response;
+            int delay = isAsync ? 4 : 8;
+
+            if (url.contains("marker")) {
+                // second request
+                response = response(request, buildSecondRequest());
+            } else if (url.contains("?comp=blobs&where=%") || url.contains("?restype=container&comp=blobs&where=%")) {
+                // first request
+                response = response(request, buildFirstRequest());
+            } else {
+                // fallback
+                return Mono.just(new MockHttpResponse(request, 404));
+            }
+
+            return Mono.delay(Duration.ofSeconds(delay)).then(Mono.just(response));
+        }
+    }
+
+    protected static final class ListContainersWithTimeoutTestClient implements HttpClient {
+        private final boolean isAsync;
+
+        ListContainersWithTimeoutTestClient(Boolean isAsync) {
+            this.isAsync = isAsync;
+        }
+
+        private HttpResponse response(HttpRequest request, String xml) {
+            HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/xml");
+            return new MockHttpResponse(request, 200, headers, xml.getBytes(StandardCharsets.UTF_8));
+        }
+
+        private String buildFirstRequest() {
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\">"
+                + "<MaxResults>3</MaxResults>" + "<Containers>" + "<Container>" + "<Name>container1</Name>"
+                + "</Container>" + "<Container>" + "<Name>container2</Name>" + "</Container>" + "<Container>"
+                + "<Name>container3</Name>" + "</Container>" + "</Containers>"
+                + "<NextMarker>/marker/marker</NextMarker>" + "</EnumerationResults>";
+        }
+
+        private String buildSecondRequest() {
+            return "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
+                + "<EnumerationResults ServiceEndpoint=\"https://account.blob.core.windows.net/\">"
+                + "<Marker>/marker/marker</Marker>" + "<MaxResults>3</MaxResults>" + "<Containers>" + "<Container>"
+                + "<Name>container4</Name>" + "</Container>" + "<Container>" + "<Name>container5</Name>"
+                + "</Container>" + "</Containers>" + "<NextMarker/>" + "</EnumerationResults>";
+        }
+
+        @Override
+        public Mono<HttpResponse> send(HttpRequest request) {
+            String url = request.getUrl().toString();
+            HttpResponse response;
+            int delay = isAsync ? 4 : 8;
+
+            if (url.contains("?comp=list&maxresults=")) {
+                // flat first request
+                response = response(request, buildFirstRequest());
+            } else if (url.contains("?comp=list&marker=")) {
+                // flat second request
+                response = response(request, buildSecondRequest());
+            } else {
+                // fallback
+                return Mono.just(new MockHttpResponse(request, 404));
+            }
+
+            return Mono.delay(Duration.ofSeconds(delay)).then(Mono.just(response));
+        }
+    }
 }