Add tests for IO Error retries (#13627)

Often, HTTP requests don't fail due to server errors, but from spurious
network errors such as connection resets. reqwest surfaces these as
`io::Error`, and we have to handle their retrying separately.

Companion PR: LukeMathWalker/wiremock-rs#159

Author: konstin

Cargo.lock

@@ -189,7 +189,7 @@ windows-core = { version = "0.59.0" }
 windows-registry = { version = "0.5.0" }
 windows-result = { version = "0.3.0" }
 windows-sys = { version = "0.59.0", features = ["Win32_Foundation", "Win32_Security", "Win32_Storage_FileSystem", "Win32_System_Ioctl", "Win32_System_IO", "Win32_System_Registry"] }
-wiremock = { version = "0.6.2" }
+wiremock = { git = "https://github.com/astral-sh/wiremock-rs", rev = "b79b69f62521df9f83a54e866432397562eae789" }
 xz2 = { version = "0.1.7" }
 zip = { version = "2.2.3", default-features = false, features = ["deflate", "zstd", "bzip2", "lzma", "xz"] }
 

Cargo.toml

@@ -1,31 +1,54 @@
-use std::env;
+use std::{env, io};
 
-use assert_fs::fixture::{FileWriteStr, PathChild};
+use assert_fs::fixture::{ChildPath, FileWriteStr, PathChild};
 use http::StatusCode;
 use serde_json::json;
 use wiremock::matchers::method;
 use wiremock::{Mock, MockServer, ResponseTemplate};
 
 use crate::common::{TestContext, uv_snapshot};
 
-/// Check the simple index error message when the server returns HTTP status 500, a retryable error.
-#[tokio::test]
-async fn simple_http_500() {
-    let context = TestContext::new("3.12");
+fn connection_reset(_request: &wiremock::Request) -> io::Error {
+    io::Error::new(io::ErrorKind::ConnectionReset, "Connection reset by peer")
+}
 
+/// Answers with a retryable HTTP status 500.
+async fn http_error_server() -> (MockServer, String) {
     let server = MockServer::start().await;
     Mock::given(method("GET"))
         .respond_with(ResponseTemplate::new(StatusCode::INTERNAL_SERVER_ERROR))
         .mount(&server)
         .await;
+
+    let mock_server_uri = server.uri();
+    (server, mock_server_uri)
+}
+
+/// Answers with a retryable connection reset IO error.
+async fn io_error_server() -> (MockServer, String) {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with_err(connection_reset)
+        .mount(&server)
+        .await;
+
     let mock_server_uri = server.uri();
+    (server, mock_server_uri)
+}
+
+/// Check the simple index error message when the server returns HTTP status 500, a retryable error.
+#[tokio::test]
+async fn simple_http_500() {
+    let context = TestContext::new("3.12");
+
+    let (_server_drop_guard, mock_server_uri) = http_error_server().await;
 
     let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
     uv_snapshot!(filters, context
         .pip_install()
         .arg("tqdm")
         .arg("--index-url")
-        .arg(server.uri()), @r"
+        .arg(&mock_server_uri), @r"
     success: false
     exit_code: 2
     ----- stdout -----
@@ -36,25 +59,46 @@ async fn simple_http_500() {
     ");
 }
 
+/// Check the simple index error message when the server returns a retryable IO error.
+#[tokio::test]
+async fn simple_io_err() {
+    let context = TestContext::new("3.12");
+
+    let (_server_drop_guard, mock_server_uri) = io_error_server().await;
+
+    let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
+    uv_snapshot!(filters, context
+        .pip_install()
+        .arg("tqdm")
+        .arg("--index-url")
+        .arg(&mock_server_uri), @r"
+    success: false
+    exit_code: 2
+    ----- stdout -----
+
+    ----- stderr -----
+    error: Failed to fetch: `[SERVER]/tqdm/`
+      Caused by: Request failed after 3 retries
+      Caused by: error sending request for url ([SERVER]/tqdm/)
+      Caused by: client error (SendRequest)
+      Caused by: connection closed before message completed
+    ");
+}
+
 /// Check the find links error message when the server returns HTTP status 500, a retryable error.
 #[tokio::test]
 async fn find_links_http_500() {
     let context = TestContext::new("3.12");
 
-    let server = MockServer::start().await;
-    Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(StatusCode::INTERNAL_SERVER_ERROR))
-        .mount(&server)
-        .await;
-    let mock_server_uri = server.uri();
+    let (_server_drop_guard, mock_server_uri) = http_error_server().await;
 
     let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
     uv_snapshot!(filters, context
         .pip_install()
         .arg("tqdm")
         .arg("--no-index")
         .arg("--find-links")
-        .arg(server.uri()), @r"
+        .arg(&mock_server_uri), @r"
     success: false
     exit_code: 2
     ----- stdout -----
@@ -66,18 +110,41 @@ async fn find_links_http_500() {
     ");
 }
 
+/// Check the find links error message when the server returns a retryable IO error.
+#[tokio::test]
+async fn find_links_io_error() {
+    let context = TestContext::new("3.12");
+
+    let (_server_drop_guard, mock_server_uri) = io_error_server().await;
+
+    let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
+    uv_snapshot!(filters, context
+        .pip_install()
+        .arg("tqdm")
+        .arg("--no-index")
+        .arg("--find-links")
+        .arg(&mock_server_uri), @r"
+    success: false
+    exit_code: 2
+    ----- stdout -----
+
+    ----- stderr -----
+    error: Failed to read `--find-links` URL: [SERVER]/
+      Caused by: Failed to fetch: `[SERVER]/`
+      Caused by: Request failed after 3 retries
+      Caused by: error sending request for url ([SERVER]/)
+      Caused by: client error (SendRequest)
+      Caused by: connection closed before message completed
+    ");
+}
+
 /// Check the direct package URL error message when the server returns HTTP status 500, a retryable
 /// error.
 #[tokio::test]
 async fn direct_url_http_500() {
     let context = TestContext::new("3.12");
 
-    let server = MockServer::start().await;
-    Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(StatusCode::INTERNAL_SERVER_ERROR))
-        .mount(&server)
-        .await;
-    let mock_server_uri = server.uri();
+    let (_server_drop_guard, mock_server_uri) = http_error_server().await;
 
     let tqdm_url = format!(
         "{mock_server_uri}/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl"
@@ -97,22 +164,35 @@ async fn direct_url_http_500() {
     ");
 }
 
-/// Check the Python install error message when the server returns HTTP status 500, a retryable
-/// error.
+/// Check the direct package URL error message when the server returns a retryable IO error.
 #[tokio::test]
-async fn python_install_http_500() {
-    let context = TestContext::new("3.12")
-        .with_filtered_python_keys()
-        .with_filtered_exe_suffix()
-        .with_managed_python_dirs();
+async fn direct_url_io_error() {
+    let context = TestContext::new("3.12");
 
-    let server = MockServer::start().await;
-    Mock::given(method("GET"))
-        .respond_with(ResponseTemplate::new(StatusCode::INTERNAL_SERVER_ERROR))
-        .mount(&server)
-        .await;
-    let mock_server_uri = server.uri();
+    let (_server_drop_guard, mock_server_uri) = io_error_server().await;
 
+    let tqdm_url = format!(
+        "{mock_server_uri}/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl"
+    );
+    let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
+    uv_snapshot!(filters, context
+        .pip_install()
+        .arg(format!("tqdm @ {tqdm_url}")), @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+
+    ----- stderr -----
+      × Failed to download `tqdm @ [SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl`
+      ├─▶ Failed to fetch: `[SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl`
+      ├─▶ Request failed after 3 retries
+      ├─▶ error sending request for url ([SERVER]/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl)
+      ├─▶ client error (SendRequest)
+      ╰─▶ connection closed before message completed
+    ");
+}
+
+fn write_python_downloads_json(context: &TestContext, mock_server_uri: &String) -> ChildPath {
     let python_downloads_json = context.temp_dir.child("python_downloads.json");
     let interpreter = json!({
         "cpython-3.10.0-darwin-aarch64-none": {
@@ -135,6 +215,21 @@ async fn python_install_http_500() {
     python_downloads_json
         .write_str(&serde_json::to_string(&interpreter).unwrap())
         .unwrap();
+    python_downloads_json
+}
+
+/// Check the Python install error message when the server returns HTTP status 500, a retryable
+/// error.
+#[tokio::test]
+async fn python_install_http_500() {
+    let context = TestContext::new("3.12")
+        .with_filtered_python_keys()
+        .with_filtered_exe_suffix()
+        .with_managed_python_dirs();
+
+    let (_server_drop_guard, mock_server_uri) = http_error_server().await;
+
+    let python_downloads_json = write_python_downloads_json(&context, &mock_server_uri);
 
     let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
     uv_snapshot!(filters, context
@@ -152,3 +247,35 @@ async fn python_install_http_500() {
       Caused by: HTTP status server error (500 Internal Server Error) for url ([SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst)
     ");
 }
+
+/// Check the Python install error message when the server returns a retryable IO error.
+#[tokio::test]
+async fn python_install_io_error() {
+    let context = TestContext::new("3.12")
+        .with_filtered_python_keys()
+        .with_filtered_exe_suffix()
+        .with_managed_python_dirs();
+
+    let (_server_drop_guard, mock_server_uri) = io_error_server().await;
+
+    let python_downloads_json = write_python_downloads_json(&context, &mock_server_uri);
+
+    let filters = vec![(mock_server_uri.as_str(), "[SERVER]")];
+    uv_snapshot!(filters, context
+        .python_install()
+        .arg("cpython-3.10.0-darwin-aarch64-none")
+        .arg("--python-downloads-json-url")
+        .arg(python_downloads_json.path()), @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+
+    ----- stderr -----
+    error: Failed to install cpython-3.10.0-macos-aarch64-none
+      Caused by: Failed to download [SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst
+      Caused by: Request failed after 3 retries
+      Caused by: error sending request for url ([SERVER]/astral-sh/python-build-standalone/releases/download/20211017/cpython-3.10.0-aarch64-apple-darwin-pgo%2Blto-20211017T1616.tar.zst)
+      Caused by: client error (SendRequest)
+      Caused by: connection closed before message completed
+    ");
+}