Back out "http: add "per batch" concurrent request limit"

Summary:
Original commit changeset: 64bd8dea2326

Original Phabricator Diff: D72823375

Reviewed By: MichaelCuevas

Differential Revision: D73602841

fbshipit-source-id: 33b08997986b0b8aaf9cafecbeaa90bc62b8569b

Author: lXXXw

eden/scm/lib/edenapi/src/builder.rs

@@ -209,9 +209,6 @@ impl HttpClientBuilder {
         let max_requests = get_config(config, "edenapi", "max-concurrent-requests")?
             .or(get_config(config, "edenapi", "maxrequests")?);
 
-        let max_requests_per_batch =
-            get_config(config, "edenapi", "max-concurrent-requests-per-batch")?;
-
         let try_route_consistently =
             get_config(config, "edenapi", "try-route-consistently")?.unwrap_or_default();
 
@@ -306,7 +303,6 @@ impl HttpClientBuilder {
         let mut http_config = hg_http::http_config(config, &server_url)?;
         http_config.verbose_stats |= debug;
         http_config.max_concurrent_requests = max_requests;
-        http_config.max_concurrent_requests_per_batch = max_requests_per_batch;
 
         let builder = HttpClientBuilder {
             repo_name,

eden/scm/lib/http-client/src/client.rs

@@ -65,13 +65,6 @@ pub struct Config {
     // library to limit the number of in-flight requests separately, _before_ the
     // requests are given to curl.
     pub max_concurrent_requests: Option<usize>,
-
-    // Limit the number of concurrent requests for a "batch" of requests passed at once to
-    // client.send_async(). This allows us to have a high global request limit for small
-    // "random" requests, while having lower limits for heavy requests (e.g. fetch 10m
-    // files across 1000 requests).
-    pub max_concurrent_requests_per_batch: Option<usize>,
-
     // Escape hatch to turn off our request limiting.
     pub limit_requests: bool,
     // Escape hatch to turn off our response body limiting.
@@ -102,7 +95,6 @@ impl Default for Config {
             client_info: None,
             disable_tls_verification: false,
             max_concurrent_requests: None, // No limit by default
-            max_concurrent_requests_per_batch: None,
             limit_requests: true,
             limit_response_buffering: true,
             unix_socket_domains: HashSet::new(),
@@ -253,37 +245,23 @@ impl HttpClient {
     /// until all of the transfers are complete, and will return
     /// the total stats across all transfers when complete.
     pub fn stream(&self, requests: Vec<StreamRequest>) -> Result<Stats, HttpClientError> {
-        // This is a "local" limit for how many concurrent requests we allow for a single
-        // batch of requests. Requests are still subject to the global limit via self.claimer.
-        let mut allowed_requests = self
-            .config
-            .max_concurrent_requests_per_batch
-            .unwrap_or(requests.len());
-
         // Add as many of remaining requests to the handle as we can, limited by the claimer.
-        let try_add = |h: &MultiDriver,
-                       reqs: &mut IntoIter<StreamRequest>,
-                       allowed_requests: &mut usize|
-         -> Result<(), HttpClientError> {
-            for claim in self
-                .claimer
-                .try_claim_requests((*allowed_requests).min(reqs.len()))
-            {
-                let mut request = match reqs.next() {
-                    Some(request) => request,
-                    // Shouldn't happen, but just in case.
-                    None => break,
-                };
-
-                self.event_listeners
-                    .trigger_new_request(request.request.ctx_mut());
-                h.add(request.into_easy(claim)?)?;
-
-                *allowed_requests -= 1;
-            }
+        let try_add =
+            |h: &MultiDriver, reqs: &mut IntoIter<StreamRequest>| -> Result<(), HttpClientError> {
+                for claim in self.claimer.try_claim_requests(reqs.len()) {
+                    let mut request = match reqs.next() {
+                        Some(request) => request,
+                        // Shouldn't happen, but just in case.
+                        None => break,
+                    };
+
+                    self.event_listeners
+                        .trigger_new_request(request.request.ctx_mut());
+                    h.add(request.into_easy(claim)?)?;
+                }
 
-            Ok(())
-        };
+                Ok(())
+            };
 
         let mut requests = requests.into_iter();
         let mut stats = Stats::default();
@@ -298,7 +276,7 @@ impl HttpClient {
             let driver = MultiDriver::new(multi.get(), self.config.verbose_stats);
 
             // Add requests to the driver. This can add anywhere from zero to all the requests.
-            try_add(&driver, &mut requests, &mut allowed_requests)?;
+            try_add(&driver, &mut requests)?;
 
             let mut tls_error = false;
             let result = driver
@@ -312,17 +290,14 @@ impl HttpClient {
 
                     self.report_result_and_drop_receiver(res)?;
 
-                    allowed_requests += 1;
-
                     // A request finished - let's see if there are pending requests we can now add
                     // to this multi. This allows pending requests to proceed without needing to
                     // wait for _all_ in-progress requests to finish. Note that there may be other
                     // curl multis active bound by the same request limit, so it is still possible
                     // for our pending requests to wait longer than they need to (i.e. when a
                     // request finishes on a different multi, our loop here will still wait for one
                     // of our requests to finish before trying to enqueue new requests).
-                    try_add(&driver, &mut requests, &mut allowed_requests)
-                        .map_err(|err| Abort::WithReason(err.into()))
+                    try_add(&driver, &mut requests).map_err(|err| Abort::WithReason(err.into()))
                 })
                 .inspect(|stats| {
                     self.event_listeners.trigger_stats(stats);