From 289865b2ebc20f4304d811bba80d24a14a888ec9 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 19:04:46 -0700 Subject: [PATCH 1/7] Move download code into rust_utils::download --- src/rustup-utils/src/download.rs | 379 +++++++++++++++++++++++++++++++ src/rustup-utils/src/lib.rs | 1 + src/rustup-utils/src/raw.rs | 357 ----------------------------- src/rustup-utils/src/utils.rs | 3 +- 4 files changed, 382 insertions(+), 358 deletions(-) create mode 100644 src/rustup-utils/src/download.rs diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs new file mode 100644 index 0000000000..702fec447f --- /dev/null +++ b/src/rustup-utils/src/download.rs @@ -0,0 +1,379 @@ + + +use errors::*; +use notifications::Notification; +use sha2::Sha256; +use std::env; +use std::path::Path; +use url::Url; + +pub fn download_file(url: &Url, + path: &Path, + hasher: Option<&mut Sha256>, + notify_handler: &Fn(Notification)) + -> Result<()> { + if env::var_os("RUSTUP_USE_HYPER").is_some() { + self::hyper::download_file(url, path, hasher, notify_handler) + } else { + self::curl::download_file(url, path, hasher, notify_handler) + } +} + +mod curl { + use curl::easy::Easy; + use errors::*; + use notifications::Notification; + use sha2::{Sha256, Digest}; + use std::cell::RefCell; + use std::fs; + use std::path::Path; + use std::str; + use std::time::Duration; + use url::Url; + + pub fn download_file(url: &Url, + path: &Path, + mut hasher: Option<&mut Sha256>, + notify_handler: &Fn(Notification)) + -> Result<()> { + use notifications::Notification; + use std::io::Write; + + let mut file = try!(fs::File::create(&path).chain_err( + || "error creating file for download")); + + // Fetch either a cached libcurl handle (which will preserve open + // connections) or create a new one if it isn't listed. + // + // Once we've acquired it, reset the lifetime from 'static to our local + // scope. + thread_local!(static EASY: RefCell = RefCell::new(Easy::new())); + EASY.with(|handle| { + let mut handle = handle.borrow_mut(); + + try!(handle.url(&url.to_string()).chain_err(|| "failed to set url")); + try!(handle.follow_location(true).chain_err(|| "failed to set follow redirects")); + + // Take at most 30s to connect + try!(handle.connect_timeout(Duration::new(30, 0)).chain_err(|| "failed to set connect timeout")); + + // Fail if less than 10 bytes are transferred every 30 seconds + try!(handle.low_speed_limit(10).chain_err(|| "failed to set low speed limit")); + try!(handle.low_speed_time(Duration::new(30, 0)).chain_err(|| "failed to set low speed time")); + + { + let fserr = RefCell::new(None); + let mut transfer = handle.transfer(); + + // Data callback for libcurl which is called with data that's + // downloaded. We just feed it into our hasher and also write it out + // to disk. + try!(transfer.write_function(|data| { + if let Some(ref mut h) = hasher { + h.input(data); + } + notify_handler(Notification::DownloadDataReceived(data.len())); + match file.write_all(data) { + Ok(()) => Ok(data.len()), + Err(e) => { + *fserr.borrow_mut() = Some(e); + Ok(0) + } + } + }).chain_err(|| "failed to set write")); + + // Listen for headers and parse out a `Content-Length` if it comes + // so we know how much we're downloading. + try!(transfer.header_function(|header| { + if let Ok(data) = str::from_utf8(header) { + let prefix = "Content-Length: "; + if data.starts_with(prefix) { + if let Ok(s) = data[prefix.len()..].trim().parse() { + let msg = Notification::DownloadContentLengthReceived(s); + notify_handler(msg); + } + } + } + true + }).chain_err(|| "failed to set header")); + + // If an error happens check to see if we had a filesystem error up + // in `fserr`, but we always want to punt it up. + try!(transfer.perform().or_else(|e| { + match fserr.borrow_mut().take() { + Some(fs) => Err(fs).chain_err(|| ErrorKind::HttpError(e)), + None => Err(ErrorKind::HttpError(e).into()) + } + })); + } + + // If we didn't get a 200 or 0 ("OK" for files) then return an error + let code = try!(handle.response_code().chain_err(|| "failed to get response code")); + if code != 200 && code != 0 { + return Err(ErrorKind::HttpStatus(code).into()); + } + + notify_handler(Notification::DownloadFinished); + Ok(()) + }) + } +} + +mod hyper { + use hyper; + use notifications::Notification; + use sha2::{Digest, Sha256}; + use std::fs; + use std::io; + use std::path::Path; + use std::time::Duration; + use url::Url; + use errors::*; + + fn proxy_from_env(url: &Url) -> Option<(String, u16)> { + use std::env::var_os; + + let mut maybe_https_proxy = var_os("https_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); + if maybe_https_proxy.is_none() { + maybe_https_proxy = var_os("HTTPS_PROXY").map(|ref v| v.to_str().unwrap_or("").to_string()); + } + let maybe_http_proxy = var_os("http_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); + let mut maybe_all_proxy = var_os("all_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); + if maybe_all_proxy.is_none() { + maybe_all_proxy = var_os("ALL_PROXY").map(|ref v| v.to_str().unwrap_or("").to_string()); + } + if let Some(url_value) = match url.scheme() { + "https" => maybe_https_proxy.or(maybe_http_proxy.or(maybe_all_proxy)), + "http" => maybe_http_proxy.or(maybe_all_proxy), + _ => maybe_all_proxy, + } { + if let Ok(proxy_url) = Url::parse(&url_value) { + if let Some(host) = proxy_url.host_str() { + let port = proxy_url.port().unwrap_or(8080); + return Some((host.to_string(), port)); + } + } + } + None + } + + pub fn download_file(url: &Url, + path: &Path, + mut hasher: Option<&mut Sha256>, + notify_handler: &Fn(Notification)) + -> Result<()> { + + // Short-circuit hyper for the "file:" URL scheme + if try!(download_from_file_url(url, path, &mut hasher)) { + return Ok(()); + } + + use hyper::client::{Client, ProxyConfig}; + use hyper::error::Result as HyperResult; + use hyper::header::ContentLength; + use hyper::net::{SslClient, NetworkStream, HttpsConnector}; + use native_tls; + use std::io::Result as IoResult; + use std::io::{Read, Write}; + use std::net::{SocketAddr, Shutdown}; + use std::sync::{Arc, Mutex}; + + // The Hyper HTTP client + let client; + + let maybe_proxy = proxy_from_env(url); + if url.scheme() == "https" { + + // All the following is adapter code to use native_tls with hyper. + + struct NativeSslClient; + + impl SslClient for NativeSslClient { + type Stream = NativeSslStream; + + fn wrap_client(&self, stream: T, host: &str) -> HyperResult { + use native_tls::ClientBuilder as TlsClientBuilder; + use hyper::error::Error as HyperError; + + let mut ssl_builder = try!(TlsClientBuilder::new() + .map_err(|e| HyperError::Ssl(Box::new(e)))); + let ssl_stream = try!(ssl_builder.handshake(host, stream) + .map_err(|e| HyperError::Ssl(Box::new(e)))); + + Ok(NativeSslStream(Arc::new(Mutex::new(ssl_stream)))) + } + } + + #[derive(Clone)] + struct NativeSslStream(Arc>>); + + #[derive(Debug)] + struct NativeSslPoisonError; + + impl ::std::error::Error for NativeSslPoisonError { + fn description(&self) -> &str { "mutex poisoned during TLS operation" } + } + + impl ::std::fmt::Display for NativeSslPoisonError { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::result::Result<(), ::std::fmt::Error> { + f.write_str(::std::error::Error::description(self)) + } + } + + impl NetworkStream for NativeSslStream + where T: NetworkStream + { + fn peer_addr(&mut self) -> IoResult { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|mut t| t.get_mut().peer_addr()) + } + fn set_read_timeout(&self, dur: Option) -> IoResult<()> { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|t| t.get_ref().set_read_timeout(dur)) + } + fn set_write_timeout(&self, dur: Option) -> IoResult<()> { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|t| t.get_ref().set_write_timeout(dur)) + } + fn close(&mut self, how: Shutdown) -> IoResult<()> { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|mut t| t.get_mut().close(how)) + } + } + + impl Read for NativeSslStream + where T: Read + Write + { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|mut t| t.read(buf)) + } + } + + impl Write for NativeSslStream + where T: Read + Write + { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|mut t| t.write(buf)) + } + fn flush(&mut self) -> IoResult<()> { + self.0.lock() + .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) + .and_then(|mut t| t.flush()) + } + } + + maybe_init_certs(); + + if maybe_proxy.is_none() { + // Connect with hyper + native_tls + client = Client::with_connector(HttpsConnector::new(NativeSslClient)); + } else { + let proxy_host_port = maybe_proxy.unwrap(); + client = Client::with_proxy_config(ProxyConfig(proxy_host_port.0, proxy_host_port.1, NativeSslClient)); + } + } else if url.scheme() == "http" { + if maybe_proxy.is_none() { + client = Client::new(); + } else { + let proxy_host_port = maybe_proxy.unwrap(); + client = Client::with_http_proxy(proxy_host_port.0, proxy_host_port.1); + } + } else { + return Err(format!("unsupported URL scheme: '{}'", url.scheme()).into()); + } + + let mut res = try!(client.get(url.clone()).send() + .chain_err(|| "failed to make network request")); + if res.status != hyper::Ok { + return Err(ErrorKind::HttpStatus(res.status.to_u16() as u32).into()); + } + + let buffer_size = 0x10000; + let mut buffer = vec![0u8; buffer_size]; + + let mut file = try!(fs::File::create(path).chain_err( + || "error creating file for download")); + + if let Some(len) = res.headers.get::().cloned() { + notify_handler(Notification::DownloadContentLengthReceived(len.0)); + } + + loop { + let bytes_read = try!(io::Read::read(&mut res, &mut buffer) + .chain_err(|| "error reading from socket")); + + if bytes_read != 0 { + if let Some(ref mut h) = hasher { + h.input(&buffer[0..bytes_read]); + } + try!(io::Write::write_all(&mut file, &mut buffer[0..bytes_read]) + .chain_err(|| "unable to write download to disk")); + notify_handler(Notification::DownloadDataReceived(bytes_read)); + } else { + try!(file.sync_data().chain_err(|| "unable to sync download to disk")); + notify_handler(Notification::DownloadFinished); + return Ok(()); + } + } + } + + // Tell our statically-linked OpenSSL where to find root certs + // cc https://github.com/alexcrichton/git2-rs/blob/master/libgit2-sys/lib.rs#L1267 + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + fn maybe_init_certs() { + use std::sync::{Once, ONCE_INIT}; + static INIT: Once = ONCE_INIT; + INIT.call_once(|| { + ::openssl_sys::probe::init_ssl_cert_env_vars(); + }); + } + + #[cfg(any(target_os = "windows", target_os = "macos"))] + fn maybe_init_certs() { } + + fn download_from_file_url(url: &Url, + path: &Path, + hasher: &mut Option<&mut Sha256>) + -> Result { + use raw::is_file; + + // The file scheme is mostly for use by tests to mock the dist server + if url.scheme() == "file" { + let src = try!(url.to_file_path() + .map_err(|_| Error::from(format!("bogus file url: '{}'", url)))); + if !is_file(&src) { + // Because some of multirust's logic depends on checking + // the error when a downloaded file doesn't exist, make + // the file case return the same error value as the + // network case. + return Err(ErrorKind::HttpStatus(hyper::status::StatusCode::NotFound.to_u16() as u32).into()); + } + try!(fs::copy(&src, path).chain_err(|| "failure copying file")); + + if let Some(ref mut h) = *hasher { + let ref mut f = try!(fs::File::open(path) + .chain_err(|| "unable to open downloaded file")); + + let ref mut buffer = vec![0u8; 0x10000]; + loop { + let bytes_read = try!(io::Read::read(f, buffer) + .chain_err(|| "unable to read downloaded file")); + if bytes_read == 0 { break } + h.input(&buffer[0..bytes_read]); + } + } + + Ok(true) + } else { + Ok(false) + } + } +} diff --git a/src/rustup-utils/src/lib.rs b/src/rustup-utils/src/lib.rs index e8655ad7d7..32142f6386 100644 --- a/src/rustup-utils/src/lib.rs +++ b/src/rustup-utils/src/lib.rs @@ -38,6 +38,7 @@ pub mod raw; pub mod tty; pub mod utils; pub mod toml_utils; +mod download; pub use errors::*; pub use notifications::{Notification}; diff --git a/src/rustup-utils/src/raw.rs b/src/rustup-utils/src/raw.rs index 63446243e8..d7709e0707 100644 --- a/src/rustup-utils/src/raw.rs +++ b/src/rustup-utils/src/raw.rs @@ -1,4 +1,3 @@ -use std::cell::RefCell; use std::char::from_u32; use std::error; use std::ffi::{OsStr, OsString}; @@ -11,15 +10,9 @@ use std::process::{Command, Stdio, ExitStatus}; use std::str; use std::thread; use std::time::Duration; -use curl::easy::Easy; -use sha2::{Sha256, Digest}; -use errors::*; -use url::Url; use rand::random; -use notifications::Notification; - pub fn ensure_dir_exists, F: FnOnce(&Path)>(path: P, callback: F) -> io::Result { @@ -155,356 +148,6 @@ pub fn tee_file(path: &Path, mut w: &mut W) -> io::Result<()> { } } -mod hyper { - use hyper; - use notifications::Notification; - use sha2::{Digest, Sha256}; - use std::fs; - use std::io; - use std::path::Path; - use std::time::Duration; - use url::Url; - use errors::*; - - fn proxy_from_env(url: &Url) -> Option<(String, u16)> { - use std::env::var_os; - - let mut maybe_https_proxy = var_os("https_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); - if maybe_https_proxy.is_none() { - maybe_https_proxy = var_os("HTTPS_PROXY").map(|ref v| v.to_str().unwrap_or("").to_string()); - } - let maybe_http_proxy = var_os("http_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); - let mut maybe_all_proxy = var_os("all_proxy").map(|ref v| v.to_str().unwrap_or("").to_string()); - if maybe_all_proxy.is_none() { - maybe_all_proxy = var_os("ALL_PROXY").map(|ref v| v.to_str().unwrap_or("").to_string()); - } - if let Some(url_value) = match url.scheme() { - "https" => maybe_https_proxy.or(maybe_http_proxy.or(maybe_all_proxy)), - "http" => maybe_http_proxy.or(maybe_all_proxy), - _ => maybe_all_proxy, - } { - if let Ok(proxy_url) = Url::parse(&url_value) { - if let Some(host) = proxy_url.host_str() { - let port = proxy_url.port().unwrap_or(8080); - return Some((host.to_string(), port)); - } - } - } - None - } - - pub fn download_file(url: &Url, - path: &Path, - mut hasher: Option<&mut Sha256>, - notify_handler: &Fn(Notification)) - -> Result<()> { - - // Short-circuit hyper for the "file:" URL scheme - if try!(download_from_file_url(url, path, &mut hasher)) { - return Ok(()); - } - - use hyper::client::{Client, ProxyConfig}; - use hyper::error::Result as HyperResult; - use hyper::header::ContentLength; - use hyper::net::{SslClient, NetworkStream, HttpsConnector}; - use native_tls; - use std::io::Result as IoResult; - use std::io::{Read, Write}; - use std::net::{SocketAddr, Shutdown}; - use std::sync::{Arc, Mutex}; - - // The Hyper HTTP client - let client; - - let maybe_proxy = proxy_from_env(url); - if url.scheme() == "https" { - - // All the following is adapter code to use native_tls with hyper. - - struct NativeSslClient; - - impl SslClient for NativeSslClient { - type Stream = NativeSslStream; - - fn wrap_client(&self, stream: T, host: &str) -> HyperResult { - use native_tls::ClientBuilder as TlsClientBuilder; - use hyper::error::Error as HyperError; - - let mut ssl_builder = try!(TlsClientBuilder::new() - .map_err(|e| HyperError::Ssl(Box::new(e)))); - let ssl_stream = try!(ssl_builder.handshake(host, stream) - .map_err(|e| HyperError::Ssl(Box::new(e)))); - - Ok(NativeSslStream(Arc::new(Mutex::new(ssl_stream)))) - } - } - - #[derive(Clone)] - struct NativeSslStream(Arc>>); - - #[derive(Debug)] - struct NativeSslPoisonError; - - impl ::std::error::Error for NativeSslPoisonError { - fn description(&self) -> &str { "mutex poisoned during TLS operation" } - } - - impl ::std::fmt::Display for NativeSslPoisonError { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::result::Result<(), ::std::fmt::Error> { - f.write_str(::std::error::Error::description(self)) - } - } - - impl NetworkStream for NativeSslStream - where T: NetworkStream - { - fn peer_addr(&mut self) -> IoResult { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|mut t| t.get_mut().peer_addr()) - } - fn set_read_timeout(&self, dur: Option) -> IoResult<()> { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|t| t.get_ref().set_read_timeout(dur)) - } - fn set_write_timeout(&self, dur: Option) -> IoResult<()> { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|t| t.get_ref().set_write_timeout(dur)) - } - fn close(&mut self, how: Shutdown) -> IoResult<()> { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|mut t| t.get_mut().close(how)) - } - } - - impl Read for NativeSslStream - where T: Read + Write - { - fn read(&mut self, buf: &mut [u8]) -> IoResult { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|mut t| t.read(buf)) - } - } - - impl Write for NativeSslStream - where T: Read + Write - { - fn write(&mut self, buf: &[u8]) -> IoResult { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|mut t| t.write(buf)) - } - fn flush(&mut self) -> IoResult<()> { - self.0.lock() - .map_err(|_| io::Error::new(io::ErrorKind::Other, NativeSslPoisonError)) - .and_then(|mut t| t.flush()) - } - } - - maybe_init_certs(); - - if maybe_proxy.is_none() { - // Connect with hyper + native_tls - client = Client::with_connector(HttpsConnector::new(NativeSslClient)); - } else { - let proxy_host_port = maybe_proxy.unwrap(); - client = Client::with_proxy_config(ProxyConfig(proxy_host_port.0, proxy_host_port.1, NativeSslClient)); - } - } else if url.scheme() == "http" { - if maybe_proxy.is_none() { - client = Client::new(); - } else { - let proxy_host_port = maybe_proxy.unwrap(); - client = Client::with_http_proxy(proxy_host_port.0, proxy_host_port.1); - } - } else { - return Err(format!("unsupported URL scheme: '{}'", url.scheme()).into()); - } - - let mut res = try!(client.get(url.clone()).send() - .chain_err(|| "failed to make network request")); - if res.status != hyper::Ok { - return Err(ErrorKind::HttpStatus(res.status.to_u16() as u32).into()); - } - - let buffer_size = 0x10000; - let mut buffer = vec![0u8; buffer_size]; - - let mut file = try!(fs::File::create(path).chain_err( - || "error creating file for download")); - - if let Some(len) = res.headers.get::().cloned() { - notify_handler(Notification::DownloadContentLengthReceived(len.0)); - } - - loop { - let bytes_read = try!(io::Read::read(&mut res, &mut buffer) - .chain_err(|| "error reading from socket")); - - if bytes_read != 0 { - if let Some(ref mut h) = hasher { - h.input(&buffer[0..bytes_read]); - } - try!(io::Write::write_all(&mut file, &mut buffer[0..bytes_read]) - .chain_err(|| "unable to write download to disk")); - notify_handler(Notification::DownloadDataReceived(bytes_read)); - } else { - try!(file.sync_data().chain_err(|| "unable to sync download to disk")); - notify_handler(Notification::DownloadFinished); - return Ok(()); - } - } - } - - // Tell our statically-linked OpenSSL where to find root certs - // cc https://github.com/alexcrichton/git2-rs/blob/master/libgit2-sys/lib.rs#L1267 - #[cfg(not(any(target_os = "windows", target_os = "macos")))] - fn maybe_init_certs() { - use std::sync::{Once, ONCE_INIT}; - static INIT: Once = ONCE_INIT; - INIT.call_once(|| { - ::openssl_sys::probe::init_ssl_cert_env_vars(); - }); - } - - #[cfg(any(target_os = "windows", target_os = "macos"))] - fn maybe_init_certs() { } - - fn download_from_file_url(url: &Url, - path: &Path, - hasher: &mut Option<&mut Sha256>) - -> Result { - use super::is_file; - - // The file scheme is mostly for use by tests to mock the dist server - if url.scheme() == "file" { - let src = try!(url.to_file_path() - .map_err(|_| Error::from(format!("bogus file url: '{}'", url)))); - if !is_file(&src) { - // Because some of multirust's logic depends on checking - // the error when a downloaded file doesn't exist, make - // the file case return the same error value as the - // network case. - return Err(ErrorKind::HttpStatus(hyper::status::StatusCode::NotFound.to_u16() as u32).into()); - } - try!(fs::copy(&src, path).chain_err(|| "failure copying file")); - - if let Some(ref mut h) = *hasher { - let ref mut f = try!(fs::File::open(path) - .chain_err(|| "unable to open downloaded file")); - - let ref mut buffer = vec![0u8; 0x10000]; - loop { - let bytes_read = try!(io::Read::read(f, buffer) - .chain_err(|| "unable to read downloaded file")); - if bytes_read == 0 { break } - h.input(&buffer[0..bytes_read]); - } - } - - Ok(true) - } else { - Ok(false) - } - } -} - -pub fn download_file(url: &Url, - path: &Path, - mut hasher: Option<&mut Sha256>, - notify_handler: &Fn(Notification)) - -> Result<()> { - use notifications::Notification; - use std::io::Write; - - if ::std::env::var_os("RUSTUP_USE_HYPER").is_some() { - return self::hyper::download_file(url, path, hasher, notify_handler); - } - - let mut file = try!(fs::File::create(&path).chain_err( - || "error creating file for download")); - - // Fetch either a cached libcurl handle (which will preserve open - // connections) or create a new one if it isn't listed. - // - // Once we've acquired it, reset the lifetime from 'static to our local - // scope. - thread_local!(static EASY: RefCell = RefCell::new(Easy::new())); - EASY.with(|handle| { - let mut handle = handle.borrow_mut(); - - try!(handle.url(&url.to_string()).chain_err(|| "failed to set url")); - try!(handle.follow_location(true).chain_err(|| "failed to set follow redirects")); - - // Take at most 30s to connect - try!(handle.connect_timeout(Duration::new(30, 0)).chain_err(|| "failed to set connect timeout")); - - // Fail if less than 10 bytes are transferred every 30 seconds - try!(handle.low_speed_limit(10).chain_err(|| "failed to set low speed limit")); - try!(handle.low_speed_time(Duration::new(30, 0)).chain_err(|| "failed to set low speed time")); - - { - let fserr = RefCell::new(None); - let mut transfer = handle.transfer(); - - // Data callback for libcurl which is called with data that's - // downloaded. We just feed it into our hasher and also write it out - // to disk. - try!(transfer.write_function(|data| { - if let Some(ref mut h) = hasher { - h.input(data); - } - notify_handler(Notification::DownloadDataReceived(data.len())); - match file.write_all(data) { - Ok(()) => Ok(data.len()), - Err(e) => { - *fserr.borrow_mut() = Some(e); - Ok(0) - } - } - }).chain_err(|| "failed to set write")); - - // Listen for headers and parse out a `Content-Length` if it comes - // so we know how much we're downloading. - try!(transfer.header_function(|header| { - if let Ok(data) = str::from_utf8(header) { - let prefix = "Content-Length: "; - if data.starts_with(prefix) { - if let Ok(s) = data[prefix.len()..].trim().parse() { - let msg = Notification::DownloadContentLengthReceived(s); - notify_handler(msg); - } - } - } - true - }).chain_err(|| "failed to set header")); - - // If an error happens check to see if we had a filesystem error up - // in `fserr`, but we always want to punt it up. - try!(transfer.perform().or_else(|e| { - match fserr.borrow_mut().take() { - Some(fs) => Err(fs).chain_err(|| ErrorKind::HttpError(e)), - None => Err(ErrorKind::HttpError(e).into()) - } - })); - } - - // If we didn't get a 200 or 0 ("OK" for files) then return an error - let code = try!(handle.response_code().chain_err(|| "failed to get response code")); - if code != 200 && code != 0 { - return Err(ErrorKind::HttpStatus(code).into()); - } - - notify_handler(Notification::DownloadFinished); - Ok(()) - }) -} - pub fn symlink_dir(src: &Path, dest: &Path) -> io::Result<()> { #[cfg(windows)] fn symlink_dir_inner(src: &Path, dest: &Path) -> io::Result<()> { diff --git a/src/rustup-utils/src/utils.rs b/src/rustup-utils/src/utils.rs index b3cb7d73ac..d21b650c45 100644 --- a/src/rustup-utils/src/utils.rs +++ b/src/rustup-utils/src/utils.rs @@ -7,6 +7,7 @@ use std::ffi::OsString; use std::env; use sha2::Sha256; use notifications::{Notification}; +use download; use raw; #[cfg(windows)] use winapi::DWORD; @@ -145,7 +146,7 @@ pub fn download_file(url: &Url, notify_handler: &Fn(Notification)) -> Result<()> { notify_handler(Notification::DownloadingFile(url, path)); - match raw::download_file(url, path, hasher, notify_handler) { + match download::download_file(url, path, hasher, notify_handler) { Ok(_) => Ok(()), Err(e) => { let is404 = match e.kind() { From ac030b0f8af485a90ad7b88aaaebcfd2fd3373b3 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 19:16:09 -0700 Subject: [PATCH 2/7] Add cursory docs to rust_utils::download --- src/rustup-utils/src/download.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index 702fec447f..f603c4878d 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -1,4 +1,4 @@ - +//! Easy file downloading use errors::*; use notifications::Notification; @@ -19,6 +19,9 @@ pub fn download_file(url: &Url, } } + +/// Download via libcurl; encrypt with the native (or OpenSSl) TLS +/// stack via libcurl mod curl { use curl::easy::Easy; use errors::*; @@ -119,6 +122,8 @@ mod curl { } } +/// Download via hyper; encrypt with the native (or OpenSSl) TLS +/// stack via native-tls mod hyper { use hyper; use notifications::Notification; From 742b3b22d98524b628d1c36239654c8b85098d80 Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 19:44:01 -0700 Subject: [PATCH 3/7] Add a notification indicating which HTTP backend is in use --- src/rustup-utils/src/download.rs | 8 ++++++-- src/rustup-utils/src/notifications.rs | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index f603c4878d..73fa7c3e58 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -13,10 +13,14 @@ pub fn download_file(url: &Url, notify_handler: &Fn(Notification)) -> Result<()> { if env::var_os("RUSTUP_USE_HYPER").is_some() { - self::hyper::download_file(url, path, hasher, notify_handler) + notify_handler(Notification::UsingHyper); + try!(self::hyper::download_file(url, path, hasher, notify_handler)); } else { - self::curl::download_file(url, path, hasher, notify_handler) + notify_handler(Notification::UsingCurl); + try!(self::curl::download_file(url, path, hasher, notify_handler)); } + + Ok(()) } diff --git a/src/rustup-utils/src/notifications.rs b/src/rustup-utils/src/notifications.rs index 42cd8e97bb..8a42b5b583 100644 --- a/src/rustup-utils/src/notifications.rs +++ b/src/rustup-utils/src/notifications.rs @@ -19,6 +19,8 @@ pub enum Notification<'a> { /// Download has finished. DownloadFinished, NoCanonicalPath(&'a Path), + UsingCurl, + UsingHyper, } impl<'a> Notification<'a> { @@ -31,7 +33,8 @@ impl<'a> Notification<'a> { DownloadingFile(_, _) | DownloadContentLengthReceived(_) | DownloadDataReceived(_) | - DownloadFinished => NotificationLevel::Verbose, + DownloadFinished | + UsingCurl | UsingHyper => NotificationLevel::Verbose, NoCanonicalPath(_) => NotificationLevel::Warn, } } @@ -54,6 +57,8 @@ impl<'a> Display for Notification<'a> { DownloadDataReceived(len) => write!(f, "received some data of size {}", len), DownloadFinished => write!(f, "download finished"), NoCanonicalPath(path) => write!(f, "could not canonicalize path: '{}'", path.display()), + UsingCurl => write!(f, "downloading with curl"), + UsingHyper => write!(f, "downloading with hyper"), } } } From a80e8035b2e2bb42675bb354bfa2309c95fc4f8f Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 19:45:42 -0700 Subject: [PATCH 4/7] Pull DownloadFinished notification into generic download_file fn --- src/rustup-utils/src/download.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index 73fa7c3e58..51ab861526 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -20,6 +20,8 @@ pub fn download_file(url: &Url, try!(self::curl::download_file(url, path, hasher, notify_handler)); } + notify_handler(Notification::DownloadFinished); + Ok(()) } @@ -120,7 +122,6 @@ mod curl { return Err(ErrorKind::HttpStatus(code).into()); } - notify_handler(Notification::DownloadFinished); Ok(()) }) } @@ -328,7 +329,6 @@ mod hyper { notify_handler(Notification::DownloadDataReceived(bytes_read)); } else { try!(file.sync_data().chain_err(|| "unable to sync download to disk")); - notify_handler(Notification::DownloadFinished); return Ok(()); } } From 75af7cacbb5d8bb5e0737c465d946cbdc3f6de7c Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 20:00:51 -0700 Subject: [PATCH 5/7] Add the data to the DownloadDataReceived notification --- src/rustup-cli/download_tracker.rs | 4 ++-- src/rustup-utils/src/download.rs | 4 ++-- src/rustup-utils/src/notifications.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rustup-cli/download_tracker.rs b/src/rustup-cli/download_tracker.rs index 0c1cc5264a..12fc4c07ab 100644 --- a/src/rustup-cli/download_tracker.rs +++ b/src/rustup-cli/download_tracker.rs @@ -57,9 +57,9 @@ impl DownloadTracker { true } - &Notification::Install(In::Utils(Un::DownloadDataReceived(len))) => { + &Notification::Install(In::Utils(Un::DownloadDataReceived(data))) => { if tty::stdout_isatty() && self.term.is_some() { - self.data_received(len); + self.data_received(data.len()); } true } diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index 51ab861526..2e22855455 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -81,7 +81,7 @@ mod curl { if let Some(ref mut h) = hasher { h.input(data); } - notify_handler(Notification::DownloadDataReceived(data.len())); + notify_handler(Notification::DownloadDataReceived(data)); match file.write_all(data) { Ok(()) => Ok(data.len()), Err(e) => { @@ -326,7 +326,7 @@ mod hyper { } try!(io::Write::write_all(&mut file, &mut buffer[0..bytes_read]) .chain_err(|| "unable to write download to disk")); - notify_handler(Notification::DownloadDataReceived(bytes_read)); + notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read])); } else { try!(file.sync_data().chain_err(|| "unable to sync download to disk")); return Ok(()); diff --git a/src/rustup-utils/src/notifications.rs b/src/rustup-utils/src/notifications.rs index 8a42b5b583..08f4a5016d 100644 --- a/src/rustup-utils/src/notifications.rs +++ b/src/rustup-utils/src/notifications.rs @@ -15,7 +15,7 @@ pub enum Notification<'a> { /// Received the Content-Length of the to-be downloaded data. DownloadContentLengthReceived(u64), /// Received some data. - DownloadDataReceived(usize), + DownloadDataReceived(&'a [u8]), /// Download has finished. DownloadFinished, NoCanonicalPath(&'a Path), @@ -54,7 +54,7 @@ impl<'a> Display for Notification<'a> { } DownloadingFile(url, _) => write!(f, "downloading file from: '{}'", url), DownloadContentLengthReceived(len) => write!(f, "download size is: '{}'", len), - DownloadDataReceived(len) => write!(f, "received some data of size {}", len), + DownloadDataReceived(data) => write!(f, "received some data of size {}", data.len()), DownloadFinished => write!(f, "download finished"), NoCanonicalPath(path) => write!(f, "could not canonicalize path: '{}'", path.display()), UsingCurl => write!(f, "downloading with curl"), From 30faf48156871b4d24e491dbb8a54b9353b9a54b Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 20:04:42 -0700 Subject: [PATCH 6/7] Pull hashing out of the HTTP impl --- src/rustup-utils/src/download.rs | 47 ++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index 2e22855455..7a666e0f11 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -2,7 +2,8 @@ use errors::*; use notifications::Notification; -use sha2::Sha256; +use sha2::{Sha256, Digest}; +use std::cell::RefCell; use std::env; use std::path::Path; use url::Url; @@ -12,12 +13,34 @@ pub fn download_file(url: &Url, hasher: Option<&mut Sha256>, notify_handler: &Fn(Notification)) -> Result<()> { + + let hasher = RefCell::new(hasher); if env::var_os("RUSTUP_USE_HYPER").is_some() { notify_handler(Notification::UsingHyper); - try!(self::hyper::download_file(url, path, hasher, notify_handler)); + try!(self::hyper::download_file(url, path, &|notice| { + match notice { + Notification::DownloadDataReceived(data) => { + if let Some(ref mut h) = *hasher.borrow_mut() { + h.input(data); + } + } + _ => () + } + notify_handler(notice); + })); } else { notify_handler(Notification::UsingCurl); - try!(self::curl::download_file(url, path, hasher, notify_handler)); + try!(self::curl::download_file(url, path, &|notice| { + match notice { + Notification::DownloadDataReceived(data) => { + if let Some(ref mut h) = *hasher.borrow_mut() { + h.input(data); + } + } + _ => () + } + notify_handler(notice); + })); } notify_handler(Notification::DownloadFinished); @@ -32,7 +55,6 @@ mod curl { use curl::easy::Easy; use errors::*; use notifications::Notification; - use sha2::{Sha256, Digest}; use std::cell::RefCell; use std::fs; use std::path::Path; @@ -42,7 +64,6 @@ mod curl { pub fn download_file(url: &Url, path: &Path, - mut hasher: Option<&mut Sha256>, notify_handler: &Fn(Notification)) -> Result<()> { use notifications::Notification; @@ -78,9 +99,6 @@ mod curl { // downloaded. We just feed it into our hasher and also write it out // to disk. try!(transfer.write_function(|data| { - if let Some(ref mut h) = hasher { - h.input(data); - } notify_handler(Notification::DownloadDataReceived(data)); match file.write_all(data) { Ok(()) => Ok(data.len()), @@ -132,7 +150,6 @@ mod curl { mod hyper { use hyper; use notifications::Notification; - use sha2::{Digest, Sha256}; use std::fs; use std::io; use std::path::Path; @@ -169,12 +186,11 @@ mod hyper { pub fn download_file(url: &Url, path: &Path, - mut hasher: Option<&mut Sha256>, notify_handler: &Fn(Notification)) -> Result<()> { // Short-circuit hyper for the "file:" URL scheme - if try!(download_from_file_url(url, path, &mut hasher)) { + if try!(download_from_file_url(url, path, notify_handler)) { return Ok(()); } @@ -321,9 +337,6 @@ mod hyper { .chain_err(|| "error reading from socket")); if bytes_read != 0 { - if let Some(ref mut h) = hasher { - h.input(&buffer[0..bytes_read]); - } try!(io::Write::write_all(&mut file, &mut buffer[0..bytes_read]) .chain_err(|| "unable to write download to disk")); notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read])); @@ -350,7 +363,7 @@ mod hyper { fn download_from_file_url(url: &Url, path: &Path, - hasher: &mut Option<&mut Sha256>) + notify_handler: &Fn(Notification)) -> Result { use raw::is_file; @@ -367,7 +380,7 @@ mod hyper { } try!(fs::copy(&src, path).chain_err(|| "failure copying file")); - if let Some(ref mut h) = *hasher { + { let ref mut f = try!(fs::File::open(path) .chain_err(|| "unable to open downloaded file")); @@ -376,7 +389,7 @@ mod hyper { let bytes_read = try!(io::Read::read(f, buffer) .chain_err(|| "unable to read downloaded file")); if bytes_read == 0 { break } - h.input(&buffer[0..bytes_read]); + notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read])); } } From 759610b780bc32cfb33aedd0acab3dd2cddf086b Mon Sep 17 00:00:00 2001 From: Brian Anderson Date: Sat, 9 Jul 2016 21:02:38 -0700 Subject: [PATCH 7/7] Stop writing to disk in the download_file impls --- src/rustup-utils/src/download.rs | 113 ++++++++++++++----------------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/src/rustup-utils/src/download.rs b/src/rustup-utils/src/download.rs index 7a666e0f11..1c96153a8a 100644 --- a/src/rustup-utils/src/download.rs +++ b/src/rustup-utils/src/download.rs @@ -4,6 +4,8 @@ use errors::*; use notifications::Notification; use sha2::{Sha256, Digest}; use std::cell::RefCell; +use std::fs; +use std::io; use std::env; use std::path::Path; use url::Url; @@ -14,35 +16,40 @@ pub fn download_file(url: &Url, notify_handler: &Fn(Notification)) -> Result<()> { + let file = RefCell::new(try!(fs::File::create(&path).chain_err( + || "error creating file for download"))); let hasher = RefCell::new(hasher); - if env::var_os("RUSTUP_USE_HYPER").is_some() { - notify_handler(Notification::UsingHyper); - try!(self::hyper::download_file(url, path, &|notice| { - match notice { - Notification::DownloadDataReceived(data) => { - if let Some(ref mut h) = *hasher.borrow_mut() { - h.input(data); - } + + // This callback will write the download to disk and optionally + // hash the contents, then forward the notification up the stack + let handler: &Fn(Notification) -> Result<()> = &|notice| { + match notice { + Notification::DownloadDataReceived(data) => { + try!(io::Write::write_all(&mut *file.borrow_mut(), data) + .chain_err(|| "unable to write download to disk")); + if let Some(ref mut h) = *hasher.borrow_mut() { + h.input(data); } - _ => () } - notify_handler(notice); - })); + _ => () + } + + notify_handler(notice); + + Ok(()) + }; + + // Download the file + if env::var_os("RUSTUP_USE_HYPER").is_some() { + notify_handler(Notification::UsingHyper); + try!(self::hyper::download_file(url, handler)); } else { notify_handler(Notification::UsingCurl); - try!(self::curl::download_file(url, path, &|notice| { - match notice { - Notification::DownloadDataReceived(data) => { - if let Some(ref mut h) = *hasher.borrow_mut() { - h.input(data); - } - } - _ => () - } - notify_handler(notice); - })); + try!(self::curl::download_file(url, handler)); } + try!(file.borrow_mut().sync_data().chain_err(|| "unable to sync download to disk")); + notify_handler(Notification::DownloadFinished); Ok(()) @@ -56,22 +63,13 @@ mod curl { use errors::*; use notifications::Notification; use std::cell::RefCell; - use std::fs; - use std::path::Path; use std::str; use std::time::Duration; use url::Url; pub fn download_file(url: &Url, - path: &Path, - notify_handler: &Fn(Notification)) + notify_handler: &Fn(Notification) -> Result<()> ) -> Result<()> { - use notifications::Notification; - use std::io::Write; - - let mut file = try!(fs::File::create(&path).chain_err( - || "error creating file for download")); - // Fetch either a cached libcurl handle (which will preserve open // connections) or create a new one if it isn't listed. // @@ -99,8 +97,7 @@ mod curl { // downloaded. We just feed it into our hasher and also write it out // to disk. try!(transfer.write_function(|data| { - notify_handler(Notification::DownloadDataReceived(data)); - match file.write_all(data) { + match notify_handler(Notification::DownloadDataReceived(data)) { Ok(()) => Ok(data.len()), Err(e) => { *fserr.borrow_mut() = Some(e); @@ -117,7 +114,13 @@ mod curl { if data.starts_with(prefix) { if let Ok(s) = data[prefix.len()..].trim().parse() { let msg = Notification::DownloadContentLengthReceived(s); - notify_handler(msg); + match notify_handler(msg) { + Ok(()) => (), + Err(_e) => { + // FIXME: discarding error _e + return false; + } + } } } } @@ -152,7 +155,6 @@ mod hyper { use notifications::Notification; use std::fs; use std::io; - use std::path::Path; use std::time::Duration; use url::Url; use errors::*; @@ -185,12 +187,11 @@ mod hyper { } pub fn download_file(url: &Url, - path: &Path, - notify_handler: &Fn(Notification)) + notify_handler: &Fn(Notification) -> Result<()>) -> Result<()> { // Short-circuit hyper for the "file:" URL scheme - if try!(download_from_file_url(url, path, notify_handler)) { + if try!(download_from_file_url(url, notify_handler)) { return Ok(()); } @@ -325,11 +326,8 @@ mod hyper { let buffer_size = 0x10000; let mut buffer = vec![0u8; buffer_size]; - let mut file = try!(fs::File::create(path).chain_err( - || "error creating file for download")); - if let Some(len) = res.headers.get::().cloned() { - notify_handler(Notification::DownloadContentLengthReceived(len.0)); + try!(notify_handler(Notification::DownloadContentLengthReceived(len.0))); } loop { @@ -337,11 +335,8 @@ mod hyper { .chain_err(|| "error reading from socket")); if bytes_read != 0 { - try!(io::Write::write_all(&mut file, &mut buffer[0..bytes_read]) - .chain_err(|| "unable to write download to disk")); - notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read])); + try!(notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read]))); } else { - try!(file.sync_data().chain_err(|| "unable to sync download to disk")); return Ok(()); } } @@ -362,8 +357,7 @@ mod hyper { fn maybe_init_certs() { } fn download_from_file_url(url: &Url, - path: &Path, - notify_handler: &Fn(Notification)) + notify_handler: &Fn(Notification) -> Result<()>) -> Result { use raw::is_file; @@ -378,19 +372,16 @@ mod hyper { // network case. return Err(ErrorKind::HttpStatus(hyper::status::StatusCode::NotFound.to_u16() as u32).into()); } - try!(fs::copy(&src, path).chain_err(|| "failure copying file")); - { - let ref mut f = try!(fs::File::open(path) - .chain_err(|| "unable to open downloaded file")); - - let ref mut buffer = vec![0u8; 0x10000]; - loop { - let bytes_read = try!(io::Read::read(f, buffer) - .chain_err(|| "unable to read downloaded file")); - if bytes_read == 0 { break } - notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read])); - } + let ref mut f = try!(fs::File::open(src) + .chain_err(|| "unable to open downloaded file")); + + let ref mut buffer = vec![0u8; 0x10000]; + loop { + let bytes_read = try!(io::Read::read(f, buffer) + .chain_err(|| "unable to read downloaded file")); + if bytes_read == 0 { break } + try!(notify_handler(Notification::DownloadDataReceived(&buffer[0..bytes_read]))); } Ok(true)