feat: Add decompress flag to sourcemaps and files upload (#1277)

* feat: Add decompress flag to sourcemaps and files upload

Author: kamilogorek

src/api.rs

@@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet};
 use std::ffi::OsStr;
 use std::fs::{create_dir_all, File};
 use std::io::{self, Read, Write};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::rc::Rc;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -20,10 +20,8 @@ use backoff::backoff::Backoff;
 use brotli2::write::BrotliEncoder;
 use chrono::{DateTime, Duration, FixedOffset, Utc};
 use clap::ArgMatches;
-use console::style;
 use flate2::write::GzEncoder;
 use if_chain::if_chain;
-use indicatif::ProgressStyle;
 use lazy_static::lazy_static;
 use log::{debug, info, warn};
 use parking_lot::{Mutex, RwLock};
@@ -41,6 +39,7 @@ use uuid::Uuid;
 use crate::config::{Auth, Config};
 use crate::constants::{ARCH, EXT, PLATFORM, RELEASE_REGISTRY_LATEST_URL, VERSION};
 use crate::utils::android::AndroidManifest;
+use crate::utils::file_upload::UploadContext;
 use crate::utils::http::{self, is_absolute_url, parse_link_header};
 use crate::utils::progress::ProgressBar;
 use crate::utils::retry::{get_default_backoff, DurationAsMilliseconds};
@@ -51,13 +50,6 @@ use crate::utils::xcode::InfoPlist;
 const QUERY_ENCODE_SET: AsciiSet = CONTROLS.add(b' ').add(b'"').add(b'#').add(b'<').add(b'>');
 const DEFAULT_ENCODE_SET: AsciiSet = QUERY_ENCODE_SET.add(b'`').add(b'?').add(b'{').add(b'}');
 
-/// Represents file contents temporarily
-#[derive(Clone, Debug)]
-pub enum FileContents<'a> {
-    FromPath(&'a Path),
-    FromBytes(&'a [u8]),
-}
-
 /// Wrapper that escapes arguments for URL path segments.
 pub struct PathArg<A: fmt::Display>(A);
 
@@ -669,48 +661,39 @@ impl Api {
 
     /// Uploads a new release file.  The file is loaded directly from the file
     /// system and uploaded as `name`.
-    // TODO: Simplify this function interface
-    #[allow(clippy::too_many_arguments)]
     pub fn upload_release_file(
         &self,
-        org: &str,
-        project: Option<&str>,
-        version: &str,
-        contents: &FileContents,
+        context: &UploadContext,
+        contents: &[u8],
         name: &str,
-        dist: Option<&str>,
         headers: Option<&[(String, String)]>,
         progress_bar_mode: ProgressBarMode,
     ) -> ApiResult<Option<Artifact>> {
-        let path = if let Some(project) = project {
+        let path = if let Some(project) = context.project {
             format!(
                 "/projects/{}/{}/releases/{}/files/",
-                PathArg(org),
+                PathArg(context.org),
                 PathArg(project),
-                PathArg(version)
+                PathArg(context.release)
             )
         } else {
             format!(
                 "/organizations/{}/releases/{}/files/",
-                PathArg(org),
-                PathArg(version)
+                PathArg(context.org),
+                PathArg(context.release)
             )
         };
         let mut form = curl::easy::Form::new();
-        match contents {
-            FileContents::FromPath(path) => {
-                form.part("file").file(path).add()?;
-            }
-            FileContents::FromBytes(bytes) => {
-                let filename = Path::new(name)
-                    .file_name()
-                    .and_then(OsStr::to_str)
-                    .unwrap_or("unknown.bin");
-                form.part("file").buffer(filename, bytes.to_vec()).add()?;
-            }
-        }
+
+        let filename = Path::new(name)
+            .file_name()
+            .and_then(OsStr::to_str)
+            .unwrap_or("unknown.bin");
+        form.part("file")
+            .buffer(filename, contents.to_vec())
+            .add()?;
         form.part("name").contents(name.as_bytes()).add()?;
-        if let Some(dist) = dist {
+        if let Some(dist) = context.dist {
             form.part("dist").contents(dist.as_bytes()).add()?;
         }
 
@@ -742,90 +725,6 @@ impl Api {
         }
     }
 
-    pub fn upload_release_files(
-        &self,
-        org: &str,
-        project: Option<&str>,
-        version: &str,
-        sources: HashSet<(String, PathBuf)>,
-        dist: Option<&str>,
-        headers: Option<&[(String, String)]>,
-    ) -> Result<(), ApiError> {
-        if sources.is_empty() {
-            return Ok(());
-        }
-
-        let mut successful_req = vec![];
-        let mut failed_req = vec![];
-
-        println!(
-            "{} Uploading {} files for release {}",
-            style(">").dim(),
-            style(sources.len()).cyan(),
-            style(version).cyan()
-        );
-
-        let pb = ProgressBar::new(sources.len());
-        pb.set_style(ProgressStyle::default_bar().template(&format!(
-            "{} {{msg}}\n{{wide_bar}} {{pos}}/{{len}}",
-            style(">").cyan()
-        )));
-
-        for (url, path) in sources {
-            pb.set_message(path.to_str().unwrap());
-
-            let upload_result = self.upload_release_file(
-                org,
-                project,
-                version,
-                &FileContents::FromPath(&path),
-                &url.to_owned(),
-                dist,
-                headers,
-                ProgressBarMode::Disabled,
-            );
-
-            match upload_result {
-                Ok(Some(artifact)) => {
-                    successful_req.push((path, artifact));
-                }
-                Ok(None) => {
-                    failed_req.push((path, url, String::from("File already present")));
-                }
-                Err(err) => {
-                    failed_req.push((path, url, err.to_string()));
-                }
-            };
-
-            pb.inc(1);
-        }
-
-        pb.finish_and_clear();
-
-        for (path, url, reason) in failed_req.iter() {
-            println!(
-                "    Failed to upload: {} as {} ({})",
-                &path.display(),
-                &url,
-                reason
-            );
-        }
-
-        for (path, artifact) in successful_req.iter() {
-            println!(
-                "    Successfully uploaded: {} as {} ({}) ({} bytes)",
-                &path.display(),
-                artifact.name,
-                artifact.sha1,
-                artifact.size
-            );
-        }
-
-        println!("{} Done uploading.", style(">").dim());
-
-        Ok(())
-    }
-
     /// Creates a new release.
     pub fn new_release(&self, org: &str, release: &NewRelease) -> ApiResult<ReleaseInfo> {
         // for single project releases use the legacy endpoint that is project bound.

src/commands/files/upload.rs

@@ -1,15 +1,18 @@
 use std::ffi::OsStr;
+use std::fs;
+use std::io::Read;
 use std::path::Path;
 
 use anyhow::{bail, format_err, Result};
 use clap::{Arg, ArgMatches, Command};
+use log::warn;
 use symbolic::debuginfo::sourcebundle::SourceFileType;
 
-use crate::api::{Api, FileContents, ProgressBarMode};
+use crate::api::{Api, ProgressBarMode};
 use crate::config::Config;
 use crate::utils::file_search::ReleaseFileSearch;
 use crate::utils::file_upload::{ReleaseFile, ReleaseFileUpload, UploadContext};
-use crate::utils::fs::path_as_url;
+use crate::utils::fs::{decompress_gzip_content, is_gzip_compressed, path_as_url};
 
 pub fn make_command(command: Command) -> Command {
     command
@@ -34,6 +37,11 @@ pub fn make_command(command: Command) -> Command {
                 .value_name("DISTRIBUTION")
                 .help("Optional distribution identifier for this file."),
         )
+        .arg(
+            Arg::new("decompress")
+                .long("decompress")
+                .help("Enable files gzip decompression prior to upload."),
+        )
         .arg(
             Arg::new("wait")
                 .long("wait")
@@ -110,8 +118,16 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
             headers.push((key.trim().to_string(), value.trim().to_string()));
         }
     };
-    let path = Path::new(matches.value_of("path").unwrap());
 
+    let context = &UploadContext {
+        org: &org,
+        project: project.as_deref(),
+        release: &release,
+        dist,
+        wait: matches.is_present("wait"),
+    };
+
+    let path = Path::new(matches.value_of("path").unwrap());
     // Batch files upload
     if path.is_dir() {
         let ignore_file = matches.value_of("ignore_file").unwrap_or("");
@@ -128,6 +144,7 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
             .ignore_file(ignore_file)
             .ignores(ignores)
             .extensions(extensions)
+            .decompress(matches.is_present("decompress"))
             .collect_files()?;
 
         let url_suffix = matches.value_of("url_suffix").unwrap_or("");
@@ -157,15 +174,7 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
             })
             .collect();
 
-        let ctx = &UploadContext {
-            org: &org,
-            project: project.as_deref(),
-            release: &release,
-            dist,
-            wait: matches.is_present("wait"),
-        };
-
-        ReleaseFileUpload::new(ctx).files(&files).upload()
+        ReleaseFileUpload::new(context).files(&files).upload()
     }
     // Single file upload
     else {
@@ -177,13 +186,21 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
                 .ok_or_else(|| format_err!("No filename provided."))?,
         };
 
+        let mut f = fs::File::open(path)?;
+        let mut contents = Vec::new();
+        f.read_to_end(&mut contents)?;
+
+        if matches.is_present("decompress") && is_gzip_compressed(&contents) {
+            contents = decompress_gzip_content(&contents).unwrap_or_else(|_| {
+                warn!("Could not decompress: {}", name);
+                contents
+            });
+        }
+
         if let Some(artifact) = api.upload_release_file(
-            &org,
-            project.as_deref(),
-            &release,
-            &FileContents::FromPath(path),
+            context,
+            &contents,
             name,
-            dist,
             Some(&headers[..]),
             ProgressBarMode::Request,
         )? {

src/commands/sourcemaps/upload.rs

@@ -51,6 +51,11 @@ pub fn make_command(command: Command) -> Command {
                 .long("validate")
                 .help("Enable basic sourcemap validation."),
         )
+        .arg(
+            Arg::new("decompress")
+                .long("decompress")
+                .help("Enable files gzip decompression prior to upload."),
+        )
         .arg(
             Arg::new("wait")
                 .long("wait")
@@ -258,6 +263,7 @@ fn process_sources_from_paths(
         };
 
         let mut search = ReleaseFileSearch::new(path.to_path_buf());
+        search.decompress(matches.is_present("decompress"));
 
         if check_ignore {
             search

src/utils/file_search.rs

@@ -8,15 +8,18 @@ use console::style;
 use ignore::overrides::OverrideBuilder;
 use ignore::types::TypesBuilder;
 use ignore::WalkBuilder;
-use log::info;
+use log::{info, warn};
 
 use crate::utils::progress::{ProgressBar, ProgressStyle};
 
+use super::fs::{decompress_gzip_content, is_gzip_compressed};
+
 pub struct ReleaseFileSearch {
     path: PathBuf,
     extensions: BTreeSet<String>,
     ignores: BTreeSet<String>,
     ignore_file: Option<String>,
+    decompress: bool,
 }
 
 #[derive(Eq, PartialEq, Hash)]
@@ -33,9 +36,15 @@ impl ReleaseFileSearch {
             extensions: BTreeSet::new(),
             ignore_file: None,
             ignores: BTreeSet::new(),
+            decompress: false,
         }
     }
 
+    pub fn decompress(&mut self, decompress: bool) -> &mut Self {
+        self.decompress = decompress;
+        self
+    }
+
     pub fn extension<E>(&mut self, extension: E) -> &mut Self
     where
         E: Into<String>,
@@ -86,9 +95,12 @@ impl ReleaseFileSearch {
     }
 
     pub fn collect_file(path: PathBuf) -> Result<ReleaseFileMatch> {
+        // NOTE: `collect_file` currently do not handle gzip decompression,
+        // as its mostly used for 3rd tools like xcode, appcenter or gradle.
         let mut f = fs::File::open(path.clone())?;
         let mut contents = Vec::new();
         f.read_to_end(&mut contents)?;
+
         Ok(ReleaseFileMatch {
             base_path: path.clone(),
             path,
@@ -155,6 +167,13 @@ impl ReleaseFileSearch {
             let mut contents = Vec::new();
             f.read_to_end(&mut contents)?;
 
+            if self.decompress && is_gzip_compressed(&contents) {
+                contents = decompress_gzip_content(&contents).unwrap_or_else(|_| {
+                    warn!("Could not decompress: {}", file.path().display());
+                    contents
+                });
+            }
+
             let file_match = ReleaseFileMatch {
                 base_path: self.path.clone(),
                 path: file.path().to_path_buf(),