wip

Author: Arpita-Jaiswal

src/commands/serve.rs

@@ -145,20 +145,31 @@ async fn serve_static(req: actix_web::HttpRequest) -> actix_web::HttpResponse {
 
 #[actix_web::main]
 pub async fn serve(port: &str) -> std::io::Result<()> {
-    // For debugging (will be refactored later)
-    let use_controller = false;
+    let use_controller = {
+        let mut use_controller = false;
+        if let Ok(val) = std::env::var("CONTROLLER") {
+            if let Ok(val) = val.parse::<bool>() {
+                use_controller = val;
+            }
+        }
+        use_controller
+    };
 
     if use_controller {
         // fpm-controller base path and ec2 instance id (hardcoded for now)
-        let fpm_controller: String = "https:///controller.fifthtry.com".to_string();
-        let fpm_instance: String = "<some-ec2-instance-id>".to_string();
+        let fpm_controller: String = std::env::var("FPM_CONTROLLER")
+            .unwrap_or("https:///controller.fifthtry.com".to_string());
+        let fpm_instance: String =
+            std::env::var("FPM_INSTANCE_ID").unwrap_or("<instance_id>".to_string());
 
         match controller::resolve_dependencies(fpm_instance, fpm_controller).await {
             Ok(_) => println!("Dependencies resolved"),
             Err(_) => panic!("Error resolving dependencies using controller!!"),
         }
     }
 
+    let mut config = fpm::Config::read(None).await.unwrap();
+
     println!("### Server Started ###");
     println!("Go to: http://127.0.0.1:{}", port);
     actix_web::HttpServer::new(|| {
@@ -199,38 +210,61 @@ mod controller {
         let package_response = get_package(fpm_instance.as_str(), fpm_controller.as_str()).await?;
         let gp_status = match package_response["success"].as_bool() {
             Some(res) => res,
-            None => panic!("success parameter doesn't exist in Json or isn't valid boolean type"),
+            None => {
+                return Err(fpm::Error::UsageError {
+                    message: "success parameter doesn't exist in Json or isn't valid boolean type"
+                        .to_string(),
+                })
+            }
         };
 
-        match gp_status {
-            true => {
-                // package name and git repo url
-                let package_name = match package_response["result"]["package"].as_str() {
-                    Some(valid_name) => valid_name,
-                    None => panic!("received invalid package name from get_package API"),
-                };
-                if let Some(git_url) = package_response["result"]["git"].as_str() {
-                    // Clone the git package into the current directory
-                    // Need to execute shell commands from rust
-                    // git_url https format: https://github.com/<user>/<repo>.git
-                    let out = std::process::Command::new("git")
-                        .arg("clone")
-                        .arg(git_url)
-                        .output()
-                        .expect("unable to execute git clone command");
-
-                    if out.status.success() {
-                        // By this time the cloned repo should be available in the current directory
-                        println!("Git cloning successful for the package {}", package_name);
-                        // Resolve dependencies by reading the FPM.ftd using config.read()
-                        // Assuming package_name and repo name are identical
-                        let _config = fpm::Config::read(Some(package_name.to_string())).await?;
-                    }
-                } else {
-                    panic!("Invalid git url for the package {}", package_name);
-                }
-            },
-            false => panic!("get-package api success status returned false!!")
+        if !gp_status {
+            return Err(fpm::Error::UsageError {
+                message: "get-package api success status returned false!!".to_string(),
+            });
+        }
+
+        // package name and git repo url
+        let package_name = match package_response["result"]["package"].as_str() {
+            Some(valid_name) => valid_name,
+            None => {
+                return Err(fpm::Error::UsageError {
+                    message: "received invalid package name from get_package API".to_string(),
+                })
+            }
+        };
+
+        if let Some(git_url) = package_response["result"]["git"].as_str() {
+            // Clone the git package into the current directory
+            // Need to execute shell commands from rust
+            // git_url https format: https://github.com/<user>/<repo>.git
+
+            let package = {
+                let mut package = fpm::Package::new(package_name);
+                package.zip = Some(git_url.to_string());
+                package
+            };
+
+            package.unzip_package().await?;
+            fpm::Config::read(None).await?;
+
+            /*let out = std::process::Command::new("git")
+                .arg("clone")
+                .arg(git_url)
+                .output()
+                .expect("unable to execute git clone command");
+
+            if out.status.success() {
+                // By this time the cloned repo should be available in the current directory
+                println!("Git cloning successful for the package {}", package_name);
+                // Resolve dependencies by reading the FPM.ftd using config.read()
+                // Assuming package_name and repo name are identical
+                let _config = fpm::Config::read(Some(package_name.to_string())).await?;
+            }*/
+        } else {
+            return Err(fpm::Error::UsageError {
+                message: "received invalid package name from get_package API".to_string(),
+            });
         }
 
         // Once the dependencies are resolved for the package
@@ -251,72 +285,61 @@ mod controller {
         Ok(())
     }
 
-    async fn make_get_request(url: url::Url) -> fpm::Result<serde_json::Value> {
-        use fpm::library::http;
-        let response = match http::_get(url).await {
-            Ok(some_response) => some_response,
-            Err(e) => {
-                panic!("failed to fetch data, error: {}", e.to_string())
-            }
-        };
-
-        match serde_json::from_str(response.as_str()) {
-            Ok(v) => Ok(v),
-            Err(e) => panic!(
-                "failed parsing json from response text, error: {}",
-                e.to_string()
-            ),
-        }
-    }
-
+    /// get-package API
+    /// input: fpm_instance
+    /// output: package_name and git repo URL
+    /// format: {
+    ///     "success": true,
+    ///     "result": {
+    ///         "package": "<package name>"
+    ///         "git": "<git url>"
+    ///     }
+    /// }
     async fn get_package(
         fpm_instance: &str,
         fpm_controller: &str,
     ) -> fpm::Result<serde_json::Value> {
-        // get-package API
-        // input: fpm_instance
-        // output: package_name and git repo URL
-        // format: {
-        //     "success": true,
-        //     "result": {
-        //         "package": "<package name>"
-        //         "git": "<git url>"
-        //     }
-        // }
-
-        let query_string = format!("instance={}", fpm_instance);
-        let controller_api = format!("{}/get-package?{}", fpm_controller, query_string);
-
+        let controller_api = format!("{}/get-package?instance={}", fpm_controller, fpm_instance);
         let url = match url::Url::parse(controller_api.as_str()) {
             Ok(safe_url) => safe_url,
-            Err(e) => panic!("Invalid get-package API endpoint, Parse error: {}",e.to_string()),
+            Err(e) => panic!(
+                "Invalid get-package API endpoint, Parse error: {}",
+                e.to_string()
+            ),
         };
 
-        make_get_request(url).await
+        let val = fpm::library::http::get(url, "", 0).await?;
+        Ok(val)
     }
 
+    /// fpm-ready API
+    /// input: fpm_instance, *(git commit hash)
+    /// output: success: true/false
+    /// format: lang: json
+    /// {
+    ///     "success": true
+    /// }
+
+    /// Git commit hash needs to be computed before making a call to the fpm_ready API
     async fn fpm_ready(fpm_instance: &str, fpm_controller: &str) -> fpm::Result<serde_json::Value> {
-        // fpm-ready API
-        // input: fpm_instance, *(git commit hash)
-        // output: success: true/false
-        // format: lang: json
-        // {
-        //     "success": true
-        // }
-
-        // Git commit hash needs to be computed before making a call to the fpm_ready API
         let git_commit = "<dummy-git-commit-hash-xxx123>";
 
-        let query_string = format!("instance={}&git-commit={}", fpm_instance, git_commit);
-        let controller_api = format!("{}/fpm-ready?{}", fpm_controller, query_string);
+        let controller_api = format!(
+            "{}/fpm-ready?instance={}&git-commit={}",
+            fpm_controller, fpm_instance, git_commit
+        );
 
         let url = match url::Url::parse(controller_api.as_str()) {
             Ok(safe_url) => safe_url,
-            Err(e) => panic!("Invalid fpm_ready API endpoint, Parse error: {}",e.to_string()),
+            Err(e) => panic!(
+                "Invalid fpm_ready API endpoint, Parse error: {}",
+                e.to_string()
+            ),
         };
 
         // This request should be put request for fpm_ready API to update the instance status to ready
         // Using http::_get() function to make request to this API for now
-        make_get_request(url).await
+        let val = fpm::library::http::get(url, "", 0).await?;
+        Ok(val)
     }
 }

src/dependency.rs

@@ -311,6 +311,74 @@ impl fpm::Package {
         }
     }
 
+    pub(crate) async fn unzip_package(&self) -> fpm::Result<()> {
+        use std::convert::TryInto;
+        use std::io::Write;
+
+        let download_url = if let Some(ref url) = self.zip {
+            url
+        } else {
+            return Ok(());
+        };
+
+        let path = std::env::temp_dir().join(format!("{}.zip", self.name.replace("/", "__")));
+
+        let start = std::time::Instant::now();
+        print!("Downloading {} ... ", self.name.as_str());
+        std::io::stdout().flush()?;
+        // Download the zip folder
+        {
+            let mut response = if download_url[1..].contains("://")
+                || download_url.starts_with("//")
+            {
+                reqwest::get(download_url.as_str())?
+            } else if let Ok(response) = reqwest::get(format!("https://{}", download_url).as_str())
+            {
+                response
+            } else {
+                reqwest::get(format!("http://{}", download_url).as_str())?
+            };
+            let mut file = std::fs::File::create(&path)?;
+            // TODO: instead of reading the whole thing in memory use tokio::io::copy() somehow?
+            let mut buf: Vec<u8> = vec![];
+            response.copy_to(&mut buf)?;
+            file.write_all(&buf)?;
+            // file.write_all(response.text().await?.as_bytes())?;
+        }
+
+        let file = std::fs::File::open(&path)?;
+        // TODO: switch to async_zip crate
+        let mut archive = zip::ZipArchive::new(file)?;
+        for i in 0..archive.len() {
+            let mut c_file = archive.by_index(i).unwrap();
+            let out_path = match c_file.enclosed_name() {
+                Some(path) => path.to_owned(),
+                None => continue,
+            };
+            let out_path_without_folder = out_path.to_str().unwrap().split_once("/").unwrap().1;
+            let file_extract_path = {
+                let mut file_extract_path: camino::Utf8PathBuf =
+                    std::env::current_dir()?.canonicalize()?.try_into()?;
+                file_extract_path = file_extract_path.join(out_path_without_folder);
+                file_extract_path
+            };
+            if (&*c_file.name()).ends_with('/') {
+                std::fs::create_dir_all(&file_extract_path)?;
+            } else {
+                if let Some(p) = file_extract_path.parent() {
+                    if !p.exists() {
+                        std::fs::create_dir_all(p)?;
+                    }
+                }
+                // Note: we will be able to use tokio::io::copy() with async_zip
+                let mut outfile = std::fs::File::create(file_extract_path)?;
+                std::io::copy(&mut c_file, &mut outfile)?;
+            }
+        }
+        fpm::utils::print_end(format!("Downloaded {}", self.name.as_str()).as_str(), start);
+        Ok(())
+    }
+
     /// This function is called by `process()` or recursively called by itself.
     /// It checks the `FPM.ftd` file of dependent package and find out all the dependency packages.
     /// If dependent package is not available, it calls `process()` to download it inside `.packages` directory

src/library/http.rs

@@ -53,7 +53,7 @@ pub async fn processor<'a>(
     doc.from_json(&json, section)
 }
 
-async fn get(
+pub(crate) async fn get(
     url: url::Url,
     doc_id: &str,
     line_number: usize,
@@ -78,7 +78,7 @@ async fn get(
     }
 }
 
-pub async fn _get(url: url::Url) -> reqwest::Result<String> {
+async fn _get(url: url::Url) -> reqwest::Result<String> {
     let mut headers = reqwest::header::HeaderMap::new();
     headers.insert(
         reqwest::header::USER_AGENT,