browse and json_file and local

src/default_data.rs → src/get_data.rs

@@ -1,3 +1,59 @@
+use std::{fs::File, io::Read, path::Path};
+
+use color_eyre::eyre;
+
+pub async fn get_json_data(json_file: String) -> color_eyre::Result<String> {
+    // Ensure that the specified data file has the correct extension.
+    if !json_file.to_lowercase().ends_with(".json") {
+        return Err(eyre::eyre!(
+            "Incorrect JSON file extension for data file '{}'.",
+            json_file
+        ));
+    }
+
+    let json_str = match json_file.to_lowercase().starts_with("http") {
+        true => {
+            // Reference is to a URL.
+            let response = reqwest::get(&json_file).await?;
+
+            if !response.status().is_success() {
+                return Err(eyre::eyre!(
+                    "Bad response while accessing data file URL '{}'.",
+                    &json_file
+                ));
+            }
+
+            let site_data: String = response.text().await.map_err(|error| {
+                eyre::eyre!(
+                    "Problem parsing JSON contents at '{}': {}.",
+                    json_file,
+                    error
+                )
+            })?;
+
+            site_data
+        }
+        false => {
+            // Reference is to a file.
+            let path = Path::new(&json_file);
+
+            let mut file = File::open(path).map_err(|_| {
+                eyre::eyre!(
+                    "Problem while attempting to access data file '{}'.",
+                    json_file
+                )
+            })?;
+
+            let mut contents = String::new();
+            file.read_to_string(&mut contents)?;
+
+            contents
+        }
+    };
+
+    Ok(json_str)
+}
+
 /// the default sites to check for sherlock
 /// includes >400 websites and their error messages
 pub fn get_default_data() -> String {

src/lib.rs

@@ -1,4 +1,4 @@
-pub mod default_data;
+pub mod get_data;
 pub mod interpolate;
 pub mod output;
 pub mod query_result;

src/main.rs

@@ -1,15 +1,17 @@
 use clap::Parser;
 use color_eyre::Result;
 use sherlock_rs::{
-    default_data::get_default_data, output::save_results, sherlock::check_username,
+    get_data::{get_default_data, get_json_data},
+    output::save_results,
+    sherlock::check_username,
     sherlock_target_manifest::SherlockTargetManifest,
 };
 
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
 #[command(name = "sherlock")]
 #[command(author = "Johannes Naylor <[email protected]>")]
-#[command(version = "1.0")]
+#[command(version = "0.1.0")]
 #[command(about = "Hunt down social media accounts by username", long_about = None)]
 struct Cli {
     /// One or more usernames to check with social networks. Check similar usernames using {?} (replace to '_', '-', '.').
@@ -49,8 +51,12 @@ struct Cli {
     dump_response: bool,
 
     /// Load data from a JSON file or an online, valid, JSON file.
-    #[clap(short, long = "json")]
-    json_file: Option<String>,
+    #[clap(
+        short,
+        long = "json",
+        default_value = "https://raw.githubusercontent.com/sherlock-project/sherlock/master/sherlock_project/resources/data.json"
+    )]
+    json_file: String,
 
     /// Time (in seconds) to wait for response to requests.
     #[clap(short, long, alias = "timeout", default_value_t = 60)]
@@ -68,8 +74,8 @@ struct Cli {
     #[clap(short, long, alias = "browse")]
     browse: bool,
 
-    /// Force the use of the local data.json file.
-    #[clap(short, long, alias = "local")]
+    /// Use local data file instead of fetching the latest version online.
+    #[clap(short, long)]
     local: bool,
 
     /// Include checking of NSFW sites from default list.
@@ -83,9 +89,13 @@ async fn main() -> Result<()> {
 
     let cli = Cli::parse();
 
-    let json_data = get_default_data();
+    let json_str = match cli.local {
+        true => get_default_data(),
+        false => get_json_data(cli.json_file).await?,
+    };
+
     // let json_data = include_str!("../resources/data.json");
-    let deserializer = &mut serde_json::Deserializer::from_str(&json_data);
+    let deserializer = &mut serde_json::Deserializer::from_str(&json_str);
     let initial_data: SherlockTargetManifest = serde_path_to_error::deserialize(deserializer)
         .inspect_err(|err| {
             println!("[!!!] error path [{}]", err.path());