use module_fetcher to load modules

Author: laktek

Cargo.lock

@@ -21,6 +21,7 @@ deno_web = { version = "0.116.0" }
 deno_websocket = { version = "0.90.0" }
 httparse = { version = "1.8.0" }
 log = { workspace = true }
+module_fetcher = { path = "../module_fetcher" }
 reqwest = { version = "0.11.13" }
 tokio = { version = "1.24", features = ["full"] }
 v8 = { version = "0.60.1", default-features = false }

base/Cargo.toml

@@ -90,10 +90,13 @@ fn start_runtime(
         permissions::init(),
     ];
 
+    // FIXME: module_loader can panic
+    let module_loader = DefaultModuleLoader::new().unwrap();
+
     let mut js_runtime = JsRuntime::new(RuntimeOptions {
         extensions,
         extensions_with_js,
-        module_loader: Some(Rc::new(DefaultModuleLoader)),
+        module_loader: Some(Rc::new(module_loader)),
         is_main: true,
         create_params: Some(v8::CreateParams::default().heap_limits(
             mib_to_bytes(1) as usize,

base/src/js_worker.rs

@@ -3,26 +3,25 @@ use deno_ast::EmitOptions;
 use deno_ast::MediaType;
 use deno_ast::ParseParams;
 use deno_ast::SourceTextInfo;
+use deno_core::error::AnyError;
 use deno_core::futures::FutureExt;
 use deno_core::ModuleLoader;
 use deno_core::ModuleSource;
 use deno_core::ModuleSourceFuture;
 use deno_core::ModuleSpecifier;
 use deno_core::ModuleType;
 use deno_core::ResolutionKind;
+use module_fetcher::cache::{DenoDir, HttpCache};
+use module_fetcher::file_fetcher::{CacheSetting, FileFetcher};
+use module_fetcher::http_util::HttpClient;
 use std::pin::Pin;
 
 struct ModuleTypeResult {
-    media_type: MediaType,
     module_type: ModuleType,
     should_transpile: bool,
 }
 
-fn get_module_type(
-    module_specifier: ModuleSpecifier,
-    content_type: &str,
-) -> Result<ModuleTypeResult, Error> {
-    let media_type = MediaType::from_content_type(&module_specifier, content_type);
+fn get_module_type(media_type: MediaType) -> Result<ModuleTypeResult, Error> {
     let (module_type, should_transpile) = match media_type {
         MediaType::JavaScript | MediaType::Mjs | MediaType::Cjs => (ModuleType::JavaScript, false),
         MediaType::Jsx => (ModuleType::JavaScript, true),
@@ -34,21 +33,52 @@ fn get_module_type(
         | MediaType::Dcts
         | MediaType::Tsx => (ModuleType::JavaScript, true),
         MediaType::Json => (ModuleType::Json, false),
-        _ => bail!(
-            "{:?} module type not supported (specifier {:?})",
-            media_type,
-            module_specifier.as_str()
-        ),
+        _ => bail!("{:?} module type not supported", media_type,),
     };
 
     Ok(ModuleTypeResult {
-        media_type,
         module_type,
         should_transpile,
     })
 }
 
-pub struct DefaultModuleLoader;
+fn make_http_client() -> Result<HttpClient, AnyError> {
+    let root_cert_store = None;
+    let unsafely_ignore_certificate_errors = None;
+    HttpClient::new(root_cert_store, unsafely_ignore_certificate_errors)
+}
+
+pub struct DefaultModuleLoader {
+    file_fetcher: FileFetcher,
+    permissions: module_fetcher::permissions::Permissions,
+}
+
+impl DefaultModuleLoader {
+    pub fn new() -> Result<Self, AnyError> {
+        // Note: we are reusing Deno dependency cache path
+        let deno_dir = DenoDir::new(None)?;
+        let deps_cache_location = deno_dir.deps_folder_path();
+
+        let http_cache = HttpCache::new(&deps_cache_location);
+        let cache_setting = CacheSetting::Use;
+        let allow_remote = true;
+        let http_client = make_http_client()?;
+        let blob_store = deno_web::BlobStore::default();
+        let file_fetcher = FileFetcher::new(
+            http_cache,
+            cache_setting,
+            allow_remote,
+            http_client,
+            blob_store,
+        )?;
+        let permissions = module_fetcher::permissions::Permissions::default();
+
+        Ok(Self {
+            file_fetcher,
+            permissions,
+        })
+    }
+}
 
 impl ModuleLoader for DefaultModuleLoader {
     fn resolve(
@@ -60,55 +90,30 @@ impl ModuleLoader for DefaultModuleLoader {
         Ok(deno_core::resolve_import(specifier, referrer)?)
     }
 
+    // TODO: implement prepare_load method
     fn load(
         &self,
         module_specifier: &ModuleSpecifier,
         _maybe_referrer: Option<ModuleSpecifier>,
         _is_dyn_import: bool,
     ) -> Pin<Box<ModuleSourceFuture>> {
+        let file_fetcher = self.file_fetcher.clone();
+        let permissions = self.permissions.clone();
         let module_specifier = module_specifier.clone();
+
         async move {
-            let (media_type, module_type, code, should_transpile) = match module_specifier.scheme()
-            {
-                "file" => {
-                    let content_type = "text/plain";
-                    let ModuleTypeResult {
-                        media_type,
-                        module_type,
-                        should_transpile,
-                    } = get_module_type(module_specifier.clone(), &content_type)?;
-                    let path = module_specifier.to_file_path().unwrap();
-                    let code = std::fs::read_to_string(&path)?;
-                    (media_type, module_type, code, should_transpile)
-                }
-                "http" | "https" => {
-                    let res = reqwest::get(module_specifier.clone()).await?;
-                    let content_type = res
-                        .headers()
-                        .get("content-type")
-                        .map(|v| v.to_str())
-                        .unwrap()?;
-                    let ModuleTypeResult {
-                        media_type,
-                        module_type,
-                        should_transpile,
-                    } = get_module_type(module_specifier.clone(), content_type)?;
-                    let code = res.text().await?;
-                    (media_type, module_type, code, should_transpile)
-                }
-                _ => {
-                    bail!(
-                        "unsupported module url scheme: {:?}",
-                        module_specifier.scheme()
-                    )
-                }
-            };
+            let fetched_file = file_fetcher.fetch(&module_specifier, permissions).await?;
+            let ModuleTypeResult {
+                module_type,
+                should_transpile,
+            } = get_module_type(fetched_file.media_type)?;
 
+            let code = fetched_file.source.to_string();
             let code = if should_transpile {
                 let parsed = deno_ast::parse_module(ParseParams {
                     specifier: module_specifier.to_string(),
                     text_info: SourceTextInfo::from_string(code),
-                    media_type,
+                    media_type: fetched_file.media_type,
                     capture_tokens: false,
                     scope_analysis: false,
                     maybe_syntax: None,
@@ -129,7 +134,7 @@ impl ModuleLoader for DefaultModuleLoader {
                 code: code.into_bytes().into_boxed_slice(),
                 module_type,
                 module_url_specified: module_specifier.to_string(),
-                module_url_found: module_specifier.to_string(),
+                module_url_found: fetched_file.specifier.to_string(),
             };
             Ok(module)
         }