diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e32996caf..c0d72b576 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,13 @@ jobs: - name: Run clippy run: cargo clippy --all -- -Dwarnings + - name: Run compute-static tests + working-directory: terragrunt/modules/crates-io/compute-static + run: | + curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin + cargo install --locked viceroy --version 0.15.0 + ./scripts/run_tests.sh + terraform: name: Terraform configuration runs-on: ubuntu-24.04 diff --git a/terragrunt/modules/crates-io/compute-static/.cargo/config.toml b/terragrunt/modules/crates-io/compute-static/.cargo/config.toml index 6b77899cb..a2d544fb9 100644 --- a/terragrunt/modules/crates-io/compute-static/.cargo/config.toml +++ b/terragrunt/modules/crates-io/compute-static/.cargo/config.toml @@ -1,2 +1,5 @@ [build] -target = "wasm32-wasi" +target = "wasm32-wasip1" + +[target.wasm32-wasip1] +runner = "viceroy run -C fastly.toml -- " \ No newline at end of file diff --git a/terragrunt/modules/crates-io/compute-static/Cargo.lock b/terragrunt/modules/crates-io/compute-static/Cargo.lock index 3776a6108..3150f8e47 100644 --- a/terragrunt/modules/crates-io/compute-static/Cargo.lock +++ b/terragrunt/modules/crates-io/compute-static/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "bitflags" @@ -40,9 +40,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.3.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cfg-if" @@ -95,7 +95,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 1.0.107", ] [[package]] @@ -106,7 +106,7 @@ checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ "darling_core", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -127,7 +127,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -137,7 +137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.107", ] [[package]] @@ -181,7 +181,7 @@ checksum = "6b5163881fe9bab8e865258351e931635d2d17ed88113e546db7c7e57f7249e9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -236,7 +236,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ - "bytes 1.3.0", + "bytes 1.10.1", "fnv", "itoa", ] @@ -271,18 +271,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "log" -version = "0.4.17" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "log-fastly" @@ -297,9 +294,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "mime" @@ -327,18 +324,18 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -368,22 +365,32 @@ checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.109", ] [[package]] @@ -439,6 +446,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "thiserror" version = "1.0.38" @@ -456,7 +474,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] diff --git a/terragrunt/modules/crates-io/compute-static/README.md b/terragrunt/modules/crates-io/compute-static/README.md index da25b0287..a905783e6 100644 --- a/terragrunt/modules/crates-io/compute-static/README.md +++ b/terragrunt/modules/crates-io/compute-static/README.md @@ -8,13 +8,43 @@ responses. ## Development -Build the function: +Install the Fastly CLI from [here](https://www.fastly.com/documentation/reference/tools/cli/#installing). + +Then, build the function: ```shell cd compute-static fastly compute build ``` +## Testing + +Testing requires [cargo-nextest](https://nexte.st/docs/installation/pre-built-binaries/), as specified in +the [Fastly documentation](https://www.fastly.com/documentation/guides/compute/developer-guides/rust/). You can install +it from source with binstall: + +```shell +cargo install cargo-binstall +cargo binstall cargo-nextest --secure +``` + +Then, install [Viceroy](https://github.com/fastly/Viceroy) to run the edge function locally: + +```shell +cargo install --locked viceroy +``` + +Due to the fact Viceroy does not allow easily mocking HTTP requests being sent ( +see [issue](https://github.com/fastly/Viceroy/issues/442)), some tests use a small Python HTTP +server to work. +For this reason, a wrapper bash script is provided that runs `cargo nextest run` with the test server active in +background. You can therefore run the tests with: +: + +```shell +./run_tests.sh +``` + ## Deployment Terraform uses an [external data source] to build the function as part of its diff --git a/terragrunt/modules/crates-io/compute-static/rust-toolchain.toml b/terragrunt/modules/crates-io/compute-static/rust-toolchain.toml index 88a12d5ff..13a15c4e0 100644 --- a/terragrunt/modules/crates-io/compute-static/rust-toolchain.toml +++ b/terragrunt/modules/crates-io/compute-static/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "stable" +channel = "1.90" targets = ["wasm32-wasip1"] diff --git a/terragrunt/modules/crates-io/compute-static/scripts/run_tests.sh b/terragrunt/modules/crates-io/compute-static/scripts/run_tests.sh new file mode 100755 index 000000000..67079aa2c --- /dev/null +++ b/terragrunt/modules/crates-io/compute-static/scripts/run_tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Start the Python HTTP server in the background, which will act as primary host for requests sent from Rust tests +python3 scripts/test_http_server.py & +SERVER_PID=$! + +echo "HTTP server started with PID: $SERVER_PID" + +# Run the tests while the HTTP server is active in background +cargo nextest run +CARGO_EXIT_CODE=$? + +kill $SERVER_PID +echo "HTTP server stopped" + +exit $CARGO_EXIT_CODE \ No newline at end of file diff --git a/terragrunt/modules/crates-io/compute-static/scripts/test_http_server.py b/terragrunt/modules/crates-io/compute-static/scripts/test_http_server.py new file mode 100644 index 000000000..35e54a2b4 --- /dev/null +++ b/terragrunt/modules/crates-io/compute-static/scripts/test_http_server.py @@ -0,0 +1,16 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer + + +class MockHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/crates/libgit2-sys/libgit2-sys-0.12.25%2B1.3.0.crate": + self.send_response(200) + self.end_headers() + # The written data is used in Rust tests to verify whether the primary host (this) has actually been queried + self.wfile.write(b'test_data') + else: + self.send_response(404) + self.end_headers() + + +HTTPServer(("127.0.0.1", 8080), MockHandler).serve_forever() diff --git a/terragrunt/modules/crates-io/compute-static/src/log_line.rs b/terragrunt/modules/crates-io/compute-static/src/log_line.rs index 0604c615e..a175e2895 100644 --- a/terragrunt/modules/crates-io/compute-static/src/log_line.rs +++ b/terragrunt/modules/crates-io/compute-static/src/log_line.rs @@ -1,9 +1,16 @@ +use std::env::var; use std::net::IpAddr; +use crate::config::Config; +use crate::http_version_to_string; use derive_builder::Builder; +use fastly::{Error, Request, Response}; use serde::Serialize; use time::OffsetDateTime; +const DATADOG_APP: &str = "crates.io"; +const DATADOG_SERVICE: &str = "static.crates.io"; + #[derive(Debug, Serialize)] #[serde(tag = "version")] pub enum LogLine { @@ -33,6 +40,65 @@ pub struct LogLineV1 { url: String, } +impl LogLineV1 { + /// Collect data for the logs from the request + pub fn collect_request(config: &Config, request: &Request) -> LogLineV1Builder { + let http_details = HttpDetailsBuilder::default() + .protocol(http_version_to_string(request.get_version())) + .referer( + request + .get_header("Referer") + .and_then(|s| s.to_str().ok()) + .map(|s| s.to_string()), + ) + .useragent( + request + .get_header("User-Agent") + .and_then(|s| s.to_str().ok()) + .map(|s| s.to_string()), + ) + .build() + .ok(); + + let tls_details = TlsDetailsBuilder::default() + .cipher(request.get_tls_cipher_openssl_name()) + .protocol(request.get_tls_protocol()) + .build() + .ok(); + + let log_line = LogLineV1Builder::default() + .ddtags(format!("app:{},env:{}", DATADOG_APP, config.datadog_env)) + .service(DATADOG_SERVICE) + .date_time(OffsetDateTime::now_utc()) + .edge_location(var("FASTLY_POP").ok()) + .host(request.get_url().host().map(|s| s.to_string())) + .http(http_details) + .ip(request.get_client_ip_addr()) + .method(Some(request.get_method().to_string())) + .url(request.get_url_str().into()) + .tls(tls_details) + .to_owned(); + + log_line + } + + /// Collect data for the logs from the response + pub fn collect_response( + log_line: &mut LogLineV1Builder, + response: &Result, + ) -> LogLineV1Builder { + if let Ok(response) = response { + log_line + .bytes(response.get_content_length()) + .content_type(response.get_content_type().map(|s| s.to_string())) + .status(Some(response.get_status().as_u16())) + .to_owned() + } else { + log_line.status(Some(500)).to_owned() + } + } +} + fn default_source() -> &'static str { "fastly" } @@ -49,3 +115,43 @@ pub struct TlsDetails { cipher: Option<&'static str>, protocol: Option<&'static str>, } + +#[cfg(test)] +mod tests { + use super::*; + + /// Verifies whether the log collector can correctly build a log line from a request/response pair. + #[test] + fn test_log_collector() { + let client_req = Request::get("https://crates.io/crates/syn"); // Arbitrary crate with no meaning + let config = Config { + primary_host: "test_backend".to_string(), + fallback_host: "fallback_host".to_string(), + static_ttl: 0, + cloudfront_url: "cloudfront_url".to_string(), + datadog_env: "datadog_env".to_string(), + datadog_host: "datadog_host".to_string(), + datadog_request_logs_endpoint: "datadog_request_logs_endpoint".to_string(), + s3_request_logs_endpoint: "s3_request_logs_endpoint".to_string(), + s3_service_logs_endpoint: "s3_service_logs_endpoint".to_string(), + }; + let mut log = LogLineV1::collect_request(&config, &client_req); + let log = LogLineV1::collect_response( + &mut log, + &Ok(Response::temporary_redirect("https://crates.io/")), + ); + let log = log.build().unwrap(); + assert_eq!(log.ddsource, "fastly"); + assert_eq!( + log.ddtags, + format!("app:crates.io,env:{}", config.datadog_env) + ); + assert_eq!(log.service, "static.crates.io"); + assert_eq!( + log.host, + Some(client_req.get_url().host().unwrap().to_string()) + ); + assert_eq!(log.url, client_req.get_url_str()); + assert_eq!(log.method, Some(client_req.get_method().to_string())); + } +} diff --git a/terragrunt/modules/crates-io/compute-static/src/main.rs b/terragrunt/modules/crates-io/compute-static/src/main.rs index ba2c90907..4ce4ae5ba 100644 --- a/terragrunt/modules/crates-io/compute-static/src/main.rs +++ b/terragrunt/modules/crates-io/compute-static/src/main.rs @@ -1,23 +1,18 @@ -use fastly::convert::ToHeaderValue; -use fastly::http::{header, Method, StatusCode, Version}; +use fastly::http::{Method, StatusCode, Version}; use fastly::{Error, Request, Response}; use log::{info, warn, LevelFilter}; use log_fastly::Logger; use once_cell::sync::Lazy; use regex::Regex; use serde_json::json; -use std::env::var; -use time::OffsetDateTime; use crate::config::Config; -use crate::log_line::{HttpDetailsBuilder, LogLine, LogLineV1Builder, TlsDetailsBuilder}; +use crate::log_line::{LogLine, LogLineV1, LogLineV1Builder}; mod config; mod log_line; -const DATADOG_APP: &str = "crates.io"; -const DATADOG_SERVICE: &str = "static.crates.io"; -const VERSION_DOWNLOADS: &str = "/archive/version-downloads/"; +const VERSION_DOWNLOADS: &str = "/archive/version-downloads"; #[fastly::main] fn main(request: Request) -> Result { @@ -30,7 +25,7 @@ fn main(request: Request) -> Result { } init_logging(&config); - let mut log = collect_request(&config, &request); + let mut log = LogLineV1::collect_request(&config, &request); let has_origin_header = request.get_header("Origin").is_some(); let mut response = handle_request(&config, request); @@ -39,7 +34,7 @@ fn main(request: Request) -> Result { add_cors_headers(&mut response); } - let log = collect_response(&mut log, &response); + let log = LogLineV1::collect_response(&mut log, &response); build_and_send_log(log, &config); response @@ -61,47 +56,6 @@ fn init_logging(config: &Config) { .init(); } -/// Collect data for the logs from the request -fn collect_request(config: &Config, request: &Request) -> LogLineV1Builder { - let http_details = HttpDetailsBuilder::default() - .protocol(http_version_to_string(request.get_version())) - .referer( - request - .get_header("Referer") - .and_then(|s| s.to_str().ok()) - .map(|s| s.to_string()), - ) - .useragent( - request - .get_header("User-Agent") - .and_then(|s| s.to_str().ok()) - .map(|s| s.to_string()), - ) - .build() - .ok(); - - let tls_details = TlsDetailsBuilder::default() - .cipher(request.get_tls_cipher_openssl_name()) - .protocol(request.get_tls_protocol()) - .build() - .ok(); - - let log_line = LogLineV1Builder::default() - .ddtags(format!("app:{},env:{}", DATADOG_APP, config.datadog_env)) - .service(DATADOG_SERVICE) - .date_time(OffsetDateTime::now_utc()) - .edge_location(var("FASTLY_POP").ok()) - .host(request.get_url().host().map(|s| s.to_string())) - .http(http_details) - .ip(request.get_client_ip_addr()) - .method(Some(request.get_method().to_string())) - .url(request.get_url_str().into()) - .tls(tls_details) - .to_owned(); - - log_line -} - /// Handle the request /// /// This method handles the incoming request and returns a response for the client. It first ensures @@ -112,17 +66,7 @@ fn handle_request(config: &Config, mut request: Request) -> Result Result Response { - Response::new() - .with_status(StatusCode::PERMANENT_REDIRECT) - .with_header(header::LOCATION, destination) +/// Applies required modifications on the request +fn rewrite_request(config: &Config, request: &mut Request) { + set_ttl(config, request); + rewrite_urls_with_plus_character(request); + rewrite_download_urls(request); + rewrite_version_downloads_urls(request); } /// Limit HTTP methods @@ -189,10 +135,10 @@ fn rewrite_urls_with_plus_character(request: &mut Request) { /// In this way, users can see what files are available for download. fn rewrite_version_downloads_urls(request: &mut Request) { let url = request.get_url_mut(); - let path = url.path(); + let path = url.path().trim_end_matches('/'); if path == VERSION_DOWNLOADS { - let new_path = format!("{path}index.html"); + let new_path = format!("{path}/index.html"); url.set_path(&new_path); } } @@ -264,22 +210,6 @@ fn add_cors_headers(response: &mut Result) { } } -/// Collect data for the logs from the response -fn collect_response( - log_line: &mut LogLineV1Builder, - response: &Result, -) -> LogLineV1Builder { - if let Ok(response) = response { - log_line - .bytes(response.get_content_length()) - .content_type(response.get_content_type().map(|s| s.to_string())) - .status(Some(response.get_status().as_u16())) - .to_owned() - } else { - log_line.status(Some(500)).to_owned() - } -} - /// Finalize the builder and log the line fn build_and_send_log(log_line: LogLineV1Builder, config: &Config) { match log_line.build() { @@ -315,6 +245,23 @@ fn http_version_to_string(version: Version) -> Option { #[cfg(test)] mod tests { use super::*; + use fastly::experimental::BackendExt; + use fastly::Backend; + + fn new_test_config() -> Config { + Config { + // For tests sending requests to S3, ensure this points at the web server config defined in scripts/test_http_server.py. + primary_host: "127.0.0.1:8080".to_string(), + fallback_host: "fallback_host".to_string(), + static_ttl: 0, + cloudfront_url: "cloudfront_url".to_string(), + datadog_env: "datadog_env".to_string(), + datadog_host: "datadog_host".to_string(), + datadog_request_logs_endpoint: "datadog_request_logs_endpoint".to_string(), + s3_request_logs_endpoint: "s3_request_logs_endpoint".to_string(), + s3_service_logs_endpoint: "s3_service_logs_endpoint".to_string(), + } + } #[test] fn test_rewrite_download_urls() { @@ -341,4 +288,57 @@ mod tests { "https://static.crates.io/crates/serde/serde-1.0.0-alpha.1+foo-bar.crate", ); } + + /// Ensure plus symbols in crates versions are properly URL encoded (see https://github.com/rust-lang/simpleinfra/pull/313) + #[test] + fn test_plus_encoding() { + // Example taken from the GitHub issue above + let mut client_req = Request::get( + "https://static.crates.io/crates/libgit2-sys/libgit2-sys-0.12.25+1.3.0.crate", + ); + let config = new_test_config(); + + rewrite_request(&config, &mut client_req); + assert_eq!( + client_req.get_path(), + "/crates/libgit2-sys/libgit2-sys-0.12.25%2B1.3.0.crate" + ); + } + + /// Ensures visiting version-downloads with and without a trailing slash properly redirects to the index.html + #[test] + fn test_version_download_url() { + let mut client_req = Request::get("https://static.crates.io/archive/version-downloads/"); + let config = new_test_config(); + rewrite_request(&config, &mut client_req); + assert_eq!( + client_req.get_path(), + "/archive/version-downloads/index.html" + ); + + let mut client_req = Request::get("https://static.crates.io/archive/version-downloads"); + rewrite_request(&config, &mut client_req); + assert_eq!( + client_req.get_path(), + "/archive/version-downloads/index.html" + ); + } + + /// Ensures that the request is forwarded to the primary host for normal queries. + /// Assumes the tests are being executed with the test HTTP Python server active in background (see README.md). + #[test] + fn test_handle_request() { + let config = new_test_config(); + Backend::builder(&config.primary_host, &config.primary_host) + .finish() + .unwrap(); + let client_req = Request::get( + "https://static.crates.io/crates/libgit2-sys/libgit2-sys-0.12.25+1.3.0.crate", + ); + let mut res = handle_request(&config, client_req).unwrap(); + assert_eq!(res.get_status(), StatusCode::OK); + // Assuming the function sent a request to the primary host, verify whether the body is the one set in the test server + let body = res.take_body_str(); + assert_eq!(body, "test_data"); + } }