From 733fc3ea3e2622f1835a35d2c2137a9da28ddfb3 Mon Sep 17 00:00:00 2001 From: mauricefisher64 <92736594+mauricefisher64@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:33:37 -0400 Subject: [PATCH] Initial fragment support (#230) * Initial fragment support * code review fixes --- Cargo.lock | 47 +++++----- Cargo.toml | 3 +- src/main.rs | 247 ++++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 232 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 21405a9..0101fce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,9 +116,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ascii-canvas" @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "async-generic" -version = "0.1.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bc01d15ca67ec7653924057aa97c30de62d06a24e6834387eba6c72c8318e7" +checksum = "80548ada198a0260b9e9d09dfce57919d6c239c3aae2309b3efc1f89096a0e29" dependencies = [ "proc-macro2", "quote", @@ -547,9 +547,9 @@ dependencies = [ [[package]] name = "bitvec-nom2" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4863ce31b7ff8812568eaffe956024c824d845a1f9f08c329706166c357cae53" +checksum = "d988fcc40055ceaa85edc55875a08f8abd29018582647fd82ad6128dba14a5f0" dependencies = [ "bitvec", "nom", @@ -634,9 +634,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.3" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" +checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2" [[package]] name = "byteorder" @@ -661,9 +661,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "c2pa" -version = "0.33.2" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75204624c8d637aef354a01df16b52fa63c833cd3766022f15e0d34dd4b15830" +checksum = "6a58cfe2a1fb0ff7a770dea52bcda16dbfd2b218019b6b56a2d496facee72b35" dependencies = [ "asn1-rs", "async-generic", @@ -744,6 +744,7 @@ dependencies = [ "c2pa", "clap", "env_logger", + "glob", "httpmock", "log", "mockall", @@ -932,9 +933,9 @@ dependencies = [ [[package]] name = "const_panic" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b" +checksum = "7782af8f90fe69a4bb41e460abe1727d493403d8b2cc43201a3a3e906b24379f" [[package]] name = "constant_time_eq" @@ -1431,12 +1432,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.31" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide 0.7.4", + "miniz_oxide 0.8.0", ] [[package]] @@ -1609,6 +1610,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -2998,7 +3005,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf9b0d03fbc7d2dcfdd35086c43ce30ac5ff62ed7eff4397e4f4f2995a2b0e2a" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "bitvec", "bitvec-nom2", "bytes", @@ -3249,9 +3256,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -4154,9 +4161,9 @@ dependencies = [ [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", diff --git a/Cargo.toml b/Cargo.toml index c4b041f..b951181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ repository = "https://github.com/contentauth/c2patool" [dependencies] anyhow = "1.0" -c2pa = { version = "0.33.2", features = [ +c2pa = { version = "0.34.0", features = [ "fetch_remote_manifests", "file_io", "add_thumbnails", @@ -30,6 +30,7 @@ c2pa = { version = "0.33.2", features = [ ] } clap = { version = "4.5.10", features = ["derive", "env"] } env_logger = "0.11.4" +glob = "0.3.1" log = "0.4" serde = { version = "1.0", features = ["derive"] } serde_derive = "1.0" diff --git a/src/main.rs b/src/main.rs index 5d0ff09..b154bb7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,7 @@ use std::{ }; use anyhow::{anyhow, bail, Context, Result}; -use c2pa::{Error, Ingredient, Manifest, ManifestStore, ManifestStoreReport}; +use c2pa::{Error, Ingredient, Manifest, ManifestStore, ManifestStoreReport, Signer}; use clap::{Parser, Subcommand}; use log::debug; use serde::Deserialize; @@ -163,6 +163,22 @@ enum Commands { #[arg(long = "trust_config", env="C2PATOOL_TRUST_CONFIG", value_parser = parse_resource_string)] trust_config: Option, }, + /// Sub-command to add manifest to fragmented BMFF content + /// + /// The init path can be a glob to process entire directories of content, for example: + /// + /// c2patool -m test2.json -o /my_output_folder "/my_renditions/**/my_init.mp4" fragment --fragments_glob "myfile_abc*[0-9].m4s" + /// + /// Note: the glob patterns are quoted to prevent shell expansion. + Fragment { + /// Glob pattern to find the fragments of the asset. The path is automatically set to be the same as + /// the init segment. + /// + /// The fragments_glob pattern should only match fragment file names not the full paths (e.g. "myfile_abc*[0-9].m4s" + /// to match [myfile_abc1.m4s, myfile_abc2180.m4s, ...] ) + #[arg(long = "fragments_glob", verbatim_doc_comment)] + fragments_glob: Option, + }, } #[derive(Debug, Default, Deserialize)] @@ -287,7 +303,7 @@ fn configure_sdk(args: &CliArgs) -> Result<()> { enable_trust_checks = true; } } - None => {} + _ => {} } // if any trust setting is provided enable the trust checks @@ -308,6 +324,103 @@ fn configure_sdk(args: &CliArgs) -> Result<()> { Ok(()) } +fn sign_fragmented( + manifest: &mut Manifest, + signer: &dyn Signer, + init_pattern: &PathBuf, + frag_pattern: &PathBuf, + output_path: &PathBuf, +) -> Result<()> { + // search folders for init segments + let ip = init_pattern.to_str().ok_or(c2pa::Error::OtherError( + "could not parse source pattern".into(), + ))?; + let inits = glob::glob(ip).context("could not process glob pattern")?; + let mut count = 0; + for init in inits { + match init { + Ok(p) => { + let mut fragments = Vec::new(); + let init_dir = p.parent().context("init segment had no parent dir")?; + let seg_glob = init_dir.join(frag_pattern); // segment match pattern + + // grab the fragments that go with this init segment + let seg_glob_str = seg_glob.to_str().context("fragment path not valid")?; + let seg_paths = glob::glob(seg_glob_str).context("fragment glob not valid")?; + for seg in seg_paths { + match seg { + Ok(f) => fragments.push(f), + Err(_) => return Err(anyhow!("fragment path not valid")), + } + } + + println!("Adding manifest to: {:?}", p); + let new_output_path = + output_path.join(init_dir.file_name().context("invalid file name")?); + manifest.embed_to_bmff_fragmented(&p, &fragments, &new_output_path, signer)?; + + count += 1; + } + Err(_) => bail!("bad path to init segment"), + } + } + if count == 0 { + println!("No files matching pattern: {}", ip); + } + Ok(()) +} + +fn verify_fragmented(init_pattern: &PathBuf, frag_pattern: &PathBuf) -> Result> { + let mut stores = Vec::new(); + + let ip = init_pattern + .to_str() + .context("could not parse source pattern")?; + let inits = glob::glob(ip).context("could not process glob pattern")?; + let mut count = 0; + + // search folders for init segments + for init in inits { + match init { + Ok(p) => { + let mut fragments = Vec::new(); + let init_dir = p.parent().context("init segment had no parent dir")?; + let seg_glob = init_dir.join(frag_pattern); // segment match pattern + + // grab the fragments that go with this init segment + let seg_glob_str = seg_glob.to_str().context("fragment path not valid")?; + let seg_paths = glob::glob(seg_glob_str).context("fragment glob not valid")?; + for seg in seg_paths { + match seg { + Ok(f) => fragments.push(f), + Err(_) => return Err(anyhow!("fragment path not valid")), + } + } + + println!("Verifying manifest: {:?}", p); + let store = ManifestStore::from_fragments(p, &fragments, true)?; + if let Some(vs) = store.validation_status() { + if let Some(e) = vs.iter().find(|v| !v.passed()) { + eprintln!("Error validating segments: {:?}", e); + return Ok(stores); + } + } + + stores.push(store); + + count += 1; + } + Err(_) => bail!("bad path to init segment"), + } + } + + if count == 0 { + println!("No files matching pattern: {}", ip); + } + + Ok(stores) +} + fn main() -> Result<()> { let args = CliArgs::parse(); @@ -333,6 +446,17 @@ fn main() -> Result<()> { return Ok(()); } + let is_fragment = if let Some(Commands::Fragment { fragments_glob: _ }) = &args.command { + true + } else { + false + }; + + // make sure path is not a glob when not fragmented + if !args.path.is_file() && !is_fragment { + bail!("glob patterns only allowed when using \"fragment\" command") + } + // configure the SDK configure_sdk(&args).context("Could not configure c2pa-rs")?; @@ -413,7 +537,7 @@ fn main() -> Result<()> { // If the source file has a manifest store, and no parent is specified treat the source as a parent. // note: This could be treated as an update manifest eventually since the image is the same - if manifest.parent().is_none() { + if manifest.parent().is_none() && !is_fragment { let source_ingredient = Ingredient::from_file(&args.path)?; if source_ingredient.manifest_data().is_some() { manifest.set_parent(source_ingredient)?; @@ -430,50 +554,69 @@ fn main() -> Result<()> { manifest.set_sidecar_manifest(); } - if let Some(output) = args.output { - if ext_normal(&output) != ext_normal(&args.path) { - bail!("Output type must match source type"); - } - if output.exists() && !args.force { - bail!("Output already exists, use -f/force to force write"); - } + let signer = if let Some(signer_process_name) = args.signer_path { + let cb_config = CallbackSignerConfig::new(&sign_config, args.reserve_size)?; - if output.file_name().is_none() { - bail!("Missing filename on output"); - } - if output.extension().is_none() { - bail!("Missing extension output"); - } + let process_runner = Box::new(ExternalProcessRunner::new( + cb_config.clone(), + signer_process_name, + )); + let signer = CallbackSigner::new(process_runner, cb_config); - let signer = if let Some(signer_process_name) = args.signer_path { - let cb_config = CallbackSignerConfig::new(&sign_config, args.reserve_size)?; + Box::new(signer) + } else { + sign_config.signer()? + }; - let process_runner = Box::new(ExternalProcessRunner::new( - cb_config.clone(), - signer_process_name, - )); - let signer = CallbackSigner::new(process_runner, cb_config); + if let Some(output) = args.output { + // fragmented embedding + if let Some(Commands::Fragment { fragments_glob }) = &args.command { + if output.exists() && !output.is_dir() { + bail!("Output cannot point to existing file, must be a directory"); + } - Box::new(signer) + if let Some(fg) = &fragments_glob { + return sign_fragmented( + &mut manifest, + signer.as_ref(), + &args.path, + fg, + &output, + ); + } else { + bail!("fragments_glob must be set"); + } } else { - sign_config.signer()? - }; + if ext_normal(&output) != ext_normal(&args.path) { + bail!("Output type must match source type"); + } + if output.exists() && !args.force { + bail!("Output already exists, use -f/force to force write"); + } - manifest - .embed(&args.path, &output, signer.as_ref()) - .context("embedding manifest")?; + if output.file_name().is_none() { + bail!("Missing filename on output"); + } + if output.extension().is_none() { + bail!("Missing extension output"); + } - // generate a report on the output file - if args.detailed { - println!( - "{}", - ManifestStoreReport::from_file(&output).map_err(special_errs)? - ); - } else { - println!( - "{}", - ManifestStore::from_file(&output).map_err(special_errs)? - ) + manifest + .embed(&args.path, &output, signer.as_ref()) + .context("embedding manifest")?; + + // generate a report on the output file + if args.detailed { + println!( + "{}", + ManifestStoreReport::from_file(&output).map_err(special_errs)? + ); + } else { + println!( + "{}", + ManifestStore::from_file(&output).map_err(special_errs)? + ) + } } } else { bail!("Output path required with manifest definition") @@ -524,10 +667,26 @@ fn main() -> Result<()> { ManifestStoreReport::from_file(&args.path).map_err(special_errs)? ) } else { - println!( - "{}", - ManifestStore::from_file(&args.path).map_err(special_errs)? - ) + if let Some(Commands::Fragment { fragments_glob }) = &args.command { + if let Some(fg) = &fragments_glob { + let stores = verify_fragmented(&args.path, fg)?; + if stores.len() == 1 { + println!("{}", stores[0]); + } else { + println!("{} Init manifests validated", stores.len()); + } + } else { + println!( + "{}", + ManifestStore::from_file(&args.path).map_err(special_errs)? + ) + } + } else { + println!( + "{}", + ManifestStore::from_file(&args.path).map_err(special_errs)? + ) + } } Ok(())