Include macro support (#291)

* Initial basic support for includes

* Use path instead of name

* Fix path trimming. Mark RFD as needs update on content update

* Only do include processing when there is at least one include

Author: augustuswm

Cargo.lock

@@ -39,15 +39,15 @@ hmac = "0.12.1"
 http = "1.2.0"
 hyper = "1.6.0"
 itertools = "0.13.0"
-jsonwebtoken = "9.3.0"
+jsonwebtoken = "9"
 meilisearch-sdk = "0.27.1"
 md-5 = "0.10.6"
 mime_guess = "2.0.5"
 mockall = "0.13.1"
 newline-converter = "0.3.0"
 newtype-uuid = { version = "1.2.1", features = ["schemars08", "serde", "v4"] }
 oauth2 = { version = "4.4.2", default-features = false, features = ["rustls-tls"] }
-octorust = "0.9.0"
+octorust = "0.10.0"
 owo-colors = "4.1.0"
 partial-struct = { git = "https://github.com/oxidecomputer/partial-struct" }
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }

Cargo.toml

@@ -157,6 +157,40 @@ impl<'a> RfdAsciidoc<'a> {
     fn body(content: &str) -> Option<&str> {
         Self::title_pattern().splitn(content, 2).nth(1)
     }
+
+    fn include_pattern() -> Regex {
+        Regex::new(r"(?m)^include::(.*)\[\]$").unwrap()
+    }
+
+    pub fn includes(&'a self) -> Vec<AsciidocInclude<'a>> {
+        Self::include_pattern()
+            .captures_iter(&self.resolved)
+            .map(|capture| {
+                let extracted = capture.extract::<1>();
+                AsciidocInclude {
+                    file: extracted.1[0],
+                    replacement: extracted.0,
+                }
+            })
+            .collect::<Vec<_>>()
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub struct AsciidocInclude<'a> {
+    file: &'a str,
+    replacement: &'a str,
+}
+
+impl<'a> AsciidocInclude<'a> {
+    pub fn name(&self) -> &str {
+        self.file
+    }
+
+    pub fn perform_replacement(&self, body: &str, new_content: &str) -> String {
+        tracing::trace!(self.replacement, "Replacing include");
+        body.replace(self.replacement, new_content)
+    }
 }
 
 impl<'a> RfdDocument for RfdAsciidoc<'a> {
@@ -197,36 +231,6 @@ impl<'a> RfdDocument for RfdAsciidoc<'a> {
 
     fn get_authors(&self) -> Option<&str> {
         Self::attr("authors", &self.resolved)
-        // // If an authors attribute is defined anywhere in the document, then it is the first choice
-        // // for the authors value
-        // if let Some(attr) = Self::attr("authors", &self.resolved) {
-        //     Some(attr)
-        // } else {
-        //     self.body().and_then(|body| {
-        //         body.lines().nth(0).and_then(|first_line| {
-        //             // If {authors} is found, instead search the header for an authors attribute
-        //             if first_line == "{authors}" {
-        //                 Self::attr("authors", &self.resolved)
-        //             } else {
-        //                 // Given that we are in a fallback case we need to be slightly picky on what
-        //                 // lines we allow. We require that the line at least include a *@*.* word to
-        //                 // try and filter out lines that are not actually author lines
-        //                 let author_fallback_pattern =
-        //                     Regex::new(r"^.*?([\S]+@[\S]+.[\S]+).*?$").unwrap();
-        //                 let fallback_matches = author_fallback_pattern.is_match(first_line);
-
-        //                 if fallback_matches {
-        //                     Some(first_line)
-        //                 } else {
-        //                     // If none of our attempts have found an author, we drop back to the
-        //                     // attribute lookup. Eventually all of this logic should be removed and only
-        //                     // the attribute version should be supported
-        //                     Self::attr("authors", &self.resolved)
-        //                 }
-        //             }
-        //         })
-        //     })
-        // }
     }
 
     fn get_labels(&self) -> Option<&str> {
@@ -970,4 +974,102 @@ This is the new body"#;
         rfd.update_body(&new_content).unwrap();
         assert_eq!(expected, rfd.raw());
     }
+
+    #[test]
+    fn test_find_includes() {
+        let contents = r#":reproducible:
+:showtitle:
+:toc: left
+:numbered:
+:icons: font
+:state: prediscussion
+:revremark: State: {state}
+:docdatetime: 2019-01-04 19:26:06 UTC
+:localdatetime: 2019-01-04 19:26:06 UTC
+:labels: label1, label2
+
+= RFD 123 Place
+FirstName LastName <[email protected]>
+
+include::sub_doc1.adoc[]
+This is the new body
+
+include::sub_doc2.adoc[]"#;
+        let rfd = RfdAsciidoc::new(Cow::Borrowed(contents)).unwrap();
+        let includes = rfd.includes();
+
+        let expected = vec![
+            AsciidocInclude {
+                file: "sub_doc1.adoc",
+                replacement: "include::sub_doc1.adoc[]",
+            },
+            AsciidocInclude {
+                file: "sub_doc2.adoc",
+                replacement: "include::sub_doc2.adoc[]",
+            },
+        ];
+        assert_eq!(expected, includes);
+    }
+
+    #[test]
+    fn test_replace_include() {
+        let original = r#":reproducible:
+:showtitle:
+:toc: left
+:numbered:
+:icons: font
+:state: prediscussion
+:revremark: State: {state}
+:docdatetime: 2019-01-04 19:26:06 UTC
+:localdatetime: 2019-01-04 19:26:06 UTC
+:labels: label1, label2
+
+= RFD 123 Place
+FirstName LastName <[email protected]>
+
+include::sub_doc1.adoc[]
+This is the new body
+
+include::sub_doc1.adoc[]
+
+include::sub_doc2.adoc[]"#;
+        let include = AsciidocInclude {
+            file: "sub_doc1.adoc",
+            replacement: "include::sub_doc1.adoc[]",
+        };
+
+        let replacement_content = "Line 1
+Line 2
+Line 3";
+
+        let expected = r#":reproducible:
+:showtitle:
+:toc: left
+:numbered:
+:icons: font
+:state: prediscussion
+:revremark: State: {state}
+:docdatetime: 2019-01-04 19:26:06 UTC
+:localdatetime: 2019-01-04 19:26:06 UTC
+:labels: label1, label2
+
+= RFD 123 Place
+FirstName LastName <[email protected]>
+
+Line 1
+Line 2
+Line 3
+This is the new body
+
+Line 1
+Line 2
+Line 3
+
+include::sub_doc2.adoc[]"#;
+
+        assert_eq!(
+            expected,
+            include.perform_replacement(original, replacement_content)
+        );
+    }
 }

rfd-data/src/content/asciidoc.rs

@@ -54,6 +54,7 @@ impl ReposExt for Repos {
 pub trait ContentFileExt {
     fn is_empty(&self) -> bool;
     fn decode(&self) -> Result<Vec<u8>, DecodeError>;
+    fn to_text(&self) -> Option<String>;
 }
 
 impl ContentFileExt for ContentFile {
@@ -66,6 +67,12 @@ impl ContentFileExt for ContentFile {
             .decode(self.content.replace('\n', ""))
             .map(|data| data.trim().to_vec())
     }
+
+    fn to_text(&self) -> Option<String> {
+        self.decode()
+            .ok()
+            .and_then(|content| String::from_utf8(content).ok())
+    }
 }
 
 trait SliceExt {

rfd-github/src/ext.rs

@@ -541,18 +541,25 @@ impl GitHubRfdLocation {
         })
     }
 
-    /// Get a list of images that are store in this branch
-    pub async fn get_images(
+    /// Download the supporting documents that are within this RFD location
+    pub async fn download_supporting_documents(
         &self,
         client: &Client,
         rfd_number: &RfdNumber,
     ) -> Result<Vec<octorust::types::ContentFile>, GitHubError> {
         let dir = rfd_number.repo_path();
-        Self::get_images_internal(client, &self.owner, &self.repo, &self.commit, dir).await
+        Self::download_supporting_documents_internal(
+            client,
+            &self.owner,
+            &self.repo,
+            &self.commit,
+            dir,
+        )
+        .await
     }
 
     #[instrument(skip(client, dir))]
-    fn get_images_internal<'a>(
+    fn download_supporting_documents_internal<'a>(
         client: &'a Client,
         owner: &'a String,
         repo: &'a String,
@@ -575,13 +582,15 @@ impl GitHubRfdLocation {
                 tracing::info!(file.path, file.type_, "Processing git entry");
 
                 if file.type_ == "dir" {
-                    let images =
-                        Self::get_images_internal(client, owner, repo, ref_, file.path).await?;
+                    let images = Self::download_supporting_documents_internal(
+                        client, owner, repo, ref_, file.path,
+                    )
+                    .await?;
 
                     for image in images {
                         files.push(image)
                     }
-                } else if is_image(&file.name) {
+                } else {
                     let file = client
                         .repos()
                         .get_content_blob(owner, repo, ref_.0.as_str(), &file.path)

rfd-github/src/lib.rs

@@ -48,7 +48,7 @@ pub enum RenderableRfdError {
 
 #[derive(Debug, Clone)]
 pub struct RenderableRfd<'a> {
-    content: RfdContent<'a>,
+    pub content: RfdContent<'a>,
     render_id: Uuid,
 }
 
@@ -110,7 +110,8 @@ impl<'a> RenderableRfd<'a> {
     ) -> Result<RfdPdf, RfdOutputError> {
         match &self.content {
             RfdContent::Asciidoc(adoc) => {
-                self.download_images(client, number, branch).await?;
+                self.download_supporting_documents(client, number, branch)
+                    .await?;
 
                 let pdf = RenderedPdf::render(adoc, self.tmp_path()?).await?;
 
@@ -127,9 +128,9 @@ impl<'a> RenderableRfd<'a> {
     }
 
     /// Downloads images that are stored on the provided GitHub branch for the given RFD number.
-    /// These are stored locally so in a tmp directory for use by asciidoctor
+    /// These are stored locally in a tmp directory for use by asciidoctor
     #[instrument(skip(self, client), fields(storage_path = ?self.tmp_path()))]
-    async fn download_images(
+    async fn download_supporting_documents(
         &self,
         client: &Client,
         number: &RfdNumber,
@@ -138,20 +139,22 @@ impl<'a> RenderableRfd<'a> {
         let dir = number.repo_path();
         let storage_path = self.tmp_path()?;
 
-        let images = location.get_images(client, number).await?;
+        let documents = location
+            .download_supporting_documents(client, number)
+            .await?;
 
-        for image in images {
-            let image_path = storage_path.join(
-                image
+        for document in documents {
+            let document_path = storage_path.join(
+                document
                     .path
                     .replace(dir.trim_start_matches('/'), "")
                     .trim_start_matches('/'),
             );
 
-            let path = PathBuf::from(image_path);
-            write_file(&path, &decode_base64(&image.content)?).await?;
+            let path = PathBuf::from(document_path);
+            write_file(&path, &decode_base64(&document.content)?).await?;
 
-            tracing::info!(?path, "Wrote embedded image",);
+            tracing::info!(?path, "Wrote supporting document",);
         }
 
         Ok(())

rfd-processor/src/content/mod.rs

@@ -124,6 +124,13 @@ impl PersistedRfd {
         })
     }
 
+    pub fn set_content(&mut self, format: ContentFormat, content: &str) {
+        self.revision.content = content.to_string();
+        self.revision.content_format = format;
+
+        *self.needs_update.lock().unwrap() = true;
+    }
+
     pub fn update_discussion(&mut self, new_discussion_url: impl ToString) -> Result<(), RfdError> {
         let new_discussion_url = new_discussion_url.to_string();
 

rfd-processor/src/rfd.rs

@@ -29,7 +29,7 @@ impl RfdUpdateAction for CopyImagesToStorage {
 
         let images = update
             .location
-            .get_images(&ctx.github.client, &update.number)
+            .download_supporting_documents(&ctx.github.client, &update.number)
             .await
             .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?;
 

rfd-processor/src/updater/copy_images_to_storage.rs

@@ -5,6 +5,7 @@
 use async_trait::async_trait;
 use newtype_uuid::TypedUuid;
 use octorust::types::{LabelsData, PullRequestData, PullRequestSimple};
+use process_includes::ProcessIncludes;
 use rfd_data::content::RfdDocument;
 use rfd_github::{GitHubError, GitHubRfdUpdate};
 use rfd_model::RfdId;
@@ -31,6 +32,7 @@ mod copy_images_to_storage;
 mod create_pull_request;
 mod ensure_default_state;
 mod ensure_pr_state;
+mod process_includes;
 mod update_discussion_url;
 mod update_pdfs;
 mod update_pull_request;
@@ -86,6 +88,7 @@ impl TryFrom<&str> for BoxedAction {
             "CreatePullRequest" => Ok(Box::new(CreatePullRequest)),
             "UpdatePullRequest" => Ok(Box::new(UpdatePullRequest)),
             "UpdateDiscussionUrl" => Ok(Box::new(UpdateDiscussionUrl)),
+            "ProcessIncludes" => Ok(Box::new(ProcessIncludes)),
             "EnsureRfdWithPullRequestIsInValidState" => {
                 Ok(Box::new(EnsureRfdWithPullRequestIsInValidState))
             }