Fix papercuts (#343)

* add log_format option to rfd-processor

* remove custom deserialization code

* fix extraneous diff when generating schema.rs

* remove invalid diesel.toml field

* update setup instructions

Author: pietroalbini

SETUP.md

@@ -39,18 +39,34 @@ emitted at:
 ### Installation
 
 Once all of the dependencies have been installed, database migrations will need to be run to prepare
-the database tables. These can be run using the [`diesel` cli ](https://diesel.rs/guides/getting-started).
-
-To run them:
+the database tables. These can be run using the `rfd-installer` tool:
 
 ```sh
 cd rfd-model
+V_ONLY=1 DATABASE_URL=<database-url> cargo run -p rfd-installer
 DATABASE_URL=<database-url> diesel migration run
 ```
 
 Replace `<database-url>` with the url to the Postgres instance that the API and processor will be
 configured to run against.
 
+### Running database migrations
+
+After the initial migration described above, any future database migration can
+be executed with the following commands:
+
+```sh
+cd rfd-model
+DATABASE_URL=<database-url> diesel migration run
+```
+
+> [!NOTE]
+>
+> If the generated `schema.rs` includes additional tables in its diff, it means
+> v-api added more tables of its own. You should exclude them in
+> `rfd-model/diesel.toml` and re-run migrations. The extraneous tables should
+> then disappear from `schema.rs`.
+
 ### Configuration
 
 Each part (the api and processor) has its own configuration file, and the respective application
@@ -90,4 +106,4 @@ cargo run -p rfd-cli --features local-dev
 
 The processor has multiple jobs that are able to be run, and configuration is only required for
 jobs that are going to be run. The `actions` key defines the jobs that should be run. By default
-all jobs are disabled. In this this mode the processor will only construct a database of RFDs.
+all jobs are disabled. In this this mode the processor will only construct a database of RFDs.

rfd-api/src/config.rs

@@ -6,22 +6,12 @@ use std::{collections::HashMap, path::PathBuf};
 
 use config::{Config, ConfigError, Environment, File};
 use rfd_data::content::RfdTemplate;
-use serde::{
-    de::{self, Visitor},
-    Deserialize, Deserializer,
-};
-use thiserror::Error;
+use serde::Deserialize;
 use v_api::config::{AsymmetricKey, AuthnProviders, JwtConfig};
 use v_model::schema_ext::MagicLinkMedium;
 
 use crate::server::SpecConfig;
 
-#[derive(Debug, Error)]
-pub enum AppConfigError {
-    #[error("Encountered invalid log format.")]
-    InvalidLogFormatVariant,
-}
-
 #[derive(Debug, Deserialize)]
 pub struct AppConfig {
     pub log_format: ServerLogFormat,
@@ -40,42 +30,13 @@ pub struct AppConfig {
     pub services: ServicesConfig,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "kebab-case")]
 pub enum ServerLogFormat {
     Json,
     Pretty,
 }
 
-impl<'de> Deserialize<'de> for ServerLogFormat {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        struct ExternalId;
-
-        impl<'de> Visitor<'de> for ExternalId {
-            type Value = ServerLogFormat;
-
-            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-                formatter.write_str("string")
-            }
-
-            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
-            where
-                E: de::Error,
-            {
-                match value {
-                    "json" => Ok(Self::Value::Json),
-                    "pretty" => Ok(Self::Value::Pretty),
-                    _ => Err(de::Error::custom(AppConfigError::InvalidLogFormatVariant)),
-                }
-            }
-        }
-
-        deserializer.deserialize_any(ExternalId)
-    }
-}
-
 #[derive(Debug, Default, Deserialize)]
 pub struct SearchConfig {
     pub host: String,

rfd-model/diesel-schema.patch

@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+diff --git a/src/schema.rs b/src/schema.rs
+index 8eb4b49..cbc1526 100644
+--- a/src/schema.rs
++++ b/src/schema.rs
+@@ -1,4 +1,6 @@
+-// @generated automatically by Diesel CLI.
++// This Source Code Form is subject to the terms of the Mozilla Public
++// License, v. 2.0. If a copy of the MPL was not distributed with this
++// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+ pub mod sql_types {
+     #[derive(diesel::sql_types::SqlType)]

rfd-model/diesel.toml

@@ -3,6 +3,24 @@
 
 [print_schema]
 file = "src/schema.rs"
+patch_file = "diesel-schema.patch"
 
-[migrations_directory]
-dir = "migrations"
+[print_schema.filter]
+except_tables = [
+    "access_groups",
+    "api_key",
+    "api_user",
+    "api_user_access_token",
+    "api_user_contact_email",
+    "api_user_provider",
+    "link_request",
+    "login_attempt",
+    "magic_link_attempt",
+    "magic_link_client",
+    "magic_link_client_redirect_uri",
+    "magic_link_client_secret",
+    "mapper",
+    "oauth_client",
+    "oauth_client_redirect_uri",
+    "oauth_client_secret",
+]

rfd-model/src/lib.rs

@@ -15,6 +15,7 @@ use std::fmt::Display;
 use thiserror::Error;
 
 pub mod db;
+#[rustfmt::skip]
 pub mod schema;
 pub mod schema_ext;
 pub mod storage;

rfd-model/src/schema.rs

@@ -91,4 +91,9 @@ diesel::joinable!(rfd_pdf -> rfd (rfd_id));
 diesel::joinable!(rfd_pdf -> rfd_revision (rfd_revision_id));
 diesel::joinable!(rfd_revision -> rfd (rfd_id));
 
-diesel::allow_tables_to_appear_in_same_query!(job, rfd, rfd_pdf, rfd_revision,);
+diesel::allow_tables_to_appear_in_same_query!(
+    job,
+    rfd,
+    rfd_pdf,
+    rfd_revision,
+);

rfd-processor/config.example.toml

@@ -1,3 +1,6 @@
+# Allowed values: json, pretty
+log_format = "json"
+
 # Controls if the processor should run
 processor_enabled = true
 
@@ -82,4 +85,4 @@ host = ""
 # API Key for reading and writing documents
 key = ""
 # Search index to store documents in
-index = ""
+index = ""

rfd-processor/src/main.rs

@@ -31,6 +31,8 @@ mod util;
 #[derive(Debug, Deserialize)]
 pub struct AppConfig {
     pub log_directory: Option<String>,
+    #[serde(default)]
+    pub log_format: LogFormat,
     pub processor_enabled: bool,
     pub processor_batch_size: i64,
     pub processor_interval: u64,
@@ -49,6 +51,15 @@ pub struct AppConfig {
     pub search_storage: Vec<SearchConfig>,
 }
 
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum LogFormat {
+    Pretty,
+    // The default value is used to avoid breaking old configuration files.
+    #[default]
+    Json,
+}
+
 #[derive(Debug, Error)]
 pub enum AppError {
     #[error("Job task failed")]
@@ -133,13 +144,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         NonBlocking::new(std::io::stdout())
     };
 
-    let _subscriber = tracing_subscriber::fmt()
+    let subscriber = tracing_subscriber::fmt()
         .with_file(false)
         .with_line_number(false)
         .with_env_filter(EnvFilter::from_default_env())
-        .with_writer(writer)
-        .json()
-        .init();
+        .with_writer(writer);
+
+    match config.log_format {
+        LogFormat::Pretty => subscriber.pretty().init(),
+        LogFormat::Json => subscriber.json().init(),
+    }
 
     let ctx = Arc::new(Context::new(Database::new(&config.database_url).await, &config).await?);