Merge remote-tracking branch 'upstream/main' into feat/pr-1820-refresh

Author: YQ-Wang

Cargo.lock

@@ -51,6 +51,13 @@ pub fn default_dfdaemon_config_path() -> PathBuf {
     crate::default_config_dir().join("dfdaemon.yaml")
 }
 
+/// Returns the default dynconfig path for dfdaemon, which is used when the
+/// manager is not configured.
+#[inline]
+pub fn default_dfdaemon_dynconfig_path() -> PathBuf {
+    crate::default_config_dir().join("dynconfig.yaml")
+}
+
 /// Returns the default log directory for dfdaemon.
 #[inline]
 pub fn default_dfdaemon_log_dir() -> PathBuf {
@@ -240,6 +247,18 @@ fn default_dynconfig_refresh_interval() -> Duration {
     Duration::from_secs(300)
 }
 
+/// Returns the default interval to refresh the local dynamic configuration.
+#[inline]
+pub fn default_local_dynconfig_refresh_interval() -> Duration {
+    Duration::from_secs(60)
+}
+
+/// Returns the default scheduler address for the local dynamic configuration.
+#[inline]
+pub fn default_local_dynconfig_scheduler_addr() -> String {
+    "dragonfly-scheduler:8002".to_string()
+}
+
 /// Returns the default port of the storage tcp server.
 #[inline]
 fn default_storage_server_tcp_port() -> u16 {
@@ -1621,9 +1640,11 @@ pub struct Config {
     #[validate]
     pub upload: Upload,
 
-    /// The manager configuration for dfdaemon.
+    /// The manager configuration for dfdaemon. If not configured, the dynamic
+    /// configuration is loaded from the local dynconfig file instead of being
+    /// fetched from the manager.
     #[validate]
-    pub manager: Manager,
+    pub manager: Option<Manager>,
 
     /// The scheduler configuration for dfdaemon.
     #[validate]

dragonfly-client-config/src/dfdaemon.rs

@@ -123,11 +123,13 @@ pub struct ContainerdRegistry {
 
     /// The flag to skip verifying the server's certificate, refer to
     /// https://github.com/containerd/containerd/blob/main/docs/hosts.md#bypass-tls-verification-example.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub skip_verify: Option<bool>,
 
     /// The Certificate Authority Certification, which can be set to a path or an array of paths each pointing
     /// to a ca file for use in authenticating with the registry namespace, refer to
     /// https://github.com/containerd/containerd/blob/main/docs/hosts.md#ca-field.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub ca: Option<Vec<String>>,
 }
 

dragonfly-client-config/src/dfinit.rs

@@ -66,10 +66,12 @@ anyhow.workspace = true
 bytes.workspace = true
 bytesize.workspace = true
 humantime.workspace = true
+humantime-serde.workspace = true
 uuid.workspace = true
 percent-encoding.workspace = true
 tokio-rustls.workspace = true
 serde_json.workspace = true
+serde_yaml.workspace = true
 fs2.workspace = true
 lazy_static.workspace = true
 futures.workspace = true

dragonfly-client/Cargo.toml

@@ -22,7 +22,7 @@ use dragonfly_client::dynconfig::Dynconfig;
 use dragonfly_client::gc::GC;
 use dragonfly_client::grpc::{
     dfdaemon_download::DfdaemonDownloadServer, dfdaemon_upload::DfdaemonUploadServer,
-    manager::ManagerClient, scheduler::SchedulerClient,
+    scheduler::SchedulerClient,
 };
 use dragonfly_client::health::Health;
 use dragonfly_client::proxy::Proxy;
@@ -84,6 +84,14 @@ struct Args {
     ]
     config: PathBuf,
 
+    #[arg(
+        long = "dynconfig",
+        default_value_os_t = dfdaemon::default_dfdaemon_dynconfig_path(),
+        env = "DFDAEMON_DYNCONFIG",
+        help = "Specify the dynconfig file to use when the manager address is not configured"
+    )]
+    dynconfig: PathBuf,
+
     #[arg(
         short = 'l',
         long,
@@ -202,22 +210,14 @@ async fn main() -> Result<(), anyhow::Error> {
     );
     let id_generator = Arc::new(id_generator);
 
-    // Initialize manager client.
-    let manager_client = ManagerClient::new(config.clone(), config.manager.addr.clone())
-        .await
-        .inspect_err(|err| {
-            error!("initialize manager client failed: {}", err);
-        })?;
-    let manager_client = Arc::new(manager_client);
-
     // Initialize channel for graceful shutdown.
     let shutdown = shutdown::Shutdown::default();
     let (shutdown_complete_tx, mut shutdown_complete_rx) = mpsc::unbounded_channel();
 
     // Initialize dynconfig server.
     let dynconfig = Dynconfig::new(
         config.clone(),
-        manager_client.clone(),
+        args.dynconfig,
         shutdown.clone(),
         shutdown_complete_tx.clone(),
     )