refactor: remove old meta_client crate (apache#205)

* refactor: remove meta_client crate

* refactor: rename some structs and mods

* chore: code style polishing

* chore: rename meta_client_v2 to meta_client

* chore: address CR issues

* chore: update crate-deps.dot

* refactor: move ShardView to server/config.rs

Cargo.toml

@@ -33,7 +33,6 @@ members = [
     "grpcio",
     "interpreters",
     "meta_client",
-    "meta_client_v2",
     "proto",
     "query_engine",
     "server",
@@ -60,7 +59,7 @@ common_util = { path = "common_util" }
 df_operator = { path = "df_operator" }
 log = "0.4"
 logger = { path = "components/logger" }
-meta_client_v2 = { path = "meta_client_v2" }
+meta_client = { path = "meta_client" }
 query_engine = { path = "query_engine" }
 server = { path = "server" }
 table_engine = { path = "table_engine" }

cluster/Cargo.toml

@@ -14,10 +14,11 @@ catalog = { path = "../catalog" }
 common_types = { path = "../common_types" }
 common_util = { path = "../common_util" }
 log = "0.4"
-meta_client_v2 = { path = "../meta_client_v2" }
+meta_client = { path = "../meta_client" }
 rust-fsm = "0.6.0"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0.60"
 snafu = { version ="0.6.10", features = ["backtraces"]}
+table_engine = { path = "../table_engine" }
 tokio = { version = "1.0", features = ["full"] }

cluster/src/cluster_impl.rs

@@ -8,7 +8,7 @@ use std::{
 use async_trait::async_trait;
 use common_util::runtime::{JoinHandle, Runtime};
 use log::{error, info, warn};
-use meta_client_v2::{
+use meta_client::{
     types::{ActionCmd, GetTablesRequest},
     EventHandler, MetaClient,
 };
@@ -21,7 +21,7 @@ use tokio::{
 use crate::{
     config::ClusterConfig,
     table_manager::{ShardTableInfo, TableManager},
-    Cluster, MetaClientFailure, Result, StartMetaClient, TableManipulator,
+    Cluster, ClusterTopologyRef, MetaClientFailure, Result, StartMetaClient, TableManipulator,
 };
 
 /// ClusterImpl is an implementation of [`Cluster`] based [`MetaClient`].
@@ -190,7 +190,7 @@ impl Cluster for ClusterImpl {
             .await
             .context(StartMetaClient)?;
 
-        // start the backgroud loop for sending heartbeat.
+        // start the background loop for sending heartbeat.
         self.start_heartbeat_loop();
 
         info!("Cluster has started");
@@ -221,4 +221,8 @@ impl Cluster for ClusterImpl {
         info!("Cluster has stopped");
         Ok(())
     }
+
+    async fn fetch_topology(&self) -> Result<ClusterTopologyRef> {
+        todo!("fetch topology from the meta")
+    }
 }

cluster/src/config.rs

@@ -1,6 +1,6 @@
 // Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.
 
-use meta_client_v2::{meta_impl::MetaClientConfig, types::NodeMetaInfo};
+use meta_client::{meta_impl::MetaClientConfig, types::NodeMetaInfo};
 use serde_derive::Deserialize;
 
 #[derive(Default, Clone, Deserialize, Debug)]

cluster/src/lib.rs

@@ -1,15 +1,18 @@
 // Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.
 
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
 
 use async_trait::async_trait;
+use common_types::{schema::TIMESTAMP_COLUMN, table::TableId};
 use common_util::define_result;
-pub use meta_client_v2::types::{
+pub use meta_client::types::{
     AllocSchemaIdRequest, AllocSchemaIdResponse, AllocTableIdRequest, AllocTableIdResponse,
     DropTableRequest, GetTablesRequest,
 };
-use meta_client_v2::types::{ShardId, TableId};
+use meta_client::types::{ShardId, ShardInfo};
+use serde::Deserialize;
 use snafu::{Backtrace, Snafu};
+use table_engine::ANALYTIC_ENGINE_TYPE;
 
 pub mod cluster_impl;
 pub mod config;
@@ -19,13 +22,13 @@ mod table_manager;
 #[snafu(visibility = "pub")]
 pub enum Error {
     #[snafu(display("Build meta client failed, err:{}.", source))]
-    BuildMetaClient { source: meta_client_v2::Error },
+    BuildMetaClient { source: meta_client::Error },
 
     #[snafu(display("Meta client start failed, err:{}.", source))]
-    StartMetaClient { source: meta_client_v2::Error },
+    StartMetaClient { source: meta_client::Error },
 
     #[snafu(display("Meta client execute failed, err:{}.", source))]
-    MetaClientFailure { source: meta_client_v2::Error },
+    MetaClientFailure { source: meta_client::Error },
 
     #[snafu(display(
         "Shard not found in current node, shard_id:{}.\nBacktrace:\n{}",
@@ -40,9 +43,54 @@ pub enum Error {
 
 define_result!(Error);
 
-pub type ClusterRef = Arc<dyn Cluster + Send + Sync>;
+pub type TableName = String;
+pub type SchemaName = String;
 
+pub type ClusterRef = Arc<dyn Cluster + Send + Sync>;
 pub type TableManipulatorRef = Arc<dyn TableManipulator + Send + Sync>;
+pub type ClusterTopologyRef = Arc<ClusterTopology>;
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(default)]
+pub struct SchemaConfig {
+    pub auto_create_tables: bool,
+    pub default_engine_type: String,
+    pub default_timestamp_column_name: String,
+}
+
+impl Default for SchemaConfig {
+    fn default() -> Self {
+        Self {
+            auto_create_tables: false,
+            default_engine_type: ANALYTIC_ENGINE_TYPE.to_string(),
+            default_timestamp_column_name: TIMESTAMP_COLUMN.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Node {
+    pub addr: String,
+    pub port: u16,
+}
+
+#[derive(Debug, Clone)]
+pub struct TableNodeShards {
+    pub table_id: TableId,
+    pub node_shards: Vec<NodeShard>,
+}
+
+#[derive(Debug, Clone)]
+pub struct NodeShard {
+    pub shard: ShardInfo,
+    pub node: Node,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct ClusterTopology {
+    pub schema_tables: HashMap<SchemaName, HashMap<TableName, TableNodeShards>>,
+    pub schema_configs: HashMap<SchemaName, SchemaConfig>,
+}
 
 #[async_trait]
 pub trait TableManipulator {
@@ -66,5 +114,5 @@ pub trait TableManipulator {
 pub trait Cluster {
     async fn start(&self) -> Result<()>;
     async fn stop(&self) -> Result<()>;
-    // TODO: add more methods, such as provide the topology of the cluster.
+    async fn fetch_topology(&self) -> Result<ClusterTopologyRef>;
 }

cluster/src/table_manager.rs

@@ -5,15 +5,11 @@ use std::{
     sync::RwLock,
 };
 
-use meta_client_v2::types::{
-    CreateTableCmd, SchemaId, ShardId, ShardInfo, ShardTables, TableId, TableInfo,
-};
+use common_types::{schema::SchemaId, table::TableId};
+use meta_client::types::{CreateTableCmd, ShardId, ShardInfo, ShardTables, TableInfo};
 use snafu::OptionExt;
 
-use crate::{Result, ShardNotFound};
-
-pub type TableName = String;
-pub type SchemaName = String;
+use crate::{Result, SchemaName, ShardNotFound, TableName};
 
 #[derive(Debug, Clone)]
 #[allow(dead_code)]

common_types/src/lib.rs

@@ -19,6 +19,7 @@ pub mod row;
 #[cfg(feature = "arrow_deps")]
 pub mod schema;
 pub mod string;
+pub mod table;
 pub mod time;
 
 /// Sequence number

common_types/src/schema.rs

@@ -152,7 +152,8 @@ pub enum Error {
     },
 }
 
-// TODO(boyan)  make these constants configurable
+pub type SchemaId = u32;
+// TODO: make these constants configurable
 pub const TSID_COLUMN: &str = "tsid";
 pub const TIMESTAMP_COLUMN: &str = "timestamp";
 

common_types/src/table.rs

@@ -0,0 +1,3 @@
+// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.
+
+pub type TableId = u64;

docs/guides/src/dev/crate-deps.dot

@@ -37,7 +37,7 @@ digraph G {
 
     cluster -> analytic_engine
     cluster -> catalog
-    cluster -> meta_client_v2
+    cluster -> meta_client
 
     interpreters -> catalog
     interpreters -> sql
@@ -46,12 +46,6 @@ digraph G {
     interpreters -> query_engine
     interpreters -> arrow_deps
 
-    meta_client -> catalog
-    meta_client -> table_engine
-
-    meta_client_v2 -> catalog
-    meta_client_v2 -> table_engine
-
     query_engine -> arrow_deps
     query_engine -> sql
     query_engine -> table_engine