Skip to content

feat(delete,update,insert): Support build Insert with schema strings. #80

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 14 commits into from
Closed
14 changes: 7 additions & 7 deletions examples/async_insert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@ use std::time::{Duration, UNIX_EPOCH};

use serde::{Deserialize, Serialize};

use clickhouse::sql::Identifier;
use clickhouse::{error::Result, Client, Row};
use clickhouse::{error::Result, sql::Identifier, Client, Row};

// This example demonstrates how to use asynchronous inserts, avoiding client side batching of the incoming data.
// Suitable for ClickHouse Cloud, too. See https://clickhouse.com/docs/en/optimize/asynchronous-inserts
// This example demonstrates how to use asynchronous inserts, avoiding client
// side batching of the incoming data. Suitable for ClickHouse Cloud, too. See https://clickhouse.com/docs/en/optimize/asynchronous-inserts

#[derive(Debug, Serialize, Deserialize, Row)]
struct Event {
Expand Down Expand Up @@ -60,9 +59,10 @@ async fn main() -> Result<()> {
println!("{events:?}");
break;
}
// If you change the `wait_for_async_insert` setting to 1, this line will never be printed;
// however, without waiting, you will see it in the console output several times,
// as the data will remain in the server buffer for a bit before the flush happens
// If you change the `wait_for_async_insert` setting to 1, this line will never
// be printed; however, without waiting, you will see it in the console
// output several times, as the data will remain in the server buffer
// for a bit before the flush happens
println!("Waiting for async insert flush...");
tokio::time::sleep(Duration::from_millis(10)).await
}
Expand Down
14 changes: 7 additions & 7 deletions examples/clickhouse_cloud.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use clickhouse::sql::Identifier;
use clickhouse::Client;
use clickhouse::{sql::Identifier, Client};
use clickhouse_derive::Row;
use serde::{Deserialize, Serialize};
use std::env;

// This example requires three environment variables with your instance credentials to be set
// This example requires three environment variables with your instance
// credentials to be set
//
// - CLICKHOUSE_URL (e.g., https://myservice.clickhouse.cloud:8443)
// - CLICKHOUSE_USER
Expand All @@ -21,8 +21,8 @@ async fn main() -> clickhouse::error::Result<()> {
.with_user(read_env_var("CLICKHOUSE_USER"))
.with_password(read_env_var("CLICKHOUSE_PASSWORD"));

// `wait_end_of_query` is required in this case, as we want these DDLs to be executed
// on the entire Cloud cluster before we receive the response.
// `wait_end_of_query` is required in this case, as we want these DDLs to be
// executed on the entire Cloud cluster before we receive the response.
// See https://clickhouse.com/docs/en/interfaces/http/#response-buffering
client
.query("DROP TABLE IF EXISTS ?")
Expand All @@ -31,8 +31,8 @@ async fn main() -> clickhouse::error::Result<()> {
.execute()
.await?;

// Note that you could just use MergeTree with CH Cloud, and omit the `ON CLUSTER` clause.
// The same applies to other engines as well;
// Note that you could just use MergeTree with CH Cloud, and omit the `ON
// CLUSTER` clause. The same applies to other engines as well;
// e.g., ReplacingMergeTree will become SharedReplacingMergeTree and so on.
// See https://clickhouse.com/docs/en/cloud/reference/shared-merge-tree#enabling-sharedmergetree
client
Expand Down
6 changes: 4 additions & 2 deletions examples/clickhouse_settings.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use clickhouse::{error::Result, Client};

/// Besides [`Client::query`], it works similarly with [`Client::insert`] and [`Client::inserter`].
/// Besides [`Client::query`], it works similarly with [`Client::insert`] and
/// [`Client::inserter`].
#[tokio::main]
async fn main() -> Result<()> {
let client = Client::default()
Expand All @@ -16,7 +17,8 @@ async fn main() -> Result<()> {
.fetch_all::<u64>()
.await?;

// note that it prints the first 3 numbers only (because of the setting override)
// note that it prints the first 3 numbers only (because of the setting
// override)
println!("{numbers:?}");

Ok(())
Expand Down
7 changes: 4 additions & 3 deletions examples/custom_http_client.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use std::time::Duration;

use hyper_util::client::legacy::connect::HttpConnector;
use hyper_util::client::legacy::Client as HyperClient;
use hyper_util::rt::TokioExecutor;
use hyper_util::{
client::legacy::{connect::HttpConnector, Client as HyperClient},
rt::TokioExecutor,
};

use clickhouse::{error::Result, Client};

Expand Down
6 changes: 2 additions & 4 deletions examples/data_types_derive_containers.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
use rand::distributions::Alphanumeric;
use rand::Rng;
use rand::{distributions::Alphanumeric, Rng};

use clickhouse::sql::Identifier;
use clickhouse::{error::Result, Client};
use clickhouse::{error::Result, sql::Identifier, Client};

// This example covers derivation of container-like ClickHouse data types.
// See also:
Expand Down
12 changes: 6 additions & 6 deletions examples/data_types_new_json.rs
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
use clickhouse_derive::Row;
use serde::{Deserialize, Serialize};

use clickhouse::sql::Identifier;
use clickhouse::{error::Result, Client};
use clickhouse::{error::Result, sql::Identifier, Client};

// Requires ClickHouse 24.10+, as the `input_format_binary_read_json_as_string` and `output_format_binary_write_json_as_string` settings were added in that version.
// Inserting and selecting a row with a JSON column as a string.
// Requires ClickHouse 24.10+, as the `input_format_binary_read_json_as_string`
// and `output_format_binary_write_json_as_string` settings were added in that
// version. Inserting and selecting a row with a JSON column as a string.
// See also: https://clickhouse.com/docs/en/sql-reference/data-types/newjson

#[tokio::main]
async fn main() -> Result<()> {
let table_name = "chrs_data_types_new_json";
let client = Client::default()
.with_url("http://localhost:8123")
// All these settings can instead be applied on the query or insert level with the same `with_option` method.
// Enable new JSON type usage
// All these settings can instead be applied on the query or insert level with the same
// `with_option` method. Enable new JSON type usage
.with_option("allow_experimental_json_type", "1")
// Enable inserting JSON columns as a string
.with_option("input_format_binary_read_json_as_string", "1")
Expand Down
17 changes: 10 additions & 7 deletions examples/data_types_variant.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use clickhouse_derive::Row;
use serde::{Deserialize, Serialize};

use clickhouse::sql::Identifier;
use clickhouse::{error::Result, Client};
use clickhouse::{error::Result, sql::Identifier, Client};

// See also: https://clickhouse.com/docs/en/sql-reference/data-types/variant

Expand All @@ -11,8 +10,10 @@ async fn main() -> Result<()> {
let table_name = "chrs_data_types_variant";
let client = Client::default().with_url("http://localhost:8123");

// No matter the order of the definition on the Variant types in the DDL, this particular Variant will always be sorted as follows:
// Variant(Array(UInt16), Bool, FixedString(6), Float32, Float64, Int128, Int16, Int32, Int64, Int8, String, UInt128, UInt16, UInt32, UInt64, UInt8)
// No matter the order of the definition on the Variant types in the DDL, this
// particular Variant will always be sorted as follows:
// Variant(Array(UInt16), Bool, FixedString(6), Float32, Float64, Int128, Int16,
// Int32, Int64, Int8, String, UInt128, UInt16, UInt32, UInt64, UInt8)
client
.query(
"
Expand Down Expand Up @@ -135,9 +136,11 @@ fn get_rows() -> Vec<MyRow> {
}

// As the inner Variant types are _always_ sorted alphabetically,
// Rust enum variants should be defined in the _exactly_ same order as it is in the data type;
// their names are irrelevant, only the order of the types matters.
// This enum represents Variant(Array(UInt16), Bool, Date, FixedString(6), Float32, Float64, Int128, Int16, Int32, Int64, Int8, String, UInt128, UInt16, UInt32, UInt64, UInt8)
// Rust enum variants should be defined in the _exactly_ same order as it is in
// the data type; their names are irrelevant, only the order of the types
// matters. This enum represents Variant(Array(UInt16), Bool, Date,
// FixedString(6), Float32, Float64, Int128, Int16, Int32, Int64, Int8, String,
// UInt128, UInt16, UInt32, UInt64, UInt8)
#[derive(Debug, PartialEq, Serialize, Deserialize)]
enum MyRowVariant {
Array(Vec<i16>),
Expand Down
8 changes: 5 additions & 3 deletions examples/query_id.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use clickhouse::{error::Result, Client};
use uuid::Uuid;

/// Besides [`Client::query`], it works similarly with [`Client::insert`] and [`Client::inserter`].
/// Besides [`Client::query`], it works similarly with [`Client::insert`] and
/// [`Client::inserter`].
#[tokio::main]
async fn main() -> Result<()> {
let client = Client::default().with_url("http://localhost:8123");
Expand All @@ -15,8 +16,9 @@ async fn main() -> Result<()> {
.await?;
println!("Numbers: {numbers:?}");

// For the sake of this example, force flush the records into the system.query_log table,
// so we can immediately fetch the query information using the query_id
// For the sake of this example, force flush the records into the
// system.query_log table, so we can immediately fetch the query information
// using the query_id
client.query("SYSTEM FLUSH LOGS").execute().await?;

let logged_query = client
Expand Down
18 changes: 9 additions & 9 deletions examples/session_id.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@ use clickhouse_derive::Row;
use serde::{Deserialize, Serialize};
use uuid::Uuid;

use clickhouse::sql::Identifier;
use clickhouse::{error::Result, Client};
use clickhouse::{error::Result, sql::Identifier, Client};

/// Besides [`Client::with_option`], which will be applied for all requests,
/// `session_id` (and other settings) can be set separately for a particular `query`, `insert`,
/// or when using the `inserter` feature.
/// `session_id` (and other settings) can be set separately for a particular
/// `query`, `insert`, or when using the `inserter` feature.
///
/// This example uses temporary tables feature to demonstrate the `session_id` usage.
/// This example uses temporary tables feature to demonstrate the `session_id`
/// usage.
///
/// # Important
/// With clustered deployments, due to lack of "sticky sessions", you need to be connected
/// to a _particular cluster node_ in order to properly utilize this feature, cause, for example,
/// a round-robin load-balancer will not guarantee that the consequent requests will be processed
/// by the same ClickHouse node.
/// With clustered deployments, due to lack of "sticky sessions", you need to be
/// connected to a _particular cluster node_ in order to properly utilize this
/// feature, cause, for example, a round-robin load-balancer will not guarantee
/// that the consequent requests will be processed by the same ClickHouse node.
///
/// See also:
/// - https://clickhouse.com/docs/en/sql-reference/statements/create/table#temporary-tables
Expand Down
9 changes: 5 additions & 4 deletions src/headers.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use crate::{Authentication, ProductInfo};
use hyper::header::{AUTHORIZATION, USER_AGENT};
use hyper::http::request::Builder;
use std::collections::HashMap;
use std::env::consts::OS;
use hyper::{
header::{AUTHORIZATION, USER_AGENT},
http::request::Builder,
};
use std::{collections::HashMap, env::consts::OS};

fn get_user_agent(products_info: &[ProductInfo]) -> String {
// See https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates
Expand Down
19 changes: 14 additions & 5 deletions src/insert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ use tokio::{
};
use url::Url;

use crate::headers::{with_authentication, with_request_headers};
use crate::{
error::{Error, Result},
headers::{with_authentication, with_request_headers},
request_body::{ChunkSender, RequestBody},
response::Response,
row::{self, Row},
Expand Down Expand Up @@ -119,18 +119,27 @@ macro_rules! timeout {
}

impl<T> Insert<T> {
// TODO: remove Result
pub(crate) fn new_with_field_names(
client: &Client,
table: &str,
fields_names: Vec<String>,
) -> Result<Self> {
Insert::new_inner(client, table, fields_names.join(","))
}

pub(crate) fn new(client: &Client, table: &str) -> Result<Self>
where
T: Row,
{
let fields = row::join_column_names::<T>()
let fields_names = row::join_column_names::<T>()
.expect("the row type must be a struct or a wrapper around it");
Insert::new_inner(client, table, fields_names)
}

pub(crate) fn new_inner(client: &Client, table: &str, fields_names: String) -> Result<Self> {
// TODO: what about escaping a table name?
// https://clickhouse.com/docs/en/sql-reference/syntax#identifiers
let sql = format!("INSERT INTO {}({}) FORMAT RowBinary", table, fields);

let sql = format!("INSERT INTO {}({}) FORMAT RowBinary", table, fields_names);
Ok(Self {
state: InsertState::NotStarted {
client: Box::new(client.clone()),
Expand Down
11 changes: 10 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,8 @@ impl Client {

/// A JWT access token to authenticate with ClickHouse.
/// JWT token authentication is supported in ClickHouse Cloud only.
/// Should not be called after [`Client::with_user`] or [`Client::with_password`].
/// Should not be called after [`Client::with_user`] or
/// [`Client::with_password`].
///
/// # Panics
/// If called after [`Client::with_user`] or [`Client::with_password`].
Expand Down Expand Up @@ -302,6 +303,14 @@ impl Client {
insert::Insert::new(self, table)
}

pub fn insert_with_fields_name<T: Row>(
&self,
table: &str,
fields_names: Vec<String>,
) -> Result<insert::Insert<T>> {
insert::Insert::new_with_field_names(self, table, fields_names)
}

/// Creates an inserter to perform multiple INSERTs.
#[cfg(feature = "inserter")]
pub fn inserter<T: Row>(&self, table: &str) -> Result<inserter::Inserter<T>> {
Expand Down
4 changes: 4 additions & 0 deletions src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ impl Query {
self
}

pub fn bind_ref(&mut self, value: impl Bind) {
self.sql.bind_arg(value);
}

/// Executes the query.
pub async fn execute(self) -> Result<()> {
self.do_execute(false)?.finish().await
Expand Down
8 changes: 5 additions & 3 deletions src/rowbinary/ser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,11 @@ impl<B: BufMut> Serializer for &'_ mut RowBinarySerializer<B> {
value: &T,
) -> Result<()> {
// TODO:
// - Now this code implicitly allows using enums at the top level.
// However, instead of a more descriptive panic, it ends with a "not enough data." error.
// - Also, it produces an unclear message for a forgotten `serde_repr` (Enum8 and Enum16).
// - Now this code implicitly allows using enums at the top level. However,
// instead of a more descriptive panic, it ends with a "not enough data."
// error.
// - Also, it produces an unclear message for a forgotten `serde_repr` (Enum8
// and Enum16).
// See https://github.com/ClickHouse/clickhouse-rs/pull/170#discussion_r1848549636

// Max number of types in the Variant data type is 255
Expand Down
4 changes: 2 additions & 2 deletions src/sql/ser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,8 +329,8 @@ impl<'a, W: Write> Serializer for ParamSerializer<'a, W> {

#[inline]
fn serialize_str(self, value: &str) -> Result {
// ClickHouse expects strings in params to be unquoted until inside a nested type
// nested types go through serialize_seq which'll quote strings
// ClickHouse expects strings in params to be unquoted until inside a nested
// type nested types go through serialize_seq which'll quote strings
Ok(escape::escape(value, self.writer)?)
}

Expand Down
3 changes: 2 additions & 1 deletion tests/it/inserter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,8 @@ async fn keeps_client_options() {
assert_eq!(rows, vec!(row))
}

/// Similar to [`crate::insert::overrides_client_options`] with minor differences.
/// Similar to [`crate::insert::overrides_client_options`] with minor
/// differences.
#[tokio::test]
async fn overrides_client_options() {
let table_name = "inserter_overrides_client_options";
Expand Down
Loading
Loading