chore: clean up unused clickhouse-related code. (#19709)

Author: youngsofun

src/common/io/src/format_settings.rs

@@ -145,6 +145,9 @@ pub struct OutputFormatSettings {
     pub geometry_format: GeometryDataType,
     pub binary_format: BinaryDisplayFormat,
     pub http_json_result_mode: HttpHandlerDataFormat,
+    pub headers: u8,
+    pub json_compact: bool,
+    pub json_strings: bool,
 
     // used only in http handler response
     pub format_null_as_str: bool,
@@ -157,6 +160,9 @@ impl Default for OutputFormatSettings {
             geometry_format: GeometryDataType::default(),
             binary_format: BinaryDisplayFormat::Hex,
             http_json_result_mode: HttpHandlerDataFormat::Display,
+            headers: 0,
+            json_compact: false,
+            json_strings: false,
             format_null_as_str: false,
         }
     }

src/query/formats/src/clickhouse.rs

@@ -19,7 +19,6 @@ use databend_common_io::prelude::OutputFormatSettings;
 use databend_common_meta_app::principal::FileFormatParams;
 use databend_common_meta_app::principal::StageFileFormatType;
 
-use crate::clickhouse::ClickhouseSuffix;
 use crate::field_encoder::FieldEncoderCSV;
 use crate::output_format::CSVOutputFormat;
 use crate::output_format::JSONOutputFormat;
@@ -36,13 +35,12 @@ pub fn get_output_format(
     schema: TableSchemaRef,
     params: FileFormatParams,
     settings: OutputFormatSettings,
-    clickhouse: Option<ClickhouseSuffix>,
 ) -> Result<Box<dyn OutputFormat>> {
     let output: Box<dyn OutputFormat> = match &params {
         FileFormatParams::Csv(params) => {
             let field_encoder = FieldEncoderCSV::create_csv(params, settings.clone());
-            let headers = if let Some(options) = &clickhouse {
-                options.headers
+            let headers = if settings.headers != 0 {
+                settings.headers
             } else if params.output_header {
                 1
             } else {
@@ -57,8 +55,8 @@ pub fn get_output_format(
         }
         FileFormatParams::Text(params) => {
             let field_encoder = FieldEncoderCSV::create_tsv(params, settings.clone());
-            let headers = if let Some(options) = &clickhouse {
-                options.headers
+            let headers = if settings.headers != 0 {
+                settings.headers
             } else if params.output_header {
                 1
             } else {
@@ -72,9 +70,8 @@ pub fn get_output_format(
             ))
         }
         FileFormatParams::NdJson(_params) => {
-            let options = clickhouse.clone().unwrap_or_default();
-            let headers = options.headers;
-            match (options.is_strings, options.is_compact) {
+            let headers = settings.headers;
+            match (settings.json_strings, settings.json_compact) {
                 // string, compact, name, type
                 // not compact
                 (false, false) => Box::new(NDJSONOutputFormatBase::<false, false>::create(

src/query/formats/src/file_format_type.rs

@@ -17,15 +17,13 @@
 #![allow(clippy::collapsible_if)]
 
 mod binary;
-mod clickhouse;
 mod common_settings;
 mod delimiter;
 mod field_decoder;
 pub mod field_encoder;
 mod file_format_type;
 pub mod output_format;
 
-pub use clickhouse::ClickhouseFormatType;
 pub use delimiter::RecordDelimiter;
 pub use field_decoder::*;
 pub use file_format_type::FileFormatTypeExt;

src/query/formats/src/lib.rs

@@ -42,6 +42,9 @@ pub trait OutputFormat: Send {
 
 #[cfg(test)]
 mod test_utils {
+    use std::str::FromStr;
+
+    use databend_common_exception::ErrorCode;
     use databend_common_exception::Result;
     use databend_common_expression::Column;
     use databend_common_expression::DataBlock;
@@ -58,14 +61,21 @@ mod test_utils {
     use databend_common_expression::types::nullable::NullableColumn;
     use databend_common_expression::types::number::Float64Type;
     use databend_common_expression::types::number::Int32Type;
+    use databend_common_io::prelude::OutputFormatSettings;
     use databend_common_meta_app::principal::FileFormatParams;
+    use databend_common_meta_app::principal::StageFileFormatType;
     use databend_common_meta_app::tenant::Tenant;
     use databend_common_settings::Settings;
 
     use super::OutputFormat;
-    use crate::ClickhouseFormatType;
     use crate::get_output_format;
 
+    const SUFFIX_WITH_NAMES_AND_TYPES: &str = "withnamesandtypes";
+    const SUFFIX_WITH_NAMES: &str = "withnames";
+    const SUFFIX_COMPACT: &str = "compact";
+    const SUFFIX_STRINGS: &str = "strings";
+    const SUFFIX_EACHROW: &str = "eachrow";
+
     pub fn gen_schema_and_block(
         fields: Vec<TableField>,
         columns: Vec<Column>,
@@ -124,15 +134,62 @@ mod test_utils {
         gen_schema_and_block(fields, columns)
     }
 
+    fn try_remove_suffix<'a>(name: &'a str, suffix: &str) -> (&'a str, bool) {
+        if name.ends_with(suffix) {
+            (&name[0..(name.len() - suffix.len())], true)
+        } else {
+            (name, false)
+        }
+    }
+
+    fn parse_clickhouse_format(
+        name: &str,
+        settings: &mut OutputFormatSettings,
+    ) -> Result<StageFileFormatType> {
+        let lower = name.to_lowercase();
+        settings.headers = 0;
+        settings.json_compact = false;
+        settings.json_strings = false;
+
+        let (mut base, mut ok) = try_remove_suffix(&lower, SUFFIX_WITH_NAMES_AND_TYPES);
+        if ok {
+            settings.headers = 2;
+        } else {
+            (base, ok) = try_remove_suffix(base, SUFFIX_WITH_NAMES);
+            if ok {
+                settings.headers = 1;
+            }
+        }
+
+        if base.starts_with("json") {
+            let is_eachrow;
+            (base, is_eachrow) = try_remove_suffix(base, SUFFIX_EACHROW);
+            (base, settings.json_strings) = try_remove_suffix(base, SUFFIX_STRINGS);
+            (base, settings.json_compact) = try_remove_suffix(base, SUFFIX_COMPACT);
+            if base != "json" {
+                return Err(ErrorCode::UnknownFormat(name));
+            } else {
+                if !settings.json_compact && settings.headers != 0 {
+                    return Err(ErrorCode::UnknownFormat(name));
+                }
+                if is_eachrow {
+                    base = "ndjson"
+                }
+            }
+        }
+
+        StageFileFormatType::from_str(base).map_err(ErrorCode::UnknownFormat)
+    }
+
     pub fn get_output_format_clickhouse(
         format_name: &str,
         schema: TableSchemaRef,
     ) -> Result<Box<dyn OutputFormat>> {
-        let format = ClickhouseFormatType::parse_clickhouse_format(format_name)?;
-        let settings =
+        let mut settings =
             Settings::create(Tenant::new_literal("default")).get_output_format_settings()?;
-        let params = FileFormatParams::default_by_type(format.typ.clone())?;
-        get_output_format(schema, params, settings, Some(format.suffixes))
+        let format = parse_clickhouse_format(format_name, &mut settings)?;
+        let params = FileFormatParams::default_by_type(format)?;
+        get_output_format(schema, params, settings)
     }
 
     pub fn test_data_block(is_nullable: bool) -> Result<()> {

src/query/formats/src/output_format/mod.rs

@@ -183,7 +183,7 @@ mod test {
                 false,
             )?;
             let mut output_format =
-                get_output_format(schema, params, settings.get_output_format_settings()?, None)?;
+                get_output_format(schema, params, settings.get_output_format_settings()?)?;
             let buffer = output_format.serialize_block(&block)?;
 
             let csv_block = String::from_utf8(buffer)?;
@@ -242,7 +242,7 @@ mod test {
         let params =
             FileFormatParams::try_from_reader(FileFormatOptionsReader::from_map(options), false)?;
         let mut output_format =
-            get_output_format(schema, params, settings.get_output_format_settings()?, None)?;
+            get_output_format(schema, params, settings.get_output_format_settings()?)?;
 
         let prefix = String::from_utf8(output_format.serialize_prefix()?)?;
         let body = String::from_utf8(output_format.serialize_block(&block)?)?;
@@ -276,7 +276,7 @@ mod test {
             false,
         )?;
         let mut output_format =
-            get_output_format(schema, params, settings.get_output_format_settings()?, None)?;
+            get_output_format(schema, params, settings.get_output_format_settings()?)?;
 
         let buffer = output_format.serialize_block(&block)?;
 

src/query/formats/src/output_format/tsv.rs

@@ -116,6 +116,9 @@ impl Settings {
             geometry_format,
             binary_format,
             http_json_result_mode,
+            headers: 0,
+            json_compact: false,
+            json_strings: false,
             format_null_as_str,
         })
     }

src/query/settings/src/settings.rs

@@ -65,22 +65,13 @@ pub(crate) fn append_data_to_row_based_files(
     let format = &info.stage.file_format_params;
 
     let format_settings = ctx.get_output_format_settings()?;
-    let output_format = get_output_format(
-        schema.clone(),
-        format.clone(),
-        format_settings.clone(),
-        None,
-    )?;
+    let output_format = get_output_format(schema.clone(), format.clone(), format_settings.clone())?;
     let compression = info.stage.file_format_params.clone().compression();
     let prefix = output_format.serialize_prefix()?;
 
     pipeline.try_add_transformer(|| {
-        let output_format = get_output_format(
-            schema.clone(),
-            format.clone(),
-            format_settings.clone(),
-            None,
-        )?;
+        let output_format =
+            get_output_format(schema.clone(), format.clone(), format_settings.clone())?;
         Ok(SerializeProcessor::new(ctx.clone(), output_format))
     })?;
     pipeline.try_resize(1)?;