diff --git a/cmd/iceberg/main.go b/cmd/iceberg/main.go index 84d04fb7b..7ebe8a403 100644 --- a/cmd/iceberg/main.go +++ b/cmd/iceberg/main.go @@ -19,9 +19,11 @@ package main import ( "context" + "crypto/tls" "errors" "fmt" "log" + "net/url" "os" "strings" @@ -83,7 +85,9 @@ Options: --description TEXT specify a description for the namespace --location-uri TEXT specify a location URI for the namespace --schema JSON specify table schema in json (for create table use only) - Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}]` + Ex: [{"name":"id","type":"int","required":false,"doc":"unique id"}] + --rest-config TEXT specify the REST configuration to use + Ex: sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue` type Config struct { List bool `docopt:"list"` @@ -125,6 +129,7 @@ type Config struct { Description string `docopt:"--description"` LocationURI string `docopt:"--location-uri"` SchemaStr string `docopt:"--schema"` + RestConfig string `docopt:"--rest-config"` } func main() { @@ -162,6 +167,29 @@ func main() { if len(cfg.Cred) > 0 { opts = append(opts, rest.WithCredential(cfg.Cred)) } + if len(cfg.RestConfig) > 0 { + restCfg := config.ParseRestConfig(cfg.RestConfig) + + if strings.ToLower(restCfg["sigv4-enabled"]) == "true" { + opts = append(opts, rest.WithSigV4()) + } + + if len(restCfg["sigv4-region"]) > 0 && len(restCfg["sigv4-service"]) > 0 { + opts = append(opts, rest.WithSigV4RegionSvc(restCfg["sigv4-region"], restCfg["sigv4-service"])) + } + + if len(restCfg["auth-url"]) > 0 { + authUri, err := url.Parse(restCfg["auth-url"]) + if err != nil { + log.Fatal(err) + } + opts = append(opts, rest.WithAuthURI(authUri)) + } + + if restCfg["tls-skip-verify"] == "true" { + opts = append(opts, rest.WithTLSConfig(&tls.Config{InsecureSkipVerify: true})) + } + } if len(cfg.Warehouse) > 0 { opts = append(opts, rest.WithWarehouseLocation(cfg.Warehouse)) @@ -475,4 +503,7 @@ func mergeConf(fileConf *config.CatalogConfig, resConfig *Config) { if len(resConfig.Warehouse) == 0 { resConfig.Warehouse = fileConf.Warehouse } + if len(resConfig.RestConfig) == 0 { + resConfig.RestConfig = fileConf.RestConfig + } } diff --git a/config/config.go b/config/config.go index 3d645ddee..02a45df74 100644 --- a/config/config.go +++ b/config/config.go @@ -20,6 +20,7 @@ package config import ( "os" "path/filepath" + "strings" "gopkg.in/yaml.v3" ) @@ -41,6 +42,7 @@ type CatalogConfig struct { Output string `yaml:"output"` Credential string `yaml:"credential"` Warehouse string `yaml:"warehouse"` + RestConfig string `yaml:"rest-config"` } func LoadConfig(configPath string) []byte { @@ -76,6 +78,21 @@ func ParseConfig(file []byte, catalogName string) *CatalogConfig { return &res } +func ParseRestConfig(restConfig string) map[string]string { + result := make(map[string]string) + if restConfig == "" { + return result + } + + pairs := strings.Split(restConfig, ",") + for _, pair := range pairs { + if kv := strings.SplitN(pair, "=", 2); len(kv) == 2 { + result[kv[0]] = kv[1] + } + } + return result +} + func fromConfigFiles() Config { dir := os.Getenv("GOICEBERG_HOME") if dir != "" { diff --git a/config/config_test.go b/config/config_test.go index c176b6382..ebd6fd51e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -78,6 +78,27 @@ catalog: Warehouse: "catalog_name", }, }, + // catalog with rest-config + { + []byte(` +catalog: + rest-catalog: + type: rest + uri: https://glue.us-east-1.amazonaws.com/iceberg + output: json + credential: client-id:client-secret + warehouse: 123456789012 + rest-config: auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false +`), "rest-catalog", + &CatalogConfig{ + CatalogType: "rest", + URI: "https://glue.us-east-1.amazonaws.com/iceberg", + Output: "json", + Credential: "client-id:client-secret", + Warehouse: "123456789012", + RestConfig: "auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false", + }, + }, } func TestParseConfig(t *testing.T) { @@ -87,3 +108,26 @@ func TestParseConfig(t *testing.T) { assert.Equal(t, tt.expected, actual) } } + +func TestRestCatalogConfig(t *testing.T) { + yamlData := []byte(` +catalog: + default: + type: rest + uri: https://glue.us-east-1.amazonaws.com/iceberg + output: json + rest-config: auth-url=https://auth.example.com,sigv4-enabled=true,sigv4-region=us-east-1,sigv4-service=glue,tls-skip-verify=false +`) + + config := ParseConfig(yamlData, "default") + assert.NotNil(t, config) + restConfig := ParseRestConfig(config.RestConfig) + assert.NotEmpty(t, restConfig) + + // Test RestCatalog config fields + assert.Equal(t, "https://auth.example.com", restConfig["auth-url"]) + assert.Equal(t, "true", restConfig["sigv4-enabled"]) + assert.Equal(t, "us-east-1", restConfig["sigv4-region"]) + assert.Equal(t, "glue", restConfig["sigv4-service"]) + assert.Equal(t, "false", restConfig["tls-skip-verify"]) +}