env.example

# Bento Flexprice Environment Configuration
# Copy this file to .env and fill in your actual values

# Flexprice API Configuration
# ===========================
# API host without protocol (e.g., api.cloud.flexprice.io or localhost:8080)
FLEXPRICE_API_HOST=api.cloud.flexprice.io

# Your Flexprice API key (get this from your Flexprice dashboard)
FLEXPRICE_API_KEY=your_api_key_here

# HTTP scheme (https for production, http for local development)
FLEXPRICE_SCHEME=https

# Kafka Configuration (Confluent Cloud )
# ================================================
# Kafka broker address (e.g., pkc-xxxxx.us-east-1.aws.confluent.cloud:9092)
FLEXPRICE_KAFKA_BROKERS=your-broker.confluent.cloud:9092

# Kafka topic to consume events from
FLEXPRICE_KAFKA_TOPIC=bento-testing

# SASL Authentication credentials
# Mechanism: PLAIN (Confluent Cloud) or SCRAM-SHA-512 (AWS MSK)
FLEXPRICE_KAFKA_SASL_MECHANISM=PLAIN
FLEXPRICE_KAFKA_SASL_USER=your_kafka_api_key
FLEXPRICE_KAFKA_SASL_PASSWORD=your_kafka_api_secret

# Consumer group name (change to reset offset position)
FLEXPRICE_KAFKA_CONSUMER_GROUP=bento-flexprice-v0

# Client ID for identification
FLEXPRICE_KAFKA_CLIENT_ID=bento-flexprice-collector

# Kafka Consumer Performance Tuning
# =================================
# Fetch buffer capacity (messages to pre-fetch)
# Increase for higher throughput (default: 1024)
KAFKA_FETCH_BUFFER_CAP=1024

# Checkpoint limit (messages to read before forced commit)
# Increase to allow larger batches and parallel processing
# IMPORTANT: Should be significantly larger than batch size * max_in_flight
# Default: 5000
KAFKA_CHECKPOINT_LIMIT=5000

# Logging Configuration
# =====================
# Log level: DEBUG, INFO, WARN, ERROR
LOG_LEVEL=INFO

# AWS S3 Bucket Output configuration
# ===================================================
FLEXPRICE_S3_BUCKET_NAME="flexprice-dev-testing"

# AWS S3 Bucket output path key prefix
# ===================================================
# FLEXPRICE_S3_KEY_PREFIX="data"

# S3 Write timeout (increase if seeing 'context deadline exceeded' errors)
# Default: 30s
S3_WRITE_TIMEOUT=30s


# Flexprice Output Kafka Configuration (for custom-kafka-to-flexprice-kafka.yaml)
# ================================================================================
# Tenant ID for Flexprice events (required)
FLEXPRICE_TENANT_ID=your-tenant-id

# Environment ID for Flexprice events (optional but recommended)
FLEXPRICE_ENVIRONMENT_ID=your-environment-id

# Kafka broker address for output (e.g., MSK cluster endpoint)
FLEXPRICE_OUTPUT_KAFKA_BROKERS=your-msk-broker.aws.amazonaws.com:9092

# Kafka topic to write transformed events to
FLEXPRICE_OUTPUT_KAFKA_TOPIC=bento-flexprice-output

# TLS enabled for output Kafka (default: true)
FLEXPRICE_OUTPUT_KAFKA_TLS_ENABLED=true

# SASL Authentication for output Kafka
# Mechanism: PLAIN (Confluent Cloud) or SCRAM-SHA-512 (AWS MSK)
FLEXPRICE_OUTPUT_KAFKA_SASL_MECHANISM=PLAIN
FLEXPRICE_OUTPUT_KAFKA_SASL_USER=your_output_kafka_user
FLEXPRICE_OUTPUT_KAFKA_SASL_PASSWORD=your_output_kafka_password

# Client ID for output Kafka producer
FLEXPRICE_OUTPUT_KAFKA_CLIENT_ID=bento-billing-flexprice-output

# Batching configuration for output Kafka
FLEXPRICE_OUTPUT_KAFKA_BATCH_SIZE=10
FLEXPRICE_OUTPUT_KAFKA_BATCH_PERIOD=2s

# Maximum concurrent requests for output Kafka
FLEXPRICE_OUTPUT_KAFKA_MAX_IN_FLIGHT=10

# Kafka DLQ Configuration (Dead Letter Queue)
# =============================================
# Topic for failed messages after all retries are exhausted
# Used by ClickHouse collectors (v1 and v2) for both ClickHouse and Kafka output failures
FLEXPRICE_KAFKA_DLQ_TOPIC=bento-clickhouse-dlq

# ClickHouse Configuration (for custom-kafka-to-flexprice-clickhouse.yaml)
# =========================================================================
# ClickHouse server address with port (native protocol uses port 9000)
# Example: clickhouse.example.com:9000
FLEXPRICE_CLICKHOUSE_ADDRESS=your-clickhouse-host:9000

# ClickHouse authentication
FLEXPRICE_CLICKHOUSE_USERNAME=default
FLEXPRICE_CLICKHOUSE_PASSWORD=your-clickhouse-password

# ClickHouse database name
FLEXPRICE_CLICKHOUSE_DATABASE=flexprice

# TLS/SSL configuration (set to true if using secure connection)
FLEXPRICE_CLICKHOUSE_TLS=false

# Complete ClickHouse DSN (Data Source Name) - recommended approach
# Format: clickhouse://username:password@host:port/database
# Note: Password must be URL-encoded if it contains special characters (?, ;, !, etc.)
# Example encoding: python3 -c "import urllib.parse; print(urllib.parse.quote('your-password', safe=''))"
FLEXPRICE_CLICKHOUSE_DSN=clickhouse://default:your-url-encoded-password@your-clickhouse-host:9000/flexprice

# Tenant and Environment IDs (required for proper data filtering)
# These should match your Flexprice tenant and environment
FLEXPRICE_TENANT_ID=tenant_your_tenant_id_here
FLEXPRICE_ENVIRONMENT_ID=env_your_environment_id_here

# ClickHouse Batching Configuration
# =================================
# Batch size for inserts (default: 1000, use 500 if traffic is low)
CLICKHOUSE_BATCH_COUNT=1000

# Flush period - flush every N seconds even if batch not full (default: 1s)
CLICKHOUSE_BATCH_PERIOD=1s

# Maximum parallel async inserts to ClickHouse (default: 8)
CLICKHOUSE_MAX_IN_FLIGHT=8

# AWS ECR Configuration (for Docker image publishing)
# ===================================================
# ECR registry URL (e.g., 123456789012.dkr.ecr.us-east-1.amazonaws.com)
ECR_REGISTRY=your-account-id.dkr.ecr.us-east-1.amazonaws.com

# ECR repository name
ECR_REPOSITORY=bento-collector

# AWS region for ECR (optional, defaults to us-east-1)
AWS_REGION=us-east-1