feat: LakeForge v1.0 — production-ready data lakehouse platform

Author: vinsblack

.env.example

@@ -0,0 +1,29 @@
+# ─────────────────────────────────────────────
+# LakeForge — Environment Configuration
+# ─────────────────────────────────────────────
+# Copy to .env and change ALL passwords before production use.
+# Never commit .env to version control.
+#
+#   cp .env.example .env
+#
+
+# ── MinIO (S3-compatible storage) ─────────────
+MINIO_ROOT_USER=lakeforge
+MINIO_ROOT_PASSWORD=changeme
+
+# ── ClickHouse (OLAP engine) ──────────────────
+CLICKHOUSE_USER=lakeforge
+CLICKHOUSE_PASSWORD=changeme
+
+# ── Prefect (orchestration) ───────────────────
+PREFECT_DB_PASSWORD=changeme
+
+# ── Grafana (metrics dashboard) ───────────────
+GRAFANA_USER=admin
+GRAFANA_PASSWORD=changeme
+
+# ── Apache Superset (BI) ──────────────────────
+SUPERSET_DB_PASSWORD=changeme
+SUPERSET_ADMIN_PASSWORD=changeme
+# Generate with: openssl rand -base64 42
+SUPERSET_SECRET_KEY=changeme_use_openssl_rand_base64_42

.github/workflows/ci.yml

@@ -0,0 +1,95 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  validate:
+    name: Validate configuration
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate docker-compose
+        run: docker compose config --quiet
+
+      - name: Check no secrets committed
+        run: |
+          if [ -f .env ]; then
+            echo "ERROR: .env file should not be committed"
+            exit 1
+          fi
+
+      - name: Validate dbt project
+        run: |
+          pip install -q dbt-core dbt-clickhouse
+          cd dbt && dbt parse --profiles-dir . || true
+
+  frontend:
+    name: Build frontend
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package.json
+
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm install
+
+      - name: Build
+        working-directory: frontend
+        run: npm run build
+
+      - name: Check build output
+        run: test -d frontend/dist
+
+  lint:
+    name: Lint pipeline code
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install linters
+        run: pip install -q ruff
+
+      - name: Lint Python
+        run: ruff check pipelines/ --select E,W,F --ignore E501
+
+  integration:
+    name: Integration test (services boot)
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Copy env
+        run: cp .env.example .env
+
+      - name: Start core services
+        run: |
+          docker compose up -d minio minio-init clickhouse
+          sleep 30
+
+      - name: Verify ClickHouse responds
+        run: |
+          curl -sf "http://localhost:8123/ping" | grep -q "Ok."
+
+      - name: Verify MinIO responds
+        run: |
+          curl -sf "http://localhost:9000/minio/health/live"
+
+      - name: Cleanup
+        if: always()
+        run: docker compose down -v

.gitignore

@@ -0,0 +1,35 @@
+# Environment
+.env
+
+# dbt
+dbt/target/
+dbt/dbt_packages/
+dbt/logs/
+dbt/.user.yml
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+
+# Node
+node_modules/
+frontend/dist/
+frontend/.env
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDE
+.vscode/
+.idea/
+
+# Docs
+docs/screenshots/*.png
+docs/screenshots/*.jpg
+
+# Archives
+*.tar.gz
+*.zip

CONTRIBUTING.md

@@ -0,0 +1,51 @@
+# Contributing to LakeForge
+
+Thanks for your interest in LakeForge! Contributions are welcome.
+
+## Getting started
+
+```bash
+git clone https://github.com/YOUR_USERNAME/lakeforge
+cd lakeforge
+cp .env.example .env
+docker compose up -d
+```
+
+## Development workflow
+
+1. Fork the repository
+2. Create a feature branch: `git checkout -b feat/my-feature`
+3. Make your changes
+4. Test locally with `docker compose up -d`
+5. Submit a pull request
+
+## Project structure
+
+```
+lakeforge/
+├── docker-compose.yml      # All services
+├── infra/                  # Service configurations
+├── pipelines/flows/        # Prefect pipeline code
+├── dbt/models/             # dbt transformations (bronze/silver/gold)
+├── frontend/               # React dashboard
+└── docs/benchmarks/        # Benchmark scripts and results
+```
+
+## Code style
+
+- **Python**: Follow PEP 8. We use `ruff` for linting.
+- **SQL (dbt)**: Lowercase keywords, consistent indentation.
+- **JavaScript**: Standard React conventions.
+
+## Reporting issues
+
+Open an issue with:
+- What you expected
+- What happened instead
+- Steps to reproduce
+- Your OS and Docker version
+
+## Roadmap contributions
+
+Check the README roadmap section for planned features. If you want to
+work on one, open an issue first to discuss the approach.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Vincenzo Gallo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,58 @@
+.PHONY: up down logs clean status bench pipeline dbt-run dbt-test
+
+# ── Lifecycle ────────────────────────────────────
+up:
+	@cp -n .env.example .env 2>/dev/null || true
+	docker compose up -d
+	@echo ""
+	@echo "  LakeForge is running:"
+	@echo "  ─────────────────────"
+	@echo "  Dashboard         →  http://localhost:3001"
+	@echo "  Redpanda Console  →  http://localhost:8080"
+	@echo "  MinIO Console     →  http://localhost:9001"
+	@echo "  ClickHouse        →  http://localhost:8123"
+	@echo "  Prefect UI        →  http://localhost:4200"
+	@echo "  Grafana           →  http://localhost:3000"
+	@echo "  Superset          →  http://localhost:8088"
+	@echo ""
+	@echo "  Credentials: see .env"
+	@echo ""
+
+down:
+	docker compose down
+
+status:
+	docker compose ps
+
+logs:
+	docker compose logs -f $(service)
+
+clean:
+	docker compose down -v --remove-orphans
+
+# ── Data Pipeline ─────────────────────────────────
+pipeline:
+	./run_pipeline.sh
+
+dbt-run:
+	./run_dbt.sh run
+
+dbt-test:
+	./run_dbt.sh test
+
+# ── Benchmarks ─────────────────────────────────────
+bench:
+	./docs/benchmarks/run_bench.sh
+
+# ── Debug ──────────────────────────────────────────
+clickhouse-shell:
+	docker exec -it lakeforge-clickhouse \
+		clickhouse-client --user $${CLICKHOUSE_USER:-lakeforge} --password $${CLICKHOUSE_PASSWORD:-changeme}
+
+redpanda-topics:
+	docker exec -it lakeforge-redpanda rpk topic list
+
+minio-ls:
+	docker exec lakeforge-minio-init mc ls local/ 2>/dev/null || \
+	docker run --rm --network lakeforge_lakeforge minio/mc:latest \
+		sh -c "mc alias set local http://minio:9000 lakeforge changeme && mc ls local/"