Makefile

SHELL := /bin/bash

build:
	docker-compose build

build-nc:
	docker-compose build --no-cache

build-progress:
	docker-compose build --no-cache --progress=plain

down:
	docker-compose down --volumes

run:
	make down && docker-compose up

run-scaled:
	make down && docker-compose up --scale spark-worker=3

run-d:
	make down && docker-compose up -d

stop:
	docker-compose stop

submit:
	docker exec da-spark-master spark-submit --master spark://spark-master:7077 --deploy-mode client /opt/spark/apps/$(app)

STACK_NAME=spark-kafka-jupyter_stack
COMPOSE_FILE=docker-compose-swarm.yml
REPLICAS=2

init-swarm:
	@echo Initializing Docker Swarm...
	docker swarm init

deploy:
	@echo "Deploying stack '${STACK_NAME}' using '${COMPOSE_FILE}'..."
	docker stack deploy -c ${COMPOSE_FILE} ${STACK_NAME}

scale-all:
	@echo "Scaling all services to ${REPLICAS} replicas..."
	docker service scale ${STACK_NAME}_postgres=${REPLICAS} \
	                     ${STACK_NAME}_kafka=${REPLICAS} \
	                     ${STACK_NAME}_spark_master=${REPLICAS} \
	                     ${STACK_NAME}_jupyter=${REPLICAS}


scale-postgres:
	@echo "Scaling postgres service to ${REPLICAS} replicas..."
	docker service scale ${STACK_NAME}_postgres=${REPLICAS}

scale-kafka:
	@echo "Scaling postgres service to ${REPLICAS} replicas..."
	docker service scale ${STACK_NAME}_kafka=${REPLICAS}

scale-spark:
	@echo "Scaling postgres service to ${REPLICAS} replicas..."
	docker service scale ${STACK_NAME}_spark_master=${REPLICAS}

scale-jupyter:
	@echo "Scaling postgres service to ${REPLICAS} replicas..."
	docker service scale ${STACK_NAME}_jupyter=${REPLICAS}

setup:
	@echo Building Docker Image...
	make build

	@echo Initializing Docker Swarm...
	docker swarm init
	
	@echo Starting Local Docker Registry...
	docker service create --name registeryamisha --publish published=5000,target=5000 registry:2

	@echo Checking Local Registry Service...
	docker service ls

	@echo Testing Local Registry Endpoint...
	curl http://localhost:5000/v2/


	@echo Tagging Spark Master Image for Local Registry...
	docker tag project5-spark-master:latest localhost:5000/spark-master:latest

	@echo Pushing Spark Master Image to Local Registry...
	docker push localhost:5000/spark-master:latest

	@echo Tagging Spark Worker Image for Local Registry...
	docker tag project5-spark-worker:latest localhost:5000/spark-worker:latest

	@echo Pushing Spark Worker Image to Local Registry...
	docker push localhost:5000/spark-worker:latest

	@echo Tagging PySpark Submit Image for Local Registry...
	docker tag project5-pyspark-submit:latest localhost:5000/pyspark-submit:latest

	@echo Pushing PySpark Submit Image to Local Registry...
	docker push localhost:5000/pyspark-submit:latest

	@echo Deploying Stack Using Docker Swarm...
	docker stack deploy -c docker-compose-swarm.yml spark-kafka-jupyter_stack

teardown:
	@echo "Removing stack and leaving swarm..."
	docker stack rm ${STACK_NAME}
	docker swarm leave --force
	docker service rm registeryamisha || echo "No local registry to remove."


check:
	@echo Checking Docker Stack Status...
	docker stack ls
	@echo Checking Docker Service Status...
	docker service ls
	@echo Opening Localhost Links in Default Browser...
	@if exist NUL ( \
		start http://localhost:8080 && \
		start http://localhost:8888 && \
		start http://localhost:5050 \
	) else ( \
		echo "Please open these links manually:" && \
		echo "Spark Master UI: http://localhost:8080" && \
		echo "Jupyter Notebook: http://localhost:8888" && \
		echo "PgAdmin: http://localhost:5050" \
	)



#Commands Description:
# build: Builds the Docker images using the docker-compose.yml file.
# build-nc: Builds the Docker images without using the cache.
# build-progress: Builds the Docker images without cache and shows plain build output.
# down: Stops and removes containers, networks, and volumes defined in the docker-compose.yml file.
# run: Stops existing containers and starts the services in the foreground.
# run-scaled: Stops existing containers and starts services with 3 replicas of the Spark worker.
# run-d: Stops existing containers and starts services in detached mode.
# stop: Stops running containers without removing them.
# submit: Executes a Spark job on the Spark master container using a provided application path.
# init-swarm: Initializes a Docker Swarm environment on the current machine.
# deploy: Deploys the Docker stack using the specified docker-compose-swarm.yml file.
# scale-all: Scales all services in the stack to a specified number of replicas.
# scale-postgres: Scales the Postgres service in the stack to the specified number of replicas.
# scale-kafka: Scales the Kafka service in the stack to the specified number of replicas.
# scale-spark: Scales the Spark master service in the stack to the specified number of replicas.
# scale-jupyter: Scales the Jupyter service in the stack to the specified number of replicas.
# setup: Sets up the environment by building images, initializing swarm, and deploying the stack.
# teardown: Removes the deployed stack and leaves the Docker Swarm environment.
# check: Checks the status of Docker stack and services, and suggests links to open in a browser.