Skip to content

Commit 8d6f39e

Browse files
ethanyzhangethanyzhang
committed
Refresh clusters config
1 parent d9ac4f6 commit 8d6f39e

File tree

73 files changed

+3016
-16
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+3016
-16
lines changed

‎Makefile

+14-1
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,21 @@ upload:
5050

5151
sync:
5252
cp -r clusters/* ../presto-performance/presto-deploy-cluster/clusters
53-
rm -f ../presto-performance/presto-deploy-cluster/clusters/*.go
53+
rm -f ../presto-performance/presto-deploy-cluster/clusters/*.go \
54+
../presto-performance/presto-deploy-cluster/clusters/large/docker-stack-spark.yaml \
55+
../presto-performance/presto-deploy-cluster/clusters/large-ssd/docker-stack-spark.yaml \
56+
../presto-performance/presto-deploy-cluster/clusters/medium-ssd/docker-stack-spark.yaml \
57+
../presto-performance/presto-deploy-cluster/clusters/medium/docker-stack-spark.yaml \
58+
../presto-performance/presto-deploy-cluster/clusters/medium-spill/docker-stack-spark.yaml \
59+
../presto-performance/presto-deploy-cluster/clusters/xlarge/docker-stack-spark.yaml \
60+
../presto-performance/presto-deploy-cluster/clusters/2xlarge/docker-stack-spark.yaml
5461

5562
.PHONY: clusters
5663
clusters:
64+
@echo "Cleaning cluster directories..."
65+
@find clusters -name config.json -type f | sed 's/\/config.json$$//' | while read dir; do \
66+
echo "Cleaning $$dir..."; \
67+
find "$$dir" -type f ! -name config.json -delete; \
68+
done
69+
@echo "Generating cluster configurations..."
5770
./pbench genconfig -t clusters/templates -p clusters/params.json clusters

‎clusters/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.idea

‎clusters/2xlarge/docker-stack-spark.yaml

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
version: '3.8'
2+
3+
networks:
4+
presto-swarm:
5+
6+
services:
7+
coordinator:
8+
image: ${PRESTO_COORDINATOR_IMAGE}
9+
environment:
10+
- SPARK_MODE=master
11+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
12+
- SPARK_RPC_ENCRYPTION_ENABLED=no
13+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
14+
- SPARK_SSL_ENABLED=no
15+
- SPARK_USER=spark
16+
- SPARK_MASTER_HOST=0.0.0.0
17+
deploy:
18+
placement:
19+
constraints:
20+
- node.role == manager
21+
mode: replicated
22+
replicas: 1
23+
resources:
24+
limits:
25+
cpus: '63'
26+
memory: 488G
27+
reservations:
28+
cpus: '63'
29+
memory: 488G
30+
ports:
31+
- 80:8080
32+
- 4040:4040
33+
- 4041:4041
34+
- 6066:6066
35+
- 7077:7077
36+
- 8081:8081
37+
- 15002:15002
38+
- 18080:18080
39+
entrypoint:
40+
- /bin/bash
41+
- /opt/entrypoint-spark.sh
42+
volumes:
43+
- /home/centos/presto/log/spark/events:/opt/spark/events
44+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
45+
- /home/centos/presto/log/spark/work:/opt/spark/work
46+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
47+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
48+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
49+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
50+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
51+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
52+
- /usr/bin/telegraf:/usr/bin/telegraf
53+
- /etc/telegraf:/etc/telegraf
54+
- /var/run/docker.sock:/var/run/docker.sock
55+
networks:
56+
presto-swarm:
57+
58+
workers:
59+
image: ${PRESTO_WORKER_IMAGE}
60+
environment:
61+
- SPARK_MODE=worker
62+
- SPARK_MASTER_URL=spark://coordinator:7077
63+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
64+
- SPARK_RPC_ENCRYPTION_ENABLED=no
65+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
66+
- SPARK_SSL_ENABLED=no
67+
- SPARK_USER=spark
68+
depends_on:
69+
- coordinator
70+
deploy:
71+
placement:
72+
constraints:
73+
- node.role == worker
74+
mode: replicated
75+
replicas: ${NUMBER_OF_WORKERS}
76+
resources:
77+
limits:
78+
cpus: '63'
79+
memory: 488G
80+
reservations:
81+
cpus: '63'
82+
memory: 488G
83+
entrypoint:
84+
- /bin/bash
85+
- /opt/entrypoint-spark.sh
86+
volumes:
87+
- /home/centos/presto/log/spark/events:/opt/spark/events
88+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
89+
- /home/centos/presto/log/spark/work:/opt/spark/work
90+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
91+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
92+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
93+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
94+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
95+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
96+
- /usr/bin/telegraf:/usr/bin/telegraf
97+
- /etc/telegraf:/etc/telegraf
98+
- /var/run/docker.sock:/var/run/docker.sock
99+
networks:
100+
presto-swarm:
101+
cap_add:
102+
- PERFMON

‎clusters/2xlarge/workers/config-native.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ system-memory-gb=461
66
query-memory-gb=437
77
query.max-memory-per-node=437GB
88
memory-arbitrator-kind=SHARED
9-
task.max-drivers-per-task=64
9+
# task.max-drivers-per-task is no longer needed after https://github.com/prestodb/presto/pull/24642
1010

1111
runtime-metrics-collection-enabled=true
1212
system-mem-pushback-enabled=true

‎clusters/large-ssd/docker-stack-spark.yaml

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
version: '3.8'
2+
3+
networks:
4+
presto-swarm:
5+
6+
services:
7+
coordinator:
8+
image: ${PRESTO_COORDINATOR_IMAGE}
9+
environment:
10+
- SPARK_MODE=master
11+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
12+
- SPARK_RPC_ENCRYPTION_ENABLED=no
13+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
14+
- SPARK_SSL_ENABLED=no
15+
- SPARK_USER=spark
16+
- SPARK_MASTER_HOST=0.0.0.0
17+
deploy:
18+
placement:
19+
constraints:
20+
- node.role == manager
21+
mode: replicated
22+
replicas: 1
23+
resources:
24+
limits:
25+
cpus: '31'
26+
memory: 246G
27+
reservations:
28+
cpus: '31'
29+
memory: 246G
30+
ports:
31+
- 80:8080
32+
- 4040:4040
33+
- 4041:4041
34+
- 6066:6066
35+
- 7077:7077
36+
- 8081:8081
37+
- 15002:15002
38+
- 18080:18080
39+
entrypoint:
40+
- /bin/bash
41+
- /opt/entrypoint-spark.sh
42+
volumes:
43+
- /home/centos/presto/log/spark/events:/opt/spark/events
44+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
45+
- /home/centos/presto/log/spark/work:/opt/spark/work
46+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
47+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
48+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
49+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
50+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
51+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
52+
- /usr/bin/telegraf:/usr/bin/telegraf
53+
- /etc/telegraf:/etc/telegraf
54+
- /var/run/docker.sock:/var/run/docker.sock
55+
networks:
56+
presto-swarm:
57+
58+
workers:
59+
image: ${PRESTO_WORKER_IMAGE}
60+
environment:
61+
- SPARK_MODE=worker
62+
- SPARK_MASTER_URL=spark://coordinator:7077
63+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
64+
- SPARK_RPC_ENCRYPTION_ENABLED=no
65+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
66+
- SPARK_SSL_ENABLED=no
67+
- SPARK_USER=spark
68+
depends_on:
69+
- coordinator
70+
deploy:
71+
placement:
72+
constraints:
73+
- node.role == worker
74+
mode: replicated
75+
replicas: ${NUMBER_OF_WORKERS}
76+
resources:
77+
limits:
78+
cpus: '31'
79+
memory: 246G
80+
reservations:
81+
cpus: '31'
82+
memory: 246G
83+
entrypoint:
84+
- /bin/bash
85+
- /opt/entrypoint-spark.sh
86+
volumes:
87+
- /home/centos/presto/log/spark/events:/opt/spark/events
88+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
89+
- /home/centos/presto/log/spark/work:/opt/spark/work
90+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
91+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
92+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
93+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
94+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
95+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
96+
- /usr/bin/telegraf:/usr/bin/telegraf
97+
- /etc/telegraf:/etc/telegraf
98+
- /var/run/docker.sock:/var/run/docker.sock
99+
networks:
100+
presto-swarm:
101+
cap_add:
102+
- PERFMON

‎clusters/large-ssd/workers/config-native.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ system-memory-gb=231
66
query-memory-gb=219
77
query.max-memory-per-node=219GB
88
memory-arbitrator-kind=SHARED
9-
task.max-drivers-per-task=32
9+
# task.max-drivers-per-task is no longer needed after https://github.com/prestodb/presto/pull/24642
1010

1111
runtime-metrics-collection-enabled=true
1212
system-mem-pushback-enabled=true

‎clusters/large/docker-stack-spark.yaml

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
version: '3.8'
2+
3+
networks:
4+
presto-swarm:
5+
6+
services:
7+
coordinator:
8+
image: ${PRESTO_COORDINATOR_IMAGE}
9+
environment:
10+
- SPARK_MODE=master
11+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
12+
- SPARK_RPC_ENCRYPTION_ENABLED=no
13+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
14+
- SPARK_SSL_ENABLED=no
15+
- SPARK_USER=spark
16+
- SPARK_MASTER_HOST=0.0.0.0
17+
deploy:
18+
placement:
19+
constraints:
20+
- node.role == manager
21+
mode: replicated
22+
replicas: 1
23+
resources:
24+
limits:
25+
cpus: '31'
26+
memory: 246G
27+
reservations:
28+
cpus: '31'
29+
memory: 246G
30+
ports:
31+
- 80:8080
32+
- 4040:4040
33+
- 4041:4041
34+
- 6066:6066
35+
- 7077:7077
36+
- 8081:8081
37+
- 15002:15002
38+
- 18080:18080
39+
entrypoint:
40+
- /bin/bash
41+
- /opt/entrypoint-spark.sh
42+
volumes:
43+
- /home/centos/presto/log/spark/events:/opt/spark/events
44+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
45+
- /home/centos/presto/log/spark/work:/opt/spark/work
46+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
47+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
48+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
49+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
50+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
51+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
52+
- /usr/bin/telegraf:/usr/bin/telegraf
53+
- /etc/telegraf:/etc/telegraf
54+
- /var/run/docker.sock:/var/run/docker.sock
55+
networks:
56+
presto-swarm:
57+
58+
workers:
59+
image: ${PRESTO_WORKER_IMAGE}
60+
environment:
61+
- SPARK_MODE=worker
62+
- SPARK_MASTER_URL=spark://coordinator:7077
63+
- SPARK_RPC_AUTHENTICATION_ENABLED=no
64+
- SPARK_RPC_ENCRYPTION_ENABLED=no
65+
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
66+
- SPARK_SSL_ENABLED=no
67+
- SPARK_USER=spark
68+
depends_on:
69+
- coordinator
70+
deploy:
71+
placement:
72+
constraints:
73+
- node.role == worker
74+
mode: replicated
75+
replicas: ${NUMBER_OF_WORKERS}
76+
resources:
77+
limits:
78+
cpus: '31'
79+
memory: 246G
80+
reservations:
81+
cpus: '31'
82+
memory: 246G
83+
entrypoint:
84+
- /bin/bash
85+
- /opt/entrypoint-spark.sh
86+
volumes:
87+
- /home/centos/presto/log/spark/events:/opt/spark/events
88+
- /home/centos/presto/log/spark/logs:/opt/spark/logs
89+
- /home/centos/presto/log/spark/work:/opt/spark/work
90+
- /home/centos/presto/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
91+
- /home/centos/presto/spark/log4j2.properties:/opt/spark/conf/log4j2.properties
92+
- /home/centos/presto/spark/metrics.properties:/opt/spark/conf/metrics.properties
93+
- /home/centos/presto/spark/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
94+
- /home/centos/presto/spark/spark-native-defaults.conf:/opt/spark/conf/spark-native-defaults.conf
95+
- /home/centos/presto/entrypoint-spark.sh:/opt/entrypoint-spark.sh
96+
- /usr/bin/telegraf:/usr/bin/telegraf
97+
- /etc/telegraf:/etc/telegraf
98+
- /var/run/docker.sock:/var/run/docker.sock
99+
networks:
100+
presto-swarm:
101+
cap_add:
102+
- PERFMON

‎clusters/large/workers/config-native.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ system-memory-gb=231
66
query-memory-gb=219
77
query.max-memory-per-node=219GB
88
memory-arbitrator-kind=SHARED
9-
task.max-drivers-per-task=32
9+
# task.max-drivers-per-task is no longer needed after https://github.com/prestodb/presto/pull/24642
1010

1111
runtime-metrics-collection-enabled=true
1212
system-mem-pushback-enabled=true

0 commit comments

Comments
 (0)