reusable-spark-pipeline/Tiltfile at main · JeffreyRoss/reusable-spark-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
allow_k8s_contexts('local')
docker_prune_settings(num_builds=1, keep_recent=1)

aissemble_version = '1.11.0-SNAPSHOT'

build_args = { 'DOCKER_BASELINE_REPO_ID': 'ghcr.io/',
               'VERSION_AISSEMBLE': aissemble_version}

# Kafka
yaml = helm(
    'reusable-spark-pipeline-deploy/src/main/resources/apps/kafka-cluster',
    values=['reusable-spark-pipeline-deploy/src/main/resources/apps/kafka-cluster/values.yaml',
        'reusable-spark-pipeline-deploy/src/main/resources/apps/kafka-cluster/values-dev.yaml']
)
k8s_yaml(yaml)

# Add deployment resources here

k8s_kind('SparkApplication', image_json_path='{.spec.image}')


yaml = local('helm template oci://ghcr.io/boozallen/aissemble-spark-application-chart --version %s --values reusable-spark-pipeline-pipelines/spark-pipeline/src/main/resources/apps/spark-pipeline-base-values.yaml,reusable-spark-pipeline-pipelines/spark-pipeline/src/main/resources/apps/spark-pipeline-dev-values.yaml' % aissemble_version)
k8s_yaml(yaml)
k8s_resource('spark-pipeline', port_forwards=[port_forward(4747, 4747, 'debug')], auto_init=False, trigger_mode=TRIGGER_MODE_MANUAL)

yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/spark-infrastructure',
   name='spark-infrastructure',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/spark-infrastructure/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/spark-infrastructure/values-dev.yaml']
)
k8s_yaml(yaml)
yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/metadata',
   name='metadata',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/metadata/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/metadata/values-dev.yaml']
)
k8s_yaml(yaml)
yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/spark-operator',
   name='spark-operator',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/spark-operator/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/spark-operator/values-dev.yaml']
)
k8s_yaml(yaml)
yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/policy-decision-point',
   name='policy-decision-point',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/policy-decision-point/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/policy-decision-point/values-dev.yaml']
)
k8s_yaml(yaml)

yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/s3-local',
   name='s3-local',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/s3-local/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/s3-local/values-dev.yaml']
)
k8s_yaml(yaml)
yaml = helm(
   'reusable-spark-pipeline-deploy/src/main/resources/apps/pipeline-invocation-service',
   name='pipeline-invocation-service',
   values=['reusable-spark-pipeline-deploy/src/main/resources/apps/pipeline-invocation-service/values.yaml',
       'reusable-spark-pipeline-deploy/src/main/resources/apps/pipeline-invocation-service/values-dev.yaml']
)
k8s_yaml(yaml)