Skip to content

Commit 8f7060f

Browse files
sitolesitole
authored
Clean up Nomad job artifact handling (#2312)
* Fix artifact download mode for template-manager and clean-nfs-cache Nomad jobs Same fix as 5c599a2 for orchestrator: set mode=file and explicit destination to prevent Nomad from switching to folder mode when multiple objects share a prefix. * Propagate artifact source URL from Terraform for template-manager and clean-nfs-cache Move env-conditional version query param logic out of HCL job templates into Terraform, matching the orchestrator pattern. Removes checksum options block from template-manager and external checksum data source from clean-nfs-cache in favor of GCS object generation. * Move clean-nfs-cache job setup into main.tf, remove separate file * Use object native checksum, always use version Removed bash script and replaced with native object version taken from object data source. Non-dev environments are now using object version as well. * Fix review comments: remove checksum, fix Terraform syntax errors - Remove apm_plugin_checksum from autoscaler module; rely on ?etag=/?version= query param for artifact pinning (consistent with orchestrator/template-manager) - Fix locals. → local. typo in AWS autoscaler module call - Remove obsolete template_manager_checksum argument from AWS template_manager module call - Inline autoscaler artifact source URL directly (was a local with empty-string fallback) - Fix clean_nfs_cache_artifact_source to always include ?version= (remove dev-only conditional)
1 parent 6b63ea4 commit 8f7060f

11 files changed

Lines changed: 58 additions & 118 deletions

File tree

‎iac/modules/job-template-manager-autoscaler/jobs/nomad-autoscaler.hcl‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ job "nomad-autoscaler" {
2727
source = "${apm_plugin_artifact_source}"
2828
destination = "local/plugins/nomad-nodepool-apm"
2929
mode = "file"
30-
options {
31-
checksum = "md5:${apm_plugin_checksum}"
32-
}
3330
}
3431

3532
template {

‎iac/modules/job-template-manager-autoscaler/main.tf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,5 @@ resource "nomad_job" "nomad_nodepool_apm" {
44
autoscaler_version = var.autoscaler_version
55
nomad_token = var.nomad_token
66
apm_plugin_artifact_source = var.apm_plugin_artifact_source
7-
apm_plugin_checksum = var.apm_plugin_checksum
87
})
98
}

‎iac/modules/job-template-manager-autoscaler/variables.tf‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,3 @@ variable "apm_plugin_artifact_source" {
1818
description = "Full artifact URL for the nomad-nodepool-apm plugin"
1919
}
2020

21-
variable "apm_plugin_checksum" {
22-
type = string
23-
description = "Hex checksum of the nomad-nodepool-apm plugin"
24-
}

‎iac/modules/job-template-manager/jobs/template-manager.hcl‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,8 @@ job "template-manager" {
142142

143143
artifact {
144144
source = "${artifact_source}"
145-
%{ if template_manager_checksum != "" }
146-
options {
147-
checksum = "md5:${template_manager_checksum}"
148-
}
149-
%{ endif }
145+
destination = "local/template-manager"
146+
mode = "file"
150147
}
151148
}
152149
}

‎iac/modules/job-template-manager/main.tf‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ resource "nomad_job" "template_manager" {
2929

3030
api_secret = var.api_secret
3131
artifact_source = var.artifact_source
32-
template_manager_checksum = var.template_manager_checksum
3332
template_bucket_name = var.template_bucket_name
3433
build_cache_bucket_name = var.build_cache_bucket_name
3534
otel_collector_grpc_endpoint = var.otel_collector_grpc_endpoint

‎iac/modules/job-template-manager/variables.tf‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,6 @@ variable "artifact_source" {
6262
description = "Full artifact URL for the template-manager binary (e.g. gcs::https://... or s3::https://...)"
6363
}
6464

65-
variable "template_manager_checksum" {
66-
type = string
67-
description = "Hex checksum of the template-manager binary"
68-
}
69-
7065
variable "api_secret" {
7166
type = string
7267
sensitive = true

‎iac/provider-aws/nomad/main.tf‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ resource "random_password" "volume_token_key" {
99

1010
locals {
1111
clickhouse_connection_string = var.clickhouse_cluster_size > 0 ? "clickhouse://${var.clickhouse_username}:${var.clickhouse_password}@clickhouse.service.consul:${var.clickhouse_port}/${var.clickhouse_database}" : ""
12-
orchestrator_artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/orchestrator?etag=${data.aws_s3_object.orchestrator.etag}"
1312
}
1413

1514
data "aws_ecr_image" "api" {
@@ -155,6 +154,10 @@ data "aws_s3_object" "orchestrator" {
155154
key = "orchestrator"
156155
}
157156

157+
locals {
158+
orchestrator_artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/orchestrator?etag=${data.aws_s3_object.orchestrator.etag}"
159+
}
160+
158161
module "orchestrator" {
159162
source = "../../modules/job-orchestrator"
160163

@@ -193,6 +196,10 @@ data "aws_s3_object" "template_manager" {
193196
key = "template-manager"
194197
}
195198

199+
locals {
200+
template_manager_artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/template-manager?etag=${data.aws_s3_object.template_manager.etag}"
201+
}
202+
196203
module "template_manager" {
197204
source = "../../modules/job-template-manager"
198205

@@ -211,8 +218,7 @@ module "template_manager" {
211218
domain_name = var.domain_name
212219

213220
api_secret = var.api_secret
214-
artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/template-manager"
215-
template_manager_checksum = ""
221+
artifact_source = local.template_manager_artifact_source
216222
template_bucket_name = var.template_bucket_name
217223
build_cache_bucket_name = var.build_cache_bucket_name
218224
otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
@@ -236,8 +242,7 @@ module "template_manager_autoscaler" {
236242

237243
node_pool = var.api_node_pool
238244
nomad_token = var.nomad_acl_token
239-
apm_plugin_artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/nomad-nodepool-apm"
240-
apm_plugin_checksum = data.aws_s3_object.nomad_nodepool_apm[0].etag
245+
apm_plugin_artifact_source = "s3::https://${var.fc_env_pipeline_bucket_name}.s3.${var.aws_region}.amazonaws.com/nomad-nodepool-apm?etag=${data.aws_s3_object.nomad_nodepool_apm[0].etag}"
241246
}
242247

243248
# ---

‎iac/provider-gcp/nomad/clean-nfs-cache.tf‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎iac/provider-gcp/nomad/jobs/clean-nfs-cache.hcl‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,9 @@ job "filestore-cleanup" {
4747
}
4848

4949
artifact {
50-
%{ if environment == "dev" }
51-
// Version hash is only available for dev to increase development speed in prod use rolling updates
52-
source = "gcs::https://www.googleapis.com/storage/v1/${bucket_name}/clean-nfs-cache?version=${clean_nfs_cache_checksum}"
53-
%{ else }
54-
source = "gcs::https://www.googleapis.com/storage/v1/${bucket_name}/clean-nfs-cache"
55-
%{ endif }
50+
source = "${artifact_source}"
51+
destination = "local/clean-nfs-cache"
52+
mode = "file"
5653
}
5754

5855
}

‎iac/provider-gcp/nomad/main.tf‎

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -394,16 +394,9 @@ data "google_storage_bucket_object" "orchestrator" {
394394
bucket = var.fc_env_pipeline_bucket_name
395395
}
396396

397-
data "external" "orchestrator_checksum" {
398-
program = ["bash", "${path.module}/scripts/checksum.sh"]
399-
400-
query = {
401-
base64 = data.google_storage_bucket_object.orchestrator.md5hash
402-
}
403-
}
404-
405397
locals {
406-
orchestrator_artifact_source = var.environment == "dev" ? "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/orchestrator?version=${data.external.orchestrator_checksum.result.hex}" : "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/orchestrator"
398+
orchestrator_checksum = data.google_storage_bucket_object.orchestrator.generation
399+
orchestrator_artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/orchestrator?version=${local.orchestrator_checksum}"
407400
}
408401

409402
module "orchestrator" {
@@ -420,7 +413,7 @@ module "orchestrator" {
420413

421414
environment = var.environment
422415
artifact_source = local.orchestrator_artifact_source
423-
orchestrator_checksum = data.external.orchestrator_checksum.result.hex
416+
orchestrator_checksum = local.orchestrator_checksum
424417

425418
logs_collector_address = "http://localhost:${var.logs_proxy_port.port}"
426419
otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
@@ -446,30 +439,8 @@ data "google_storage_bucket_object" "template_manager" {
446439
bucket = var.fc_env_pipeline_bucket_name
447440
}
448441

449-
450-
data "external" "template_manager" {
451-
program = ["bash", "${path.module}/scripts/checksum.sh"]
452-
453-
query = {
454-
base64 = data.google_storage_bucket_object.template_manager.md5hash
455-
}
456-
}
457-
458-
data "google_storage_bucket_object" "nomad_nodepool_apm" {
459-
count = var.template_manages_clusters_size_gt_1 ? 1 : 0
460-
461-
name = "nomad-nodepool-apm"
462-
bucket = var.fc_env_pipeline_bucket_name
463-
}
464-
465-
data "external" "nomad_nodepool_apm_checksum" {
466-
count = var.template_manages_clusters_size_gt_1 ? 1 : 0
467-
468-
program = ["bash", "${path.module}/scripts/checksum.sh"]
469-
470-
query = {
471-
base64 = data.google_storage_bucket_object.nomad_nodepool_apm[0].md5hash
472-
}
442+
locals {
443+
template_manager_artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/template-manager?version=${data.google_storage_bucket_object.template_manager.generation}"
473444
}
474445

475446
module "template_manager" {
@@ -493,8 +464,7 @@ module "template_manager" {
493464
domain_name = var.domain_name
494465

495466
api_secret = var.api_secret
496-
artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/template-manager"
497-
template_manager_checksum = data.external.template_manager.result.hex
467+
artifact_source = local.template_manager_artifact_source
498468
template_bucket_name = var.template_bucket_name
499469
build_cache_bucket_name = var.build_cache_bucket_name
500470
otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
@@ -508,15 +478,21 @@ module "template_manager" {
508478
nomad_token = var.nomad_acl_token_secret
509479
}
510480

481+
data "google_storage_bucket_object" "nomad_nodepool_apm" {
482+
count = var.template_manages_clusters_size_gt_1 ? 1 : 0
483+
484+
name = "nomad-nodepool-apm"
485+
bucket = var.fc_env_pipeline_bucket_name
486+
}
487+
511488
module "template_manager_autoscaler" {
512489
source = "../../modules/job-template-manager-autoscaler"
513490
count = var.template_manages_clusters_size_gt_1 ? 1 : 0
514491

515492
node_pool = var.api_node_pool
516493
autoscaler_version = var.nomad_autoscaler_version
517494
nomad_token = var.nomad_acl_token_secret
518-
apm_plugin_artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/nomad-nodepool-apm"
519-
apm_plugin_checksum = data.external.nomad_nodepool_apm_checksum[0].result.hex
495+
apm_plugin_artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/nomad-nodepool-apm?version=${data.google_storage_bucket_object.nomad_nodepool_apm[0].generation}"
520496
}
521497

522498
module "loki" {
@@ -625,3 +601,32 @@ module "clickhouse" {
625601
# Migrator
626602
clickhouse_migrator_image = data.google_artifact_registry_docker_image.clickhouse_migrator_image.self_link
627603
}
604+
605+
data "google_storage_bucket_object" "filestore_cleanup" {
606+
name = "clean-nfs-cache"
607+
bucket = var.fc_env_pipeline_bucket_name
608+
}
609+
610+
locals {
611+
clean_nfs_cache_artifact_source = "gcs::https://www.googleapis.com/storage/v1/${var.fc_env_pipeline_bucket_name}/clean-nfs-cache?version=${data.google_storage_bucket_object.filestore_cleanup.generation}"
612+
}
613+
614+
resource "nomad_job" "clean_nfs_cache" {
615+
count = var.shared_chunk_cache_path != "" ? 1 : 0
616+
617+
jobspec = templatefile("${path.module}/jobs/clean-nfs-cache.hcl", {
618+
node_pool = var.builder_node_pool
619+
artifact_source = local.clean_nfs_cache_artifact_source
620+
nfs_cache_mount_path = var.shared_chunk_cache_path
621+
max_disk_usage_target = var.filestore_cache_cleanup_disk_usage_target
622+
dry_run = var.filestore_cache_cleanup_dry_run
623+
deletions_per_loop = var.filestore_cache_cleanup_deletions_per_loop
624+
files_per_loop = var.filestore_cache_cleanup_files_per_loop
625+
max_concurrent_stat = var.filestore_cache_cleanup_max_concurrent_stat
626+
max_concurrent_scan = var.filestore_cache_cleanup_max_concurrent_scan
627+
max_concurrent_delete = var.filestore_cache_cleanup_max_concurrent_delete
628+
max_retries = var.filestore_cache_cleanup_max_retries
629+
otel_collector_grpc_endpoint = "localhost:${var.otel_collector_grpc_port}"
630+
launch_darkly_api_key = trimspace(data.google_secret_manager_secret_version.launch_darkly_api_key.secret_data)
631+
})
632+
}

0 commit comments

Comments
 (0)