Skip to content

Commit cafb5d0

Browse files
furkansahinclaude
authored and
Ubicloud Dev
committed
Add GCE as upload target in main build workflow
Add upload_gce toggle alongside MinIO, R2, and AWS AMI. GCE steps post-process a copy of the raw image (preserving the original for other targets), upload the tar.gz to GCS, create a GCE image, and grant public access. Both x64 and arm64 jobs get GCE steps with arch-specific guest OS features. The create-ubicloud-pr job generates a pg_gce_image migration alongside the existing pg_aws_ami migration. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 919fe72 commit cafb5d0

1 file changed

Lines changed: 284 additions & 0 deletions

File tree

‎.github/workflows/postgres-vm-image.yml‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,18 @@ on:
5757
description: "Use AWS role-based authentication (if false, uses access keys)"
5858
default: false
5959
type: boolean
60+
upload_gce:
61+
description: "📤 Create GCE image"
62+
default: false
63+
type: boolean
64+
gcp_project:
65+
description: "GCP project ID for GCE image"
66+
type: string
67+
default: "pelagic-logic-394811"
68+
gcs_bucket:
69+
description: "GCS bucket for GCE image upload"
70+
type: string
71+
default: "ubicloud-gce-images"
6072

6173
permissions:
6274
id-token: write
@@ -72,6 +84,8 @@ jobs:
7284
sha256: ${{ steps.compute_sha.outputs.sha256 }}
7385
all_ami_ids: ${{ steps.copy_ami.outputs.all_ami_ids }}
7486
source_ami_id: ${{ steps.register_ami.outputs.ami_id }}
87+
gce_image_name: ${{ steps.create_gce_image.outputs.image_name }}
88+
gce_image_project: ${{ steps.create_gce_image.outputs.image_project }}
7589
steps:
7690
- name: Print inputs
7791
run: |
@@ -492,6 +506,110 @@ jobs:
492506
echo "Cleaning up S3..."
493507
aws s3 rm s3://${{ steps.s3_upload.outputs.s3_bucket }}/${{ steps.set_image_name.outputs.S3_BUCKET_IMAGE_PREFIX }}/${{ steps.s3_upload.outputs.image_filename }}
494508
509+
# === GCE Image Steps ===
510+
- name: GCE post-processing
511+
if: ${{ inputs.upload_gce && !inputs.build_only }}
512+
run: |
513+
image_filename=${{ steps.set_image_name.outputs.MINIO_IMAGE_NAME }}.raw
514+
cp "${image_filename}" postgres-x64-gce-work.raw
515+
sudo ./gce-postprocess.sh postgres-x64-gce-work.raw
516+
rm -f postgres-x64-gce-work.raw
517+
518+
- name: Set GCE image name
519+
if: ${{ inputs.upload_gce && !inputs.build_only }}
520+
id: set_gce_image_name
521+
run: |
522+
gce_image_name="postgres-ubuntu-2204-x64-${{ inputs.image_suffix }}"
523+
echo "gce_image_name=${gce_image_name}" >> $GITHUB_OUTPUT
524+
echo "GCE Image name: ${gce_image_name}"
525+
526+
- name: Rename GCE tar.gz and compute SHA256
527+
if: ${{ inputs.upload_gce && !inputs.build_only }}
528+
run: |
529+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
530+
mv postgres-x64-gce-image.tar.gz "${gce_image_name}.tar.gz"
531+
sha256sum "${gce_image_name}.tar.gz" > "${gce_image_name}.tar.gz.sha256"
532+
533+
echo "### GCE Image (x64)" >> $GITHUB_STEP_SUMMARY
534+
du -h "${gce_image_name}.tar.gz" >> $GITHUB_STEP_SUMMARY
535+
echo "### GCE SHA256" >> $GITHUB_STEP_SUMMARY
536+
cat "${gce_image_name}.tar.gz.sha256" >> $GITHUB_STEP_SUMMARY
537+
538+
- name: Authenticate to GCP
539+
if: ${{ inputs.upload_gce && !inputs.build_only }}
540+
uses: google-github-actions/auth@v2
541+
with:
542+
credentials_json: ${{ secrets.GCP_SA_KEY }}
543+
544+
- name: Set up Cloud SDK
545+
if: ${{ inputs.upload_gce && !inputs.build_only }}
546+
uses: google-github-actions/setup-gcloud@v2
547+
548+
- name: Upload to GCS
549+
if: ${{ inputs.upload_gce && !inputs.build_only }}
550+
run: |
551+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
552+
tar_file="${gce_image_name}.tar.gz"
553+
bucket="${{ inputs.gcs_bucket }}"
554+
555+
echo "Uploading ${tar_file} to gs://${bucket}/..."
556+
gcloud storage cp "${tar_file}" "gs://${bucket}/${tar_file}"
557+
558+
echo "### GCS Upload (x64)" >> $GITHUB_STEP_SUMMARY
559+
echo "Uploaded to gs://${bucket}/${tar_file}" >> $GITHUB_STEP_SUMMARY
560+
561+
- name: Create GCE image
562+
if: ${{ inputs.upload_gce && !inputs.build_only }}
563+
id: create_gce_image
564+
run: |
565+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
566+
tar_file="${gce_image_name}.tar.gz"
567+
bucket="${{ inputs.gcs_bucket }}"
568+
project="${{ inputs.gcp_project }}"
569+
commit_sha="${{ github.sha }}"
570+
571+
echo "Creating GCE image: ${gce_image_name}"
572+
gcloud compute images create "${gce_image_name}" \
573+
--project="${project}" \
574+
--source-uri="gs://${bucket}/${tar_file}" \
575+
--guest-os-features=VIRTIO_SCSI_MULTIQUEUE,GVNIC \
576+
--labels="source=postgres-vm-images,commit=${commit_sha:0:8},arch=x64"
577+
578+
echo "image_name=${gce_image_name}" >> $GITHUB_OUTPUT
579+
echo "image_project=${project}" >> $GITHUB_OUTPUT
580+
581+
echo "### GCE Image Created (x64)" >> $GITHUB_STEP_SUMMARY
582+
echo "- Name: ${gce_image_name}" >> $GITHUB_STEP_SUMMARY
583+
echo "- Project: ${project}" >> $GITHUB_STEP_SUMMARY
584+
echo "- Commit: ${commit_sha:0:8}" >> $GITHUB_STEP_SUMMARY
585+
586+
- name: Make GCE image public
587+
if: ${{ inputs.upload_gce && !inputs.build_only }}
588+
run: |
589+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
590+
project="${{ inputs.gcp_project }}"
591+
gcloud compute images add-iam-policy-binding "${gce_image_name}" \
592+
--project="${project}" \
593+
--member="allAuthenticatedUsers" \
594+
--role="roles/compute.imageUser"
595+
596+
- name: Verify GCE image
597+
if: ${{ inputs.upload_gce && !inputs.build_only }}
598+
run: |
599+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
600+
project="${{ inputs.gcp_project }}"
601+
gcloud compute images describe "${gce_image_name}" \
602+
--project="${project}" \
603+
--format="table(name,family,status,diskSizeGb,creationTimestamp)"
604+
605+
- name: Clean up GCS tar.gz
606+
if: ${{ inputs.upload_gce && !inputs.build_only }}
607+
continue-on-error: true
608+
run: |
609+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
610+
bucket="${{ inputs.gcs_bucket }}"
611+
gcloud storage rm "gs://${bucket}/${gce_image_name}.tar.gz"
612+
495613
# arm64 build
496614
build-arm64:
497615
name: Build postgres-ubuntu-2204-arm64-${{ inputs.image_suffix }}
@@ -501,6 +619,8 @@ jobs:
501619
sha256: ${{ steps.compute_sha.outputs.sha256 }}
502620
all_ami_ids: ${{ steps.copy_ami.outputs.all_ami_ids }}
503621
source_ami_id: ${{ steps.register_ami.outputs.ami_id }}
622+
gce_image_name: ${{ steps.create_gce_image.outputs.image_name }}
623+
gce_image_project: ${{ steps.create_gce_image.outputs.image_project }}
504624
steps:
505625
- name: Print inputs
506626
run: |
@@ -872,6 +992,111 @@ jobs:
872992
echo "Cleaning up S3..."
873993
aws s3 rm s3://${{ steps.s3_upload.outputs.s3_bucket }}/${{ steps.set_image_name.outputs.S3_BUCKET_IMAGE_PREFIX }}/${{ steps.s3_upload.outputs.image_filename }}
874994
995+
# === GCE Image Steps ===
996+
- name: GCE post-processing
997+
if: ${{ inputs.upload_gce && !inputs.build_only }}
998+
run: |
999+
image_filename=${{ steps.set_image_name.outputs.IMAGE_NAME }}.raw
1000+
cp "${image_filename}" postgres-arm64-gce-work.raw
1001+
sudo ./gce-postprocess.sh postgres-arm64-gce-work.raw
1002+
rm -f postgres-arm64-gce-work.raw
1003+
1004+
- name: Set GCE image name
1005+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1006+
id: set_gce_image_name
1007+
run: |
1008+
gce_image_name="postgres-ubuntu-2204-arm64-${{ inputs.image_suffix }}"
1009+
echo "gce_image_name=${gce_image_name}" >> $GITHUB_OUTPUT
1010+
echo "GCE Image name: ${gce_image_name}"
1011+
1012+
- name: Rename GCE tar.gz and compute SHA256
1013+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1014+
run: |
1015+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1016+
mv postgres-arm64-gce-image.tar.gz "${gce_image_name}.tar.gz"
1017+
sha256sum "${gce_image_name}.tar.gz" > "${gce_image_name}.tar.gz.sha256"
1018+
1019+
echo "### GCE Image (arm64)" >> $GITHUB_STEP_SUMMARY
1020+
du -h "${gce_image_name}.tar.gz" >> $GITHUB_STEP_SUMMARY
1021+
echo "### GCE SHA256" >> $GITHUB_STEP_SUMMARY
1022+
cat "${gce_image_name}.tar.gz.sha256" >> $GITHUB_STEP_SUMMARY
1023+
1024+
- name: Authenticate to GCP
1025+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1026+
uses: google-github-actions/auth@v2
1027+
with:
1028+
credentials_json: ${{ secrets.GCP_SA_KEY }}
1029+
1030+
- name: Set up Cloud SDK
1031+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1032+
uses: google-github-actions/setup-gcloud@v2
1033+
1034+
- name: Upload to GCS
1035+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1036+
run: |
1037+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1038+
tar_file="${gce_image_name}.tar.gz"
1039+
bucket="${{ inputs.gcs_bucket }}"
1040+
1041+
echo "Uploading ${tar_file} to gs://${bucket}/..."
1042+
gcloud storage cp "${tar_file}" "gs://${bucket}/${tar_file}"
1043+
1044+
echo "### GCS Upload (arm64)" >> $GITHUB_STEP_SUMMARY
1045+
echo "Uploaded to gs://${bucket}/${tar_file}" >> $GITHUB_STEP_SUMMARY
1046+
1047+
- name: Create GCE image
1048+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1049+
id: create_gce_image
1050+
run: |
1051+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1052+
tar_file="${gce_image_name}.tar.gz"
1053+
bucket="${{ inputs.gcs_bucket }}"
1054+
project="${{ inputs.gcp_project }}"
1055+
commit_sha="${{ github.sha }}"
1056+
1057+
echo "Creating GCE image: ${gce_image_name}"
1058+
gcloud compute images create "${gce_image_name}" \
1059+
--project="${project}" \
1060+
--source-uri="gs://${bucket}/${tar_file}" \
1061+
--guest-os-features=GVNIC,UEFI_COMPATIBLE \
1062+
--architecture=ARM64 \
1063+
--labels="source=postgres-vm-images,commit=${commit_sha:0:8},arch=arm64"
1064+
1065+
echo "image_name=${gce_image_name}" >> $GITHUB_OUTPUT
1066+
echo "image_project=${project}" >> $GITHUB_OUTPUT
1067+
1068+
echo "### GCE Image Created (arm64)" >> $GITHUB_STEP_SUMMARY
1069+
echo "- Name: ${gce_image_name}" >> $GITHUB_STEP_SUMMARY
1070+
echo "- Project: ${project}" >> $GITHUB_STEP_SUMMARY
1071+
echo "- Commit: ${commit_sha:0:8}" >> $GITHUB_STEP_SUMMARY
1072+
1073+
- name: Make GCE image public
1074+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1075+
run: |
1076+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1077+
project="${{ inputs.gcp_project }}"
1078+
gcloud compute images add-iam-policy-binding "${gce_image_name}" \
1079+
--project="${project}" \
1080+
--member="allAuthenticatedUsers" \
1081+
--role="roles/compute.imageUser"
1082+
1083+
- name: Verify GCE image
1084+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1085+
run: |
1086+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1087+
project="${{ inputs.gcp_project }}"
1088+
gcloud compute images describe "${gce_image_name}" \
1089+
--project="${project}" \
1090+
--format="table(name,family,status,diskSizeGb,creationTimestamp)"
1091+
1092+
- name: Clean up GCS tar.gz
1093+
if: ${{ inputs.upload_gce && !inputs.build_only }}
1094+
continue-on-error: true
1095+
run: |
1096+
gce_image_name="${{ steps.set_gce_image_name.outputs.gce_image_name }}"
1097+
bucket="${{ inputs.gcs_bucket }}"
1098+
gcloud storage rm "gs://${bucket}/${gce_image_name}.tar.gz"
1099+
8751100
# Create PR to ubicloud/ubicloud with updated image versions
8761101
create-ubicloud-pr:
8771102
name: Create PR to ubicloud/ubicloud
@@ -913,6 +1138,11 @@ jobs:
9131138
arm64_amis="us-west-2:${{ needs.build-arm64.outputs.source_ami_id }}"
9141139
fi
9151140
echo "arm64_ami_ids=${arm64_amis}" >> $GITHUB_OUTPUT
1141+
1142+
# GCE image names
1143+
echo "x64_gce_image_name=${{ needs.build-x64.outputs.gce_image_name }}" >> $GITHUB_OUTPUT
1144+
echo "arm64_gce_image_name=${{ needs.build-arm64.outputs.gce_image_name }}" >> $GITHUB_OUTPUT
1145+
echo "gce_image_project=${{ needs.build-x64.outputs.gce_image_project }}" >> $GITHUB_OUTPUT
9161146
fi
9171147
9181148
- name: Clone ubicloud/ubicloud
@@ -1037,6 +1267,56 @@ jobs:
10371267
echo "Created migration file: ${migration_file}"
10381268
cat "${migration_file}"
10391269
1270+
- name: Create GCE migration file
1271+
if: ${{ steps.collect.outputs.x64_gce_image_name != '' || steps.collect.outputs.arm64_gce_image_name != '' }}
1272+
run: |
1273+
cd ubicloud
1274+
timestamp=$(date +%Y%m%d)
1275+
migration_file="migrate/${timestamp}_update_pg_gce_images.rb"
1276+
1277+
x64_gce_image="${{ steps.collect.outputs.x64_gce_image_name }}"
1278+
arm64_gce_image="${{ steps.collect.outputs.arm64_gce_image_name }}"
1279+
project="${{ steps.collect.outputs.gce_image_project }}"
1280+
1281+
cat > "${migration_file}" << MIGRATION
1282+
# frozen_string_literal: true
1283+
1284+
Sequel.migration do
1285+
up do
1286+
MIGRATION
1287+
sed -i 's/^ //' "${migration_file}"
1288+
1289+
if [ -n "$x64_gce_image" ]; then
1290+
cat >> "${migration_file}" << MIGRATION
1291+
from(:pg_gce_image)
1292+
.where(gcp_project_id: "${project}", arch: "x64")
1293+
.update(gce_image_name: "${x64_gce_image}")
1294+
MIGRATION
1295+
sed -i 's/^ //' "${migration_file}"
1296+
fi
1297+
1298+
if [ -n "$arm64_gce_image" ]; then
1299+
cat >> "${migration_file}" << MIGRATION
1300+
from(:pg_gce_image)
1301+
.where(gcp_project_id: "${project}", arch: "arm64")
1302+
.update(gce_image_name: "${arm64_gce_image}")
1303+
MIGRATION
1304+
sed -i 's/^ //' "${migration_file}"
1305+
fi
1306+
1307+
cat >> "${migration_file}" << MIGRATION
1308+
end
1309+
1310+
down do
1311+
raise Sequel::Error, "irreversible: previous GCE image names unknown"
1312+
end
1313+
end
1314+
MIGRATION
1315+
sed -i 's/^ //' "${migration_file}"
1316+
1317+
echo "Created GCE migration file: ${migration_file}"
1318+
cat "${migration_file}"
1319+
10401320
- name: Create Pull Request
10411321
env:
10421322
GH_TOKEN: ${{ secrets.UBICLOUD_REPO_PAT }}
@@ -1080,13 +1360,17 @@ jobs:
10801360
--body "## Summary
10811361
- Updates boot image version and SHA256 hashes in \`prog/download_boot_image.rb\`
10821362
- Adds migration to update AWS AMI IDs in \`pg_aws_ami\` table
1363+
- Adds migration to update GCE image names in \`pg_gce_image\` table (if GCE images built)
10831364
10841365
## Image Version
10851366
\`${{ inputs.image_suffix }}\`
10861367
10871368
## Changes
10881369
- x64 SHA256: \`${{ steps.collect.outputs.x64_sha256 }}\`
10891370
- arm64 SHA256: \`${{ steps.collect.outputs.arm64_sha256 }}\`
1371+
- GCE x64: \`${{ steps.collect.outputs.x64_gce_image_name }}\`
1372+
- GCE arm64: \`${{ steps.collect.outputs.arm64_gce_image_name }}\`
1373+
- GCE project: \`${{ steps.collect.outputs.gce_image_project }}\`
10901374
10911375
🤖 Generated by [postgres-vm-images](https://github.com/ubicloud/postgres-vm-images) workflow"
10921376

0 commit comments

Comments
 (0)