diff --git a/bicep/ccw.bicep b/bicep/ccw.bicep index e165451f..dfaa7aa3 100644 --- a/bicep/ccw.bicep +++ b/bicep/ccw.bicep @@ -15,6 +15,7 @@ param adminUsername string param adminPassword string param adminSshPublicKey string param storedKey types.storedKey_t +param storageAccount types.storageAccount_t param ccVMName string param ccVMSize string param resourceGroup string @@ -113,7 +114,17 @@ module ccwBastion './bastion.bicep' = if (deploy_bastion) { } param cyclecloudBaseImage string = 'azurecyclecloud:azure-cyclecloud:cyclecloud8-gen2:8.7.320250909' +var vmMiName = 'ccwCycleCloudVirtualMachineManagedIdentity' +module ccwVirtualMachineManagedIdentity './vmManagedIdentity.bicep' = if (!infrastructureOnly && storageAccount.type == 'new') { + name: vmMiName + params: { + name: vmMiName + location: location + tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags) + } +} +var ccwVirtualMachineManagedIdentityId = !infrastructureOnly ? ( storageAccount.type == 'new' ? ccwVirtualMachineManagedIdentity!.outputs.managedIdentityId : storageAccount.vmManagedIdentityId) : '' module ccwVM './vm.bicep' = if (!infrastructureOnly) { name: 'ccwVM-cyclecloud' params: { @@ -151,49 +162,60 @@ module ccwVM './vm.bicep' = if (!infrastructureOnly) { createOption: split(cyclecloudBaseImage, ':')[0] == 'azurecyclecloud' ? 'FromImage' : 'Empty' } ] + managedIdentityId: ccwVirtualMachineManagedIdentityId } dependsOn: [ ccwNetwork ] } -var miName = 'ccwLockerManagedIdentity' -module ccwManagedIdentity 'mi.bicep' = if (!infrastructureOnly) { - name: miName +module ccwNewStorageAccount './storage-new.bicep' = if (storageAccount.type == 'new') { + name: 'ccwNewStorageAccount' params: { - name: miName location: location - storageAccountName: ccwStorage.outputs.storageAccountName - tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags) + tags: getTags('Microsoft.Storage/storageAccounts', tags) } } +var storageAccountName = storageAccount.type == 'existing' ? split(storageAccount.storageAccountId, '/')[8] : ccwNewStorageAccount!.outputs.storageAccountName -module ccwRoleAssignments './vmRoleAssignments.bicep' = if (!infrastructureOnly) { - name: 'ccwRoleFor-${ccVMName}-${location}' - scope: subscription() +module ccwStorageNetworking './storage-networking.bicep' = { + name: 'ccwStorageAccountNetworking' params: { - roles: [ - 'Contributor' - 'Storage Account Contributor' - 'Storage Blob Data Contributor' - ] - principalId: ccwVM.outputs.principalId + location: location + saName: storageAccountName + tags: getTags('Microsoft.Storage/storageAccounts', tags) + subnetId: subnets.cyclecloud.id + storagePrivateDnsZone: storagePrivateDnsZone } - dependsOn: [ - ccwVM - ] } -module ccwStorage './storage.bicep' = { - name: 'ccwStorage' +var vmssMiName = 'ccwLockerManagedIdentity' +module ccwVMSSManagedIdentity 'vmssManagedIdentity.bicep' = if (!infrastructureOnly && storageAccount.type == 'new') { + name: vmssMiName params: { + name: vmssMiName location: location - tags: getTags('Microsoft.Storage/storageAccounts', tags) - saName: 'ccwstorage${uniqueString(az.resourceGroup().id)}' - subnetId: subnets.cyclecloud.id - storagePrivateDnsZone: storagePrivateDnsZone + storageAccountName: storageAccountName + tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags) } } +var vmssManagedIdentityId = !infrastructureOnly ? ( storageAccount.type == 'new' ? ccwVMSSManagedIdentity!.outputs.managedIdentityId : storageAccount.vmssManagedIdentityId) : '' + +// module ccwRoleAssignments './vmRoleAssignments.bicep' = if (!infrastructureOnly) { +// name: 'ccwRoleFor-${ccVMName}-${location}' +// scope: subscription() +// params: { +// roles: [ +// 'Contributor' +// 'Storage Account Contributor' +// 'Storage Blob Data Contributor' +// ] +// principalId: ccwVM.outputs.principalId +// } +// dependsOn: [ +// ccwVM +// ] +// } var create_database = contains(slurmSettings, 'databaseAdminPassword') var db_name = 'ccw-mysqldb-${uniqueString(az.resourceGroup().id)}' @@ -313,9 +335,10 @@ output filerInfoFinal types.filerInfo_t = { } } -output cyclecloudPrincipalId string = infrastructureOnly ? '' : ccwVM.outputs.principalId +output cyclecloudPrincipalId string = infrastructureOnly ? '' : ccwVM!.outputs.principalId -output managedIdentityId string = infrastructureOnly ? '' : ccwManagedIdentity.outputs.managedIdentityId +// MI for VMSS +output managedIdentityId string = vmssManagedIdentityId // Automatically inject the ccw and pyxis cluster init specs @@ -369,7 +392,7 @@ var clusterNameCleaned = join(clusterNameArrCleaned,'') output resourceGroup string = resourceGroup output location string = location -output storageAccountName string = ccwStorage.outputs.storageAccountName +output storageAccountName string = storageAccountName output clusterName string = clusterNameCleaned output publicKey string = publicKey output adminUsername string = adminUsername diff --git a/bicep/mainTemplate.bicep b/bicep/mainTemplate.bicep index aad183fa..b4d40b47 100644 --- a/bicep/mainTemplate.bicep +++ b/bicep/mainTemplate.bicep @@ -12,6 +12,7 @@ param storedKey types.storedKey_t = {id: 'foo', location: 'foo', name:'foo'} param ccVMName string param ccVMSize string param resourceGroup string +param storageAccount types.storageAccount_t param sharedFilesystem types.sharedFilesystem_t param additionalFilesystem types.additionalFilesystem_t = { type: 'disabled' } param network types.vnet_t @@ -62,6 +63,7 @@ module makeCCWresources 'ccw.bicep' = { adminUsername: adminUsername adminPassword: adminPassword adminSshPublicKey: adminSshPublicKey + storageAccount: storageAccount sharedFilesystem: sharedFilesystem additionalFilesystem: additionalFilesystem network: network diff --git a/bicep/storage.bicep b/bicep/storage-networking.bicep similarity index 89% rename from bicep/storage.bicep rename to bicep/storage-networking.bicep index 0497ad13..392acb52 100644 --- a/bicep/storage.bicep +++ b/bicep/storage-networking.bicep @@ -12,24 +12,8 @@ var privateDnsZoneResourceGroup = split(privateDnsZoneId, '/')[4] var createVnetLink = storagePrivateDnsZone.type == 'existing' ? storagePrivateDnsZone.vnetLink : storagePrivateDnsZone.type == 'new' var vnetLinkScope = contains(storagePrivateDnsZone,'id') ? split(privateDnsZoneId, '/')[4] : az.resourceGroup().name -resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' = { +resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' existing = { name: saName - location: location - tags: tags - sku: { - name: 'Standard_LRS' - } - kind: 'StorageV2' - properties:{ - accessTier: 'Hot' - minimumTlsVersion: 'TLS1_2' - allowSharedKeyAccess: false - publicNetworkAccess: 'Disabled' - allowBlobPublicAccess: false - networkAcls: { - defaultAction: 'Deny' - } - } } var storageBlobPrivateEndpointName = 'ccwstorage-blob-pe' diff --git a/bicep/storage-new.bicep b/bicep/storage-new.bicep new file mode 100644 index 00000000..5f4b8922 --- /dev/null +++ b/bicep/storage-new.bicep @@ -0,0 +1,28 @@ +targetScope = 'resourceGroup' +import {tags_t} from './types.bicep' + +var storageAccountName = 'ccwstorage${uniqueString(az.resourceGroup().id)}' +param location string +param tags tags_t = {} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' = { + name: storageAccountName + location: location + tags: tags + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties:{ + accessTier: 'Hot' + minimumTlsVersion: 'TLS1_2' + allowSharedKeyAccess: false + publicNetworkAccess: 'Disabled' + allowBlobPublicAccess: false + networkAcls: { + defaultAction: 'Deny' + } + } +} + +output storageAccountName string = storageAccount.name diff --git a/bicep/types.bicep b/bicep/types.bicep index dcdced07..567cff0d 100644 --- a/bicep/types.bicep +++ b/bicep/types.bicep @@ -317,3 +317,18 @@ type cluster_init_t = github_cluster_init_t | prestaged_cluster_init_t @export() type cluster_init_param_t = cluster_init_t[] + +type storageAccount_new_t = { + type: 'new' +} + +type storageAccount_existing_t = { + type: 'existing' + storageAccountId: string + vmManagedIdentityId: string + vmssManagedIdentityId: string +} + +@export() +@discriminator('type') +type storageAccount_t = storageAccount_new_t | storageAccount_existing_t diff --git a/bicep/vm.bicep b/bicep/vm.bicep index 2ddb96ef..a769d3e8 100644 --- a/bicep/vm.bicep +++ b/bicep/vm.bicep @@ -18,6 +18,7 @@ param adminSshPublicKey string param vmSize string param dataDisks array param osDiskSize int = 0 //TODO: add to UI +param managedIdentityId string resource nic 'Microsoft.Network/networkInterfaces@2023-11-01' = { name: '${name}-nic' @@ -39,7 +40,11 @@ resource nic 'Microsoft.Network/networkInterfaces@2023-11-01' = { } } -resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-03-01' = { +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = { + name: split(managedIdentityId, '/')[8] +} + +resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-11-01' = { name: name location: location tags: tags @@ -49,7 +54,10 @@ resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-03-01' = { name: split(image.plan,':')[2] } : null identity: { - type: 'SystemAssigned' + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}': {} + } } properties: { hardwareProfile: { @@ -132,6 +140,6 @@ resource cse 'Microsoft.Compute/virtualMachines/extensions@2024-03-01' = { output fqdn string = '' //contains(vm, 'pip') && vm.pip ? publicIp.properties.dnsSettings.fqdn : '' output publicIp string = '' //contains(vm, 'pip') && vm.pip ? publicIp.properties.ipAddress : '' output privateIp string = nic.properties.ipConfigurations[0].properties.privateIPAddress -output principalId string = virtualMachine.identity.principalId +output principalId string = managedIdentity.properties.principalId //output privateIps array = [ for i in range(0, count): nic[i].properties.ipConfigurations[0].properties.privateIPAddress ] //output principalIds array = [ for i in range(0, count): virtualMachine[i].identity.principalId ] diff --git a/bicep/vmManagedIdentity.bicep b/bicep/vmManagedIdentity.bicep new file mode 100644 index 00000000..cd5d3e4f --- /dev/null +++ b/bicep/vmManagedIdentity.bicep @@ -0,0 +1,29 @@ +targetScope = 'resourceGroup' +import {tags_t} from './types.bicep' + +param name string = 'ccwCycleCloudVirtualMachineManagedIdentity' +param location string +param applyRoleAssignments bool = true +param tags tags_t = {} + +//create managed identity for CycleCloud VM +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: name + location: location + tags: tags +} + +module ccwCycleCloudVirtualMachineRoleAssignments './vmManagedIdentityRoleAssignments.bicep' = if (applyRoleAssignments) { + name: 'ccwRoleForCycleCloudVirtualMachine-${location}' + scope: subscription() + params: { + roles: [ + 'Contributor' + 'Storage Account Contributor' + 'Storage Blob Data Contributor' + ] + principalId: managedIdentity.properties.principalId + } +} + +output managedIdentityId string = managedIdentity.id diff --git a/bicep/vmRoleAssignments.bicep b/bicep/vmManagedIdentityRoleAssignments.bicep similarity index 100% rename from bicep/vmRoleAssignments.bicep rename to bicep/vmManagedIdentityRoleAssignments.bicep diff --git a/bicep/mi.bicep b/bicep/vmssManagedIdentity.bicep similarity index 75% rename from bicep/mi.bicep rename to bicep/vmssManagedIdentity.bicep index 36185743..174cbb61 100644 --- a/bicep/mi.bicep +++ b/bicep/vmssManagedIdentity.bicep @@ -4,7 +4,8 @@ import {tags_t} from './types.bicep' param name string param location string param storageAccountName string -param tags tags_t +param applyRoleAssignments bool = true +param tags tags_t = {} //create managed identity for VMSSs resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { @@ -13,7 +14,7 @@ resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023- tags: tags } -module ccwMIRoleAssignments './miRoleAssignments.bicep' = { +module ccwLockerManagedIdentityRoleAssignments './vmssManagedIdentityRoleAssignments.bicep' = if (applyRoleAssignments) { name: 'ccwRoleForLockerManagedIdentity' params: { principalId: managedIdentity.properties.principalId diff --git a/bicep/miRoleAssignments.bicep b/bicep/vmssManagedIdentityRoleAssignments.bicep similarity index 100% rename from bicep/miRoleAssignments.bicep rename to bicep/vmssManagedIdentityRoleAssignments.bicep diff --git a/uidefinitions/createUiDefinition.json b/uidefinitions/createUiDefinition.json index 229823ac..77f15fc1 100644 --- a/uidefinitions/createUiDefinition.json +++ b/uidefinitions/createUiDefinition.json @@ -98,6 +98,88 @@ }, "visible": "[equals(basics('newexisting'),'new')]" }, + { + "name": "byoStorageAndIdentities", + "type": "Microsoft.Common.OptionsGroup", + "label": "TODO AGB: LABEL", + "defaultValue": "Create new storage account and managed identities", + "toolTip": "TODO AGB: TOOLTIP", + "constraints": { + "allowedValues": [ + { + "label": "Create new storage account and managed identities", + "value": "new" + }, + { + "label": "Use existing storage account and managed identities", + "value": "existing" + } + ], + "required": true + }, + "visible": true + }, + { + "name": "brownfieldStorageInfo", + "type": "Microsoft.Common.InfoBox", + "visible": "[equals(basics('byoStorageAndIdentities'),'existing')]", + "options": { + "text": "TODO AGB: Ensure that the script has been run in RG, link is to deployment docs...", + "uri": "https://learn.microsoft.com/azure/cyclecloud/?view=cyclecloud-8" + } + }, + { + "name": "storageApi", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id,'/providers/Microsoft.Storage/storageAccounts?api-version=2024-01-01')]" + } + }, + { + "name": "storageDropDown", + "type": "Microsoft.Common.DropDown", + "label": "Storage Account", + "toolTip": "TODO AGB: STORAGE TOOLTIP", + "multiLine": true, + "constraints": { + "allowedValues": "[map(basics('storageApi').value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"description\":\"', concat('Resource group: ',first(skip(split(item.id,'/'),4)),'\\nRegion: ',item.location), '\",\"value\":\"', item.id, '\"}')))]", + "required": true + }, + "visible": "[equals(basics('byoStorageAndIdentities'),'existing')]" + }, + { + "name": "managedIdentityApi", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id,'/providers/Microsoft.ManagedIdentity/userAssignedIdentities?api-version=2024-11-30')]" + } + }, + { + "name": "vmManagedIdentityDropDown", + "type": "Microsoft.Common.DropDown", + "label": "VM Managed Identity", + "toolTip": "TODO AGB: VM MANAGED IDENTITY TOOLTIP", + "multiLine": true, + "constraints": { + "allowedValues": "[map(basics('managedIdentityApi').value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"description\":\"', concat('Resource group: ',first(skip(split(item.id,'/'),4)),'\\nRegion: ',item.location), '\",\"value\":\"', item.id, '\"}')))]", + "required": true + }, + "visible": "[equals(basics('byoStorageAndIdentities'),'existing')]" + }, + { + "name": "vmssManagedIdentityDropDown", + "type": "Microsoft.Common.DropDown", + "label": "VMSS Managed Identity", + "toolTip": "TODO AGB: VMSS MANAGED IDENTITY TOOLTIP", + "multiLine": true, + "constraints": { + "allowedValues": "[map(basics('managedIdentityApi').value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"description\":\"', concat('Resource group: ',first(skip(split(item.id,'/'),4)),'\\nRegion: ',item.location), '\",\"value\":\"', item.id, '\"}')))]", + "required": true + }, + "visible": "[equals(basics('byoStorageAndIdentities'),'existing')]" + }, { "name": "CycleCloudVmName", "type": "Microsoft.Common.TextBox", @@ -2350,6 +2432,12 @@ "ccVMName": "[basics('CycleCloudVmName')]", "ccVMSize": "[basics('CycleCloudVmSize')]", "resourceGroup": "[if(equals(basics('newexisting'),'existing'),first(split(basics('rgExisting'),'~')),basics('rgNew'))]", + "storageAccount": { + "type": "[basics('byoStorageAndIdentities')]", + "storageAccountId": "[if(equals(basics('byoStorageAndIdentities'),'existing'),basics('storageDropDown'),basics('nullValue'))]", + "vmManagedIdentityId": "[if(equals(basics('byoStorageAndIdentities'),'existing'),basics('vmManagedIdentityDropDown'),basics('nullValue'))]", + "vmssManagedIdentityId": "[if(equals(basics('byoStorageAndIdentities'),'existing'),basics('vmssManagedIdentityDropDown'),basics('nullValue'))]" + }, "sharedFilesystem": { "type": "[concat(if(equals(steps('filesystem').shared.newexisting,'new'),steps('filesystem').shared.filertype,'nfs'),'-',steps('filesystem').shared.newexisting)]", "nfsCapacityInGb": "[if(equals(steps('filesystem').shared.filertype,'nfs'),int(steps('filesystem').shared.nfscapacity),basics('nullValue'))]", diff --git a/util/ccw_prerequisites.sh b/util/ccw_prerequisites.sh new file mode 100644 index 00000000..b29e407f --- /dev/null +++ b/util/ccw_prerequisites.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")/.." + +# Initialize variables +RESOURCE_GROUP="" +LOCATION="" +APPLY_ROLE_ASSIGNMENTS=true + +# Parse arguments +while [ "$#" -gt 0 ]; do + case "$1" in + -rg|--resource-group) + RESOURCE_GROUP="$2" + shift 2 + ;; + -l|--location) + LOCATION="$2" + shift 2 + ;; + --no-role-assignments) + APPLY_ROLE_ASSIGNMENTS=false + shift + ;; + -h|--help) + # TODO AGB: Clean up + echo "Usage: $0 --resource-group --location [--no-role-assignments]" + echo " or: $0 -rg -l [--no-role-assignments]" + echo " --no-role-assignments: Do not apply role assignments to the managed identities." + exit 0 + ;; + *) + echo "Unknown parameter: $1" + echo "Use --help for usage information." + exit 1 + ;; + esac +done + +# Check if the resource group exists and create it if it doesn't +echo Checking if resource group "${RESOURCE_GROUP}" exists... +RG_EXISTS=$(az group exists -n "$RESOURCE_GROUP" | tr -d '\r\n') +if [ "$RG_EXISTS" = "false" ]; then + echo "Resource group '$RESOURCE_GROUP' does not exist. Creating it in location '$LOCATION'." + az group create -n "$RESOURCE_GROUP" -l "$LOCATION" + + while RG_CREATED=$(az group exists -n "$RESOURCE_GROUP" | tr -d '\r\n'); [ "$RG_CREATED" = "false" ]; do + echo "Waiting for resource group '$RESOURCE_GROUP' to be created..." + sleep 1 + done +fi + +echo Deploying storage account to resource group "${RESOURCE_GROUP}" in location "${LOCATION}"... +STORAGE_DEPLOYMENT_NAME="ccw-storage-deployment-${RESOURCE_GROUP}-${LOCATION}" +az deployment group create \ + --resource-group "$RESOURCE_GROUP" \ + --template-file $(pwd)/bicep/storage-new.bicep \ + --parameters location="$LOCATION" \ + --name "$STORAGE_DEPLOYMENT_NAME" + +STORAGE_ACCOUNT_NAME=$(az deployment group show -g "$RESOURCE_GROUP" -n "$STORAGE_DEPLOYMENT_NAME" --query "properties.outputs.storageAccountName.value" -o tsv | tr -d '\r\n') + +echo Creating managed identity for virtual machine scale sets in resource group "${RESOURCE_GROUP}" in location "${LOCATION}"... +if [ "$APPLY_ROLE_ASSIGNMENTS" = true ]; then + echo "Role assignments will be applied after the managed identity is created." +else + echo "Role assignments will NOT be applied as requested." +fi +az deployment group create \ + --resource-group "$RESOURCE_GROUP" \ + --template-file $(pwd)/bicep/vmssManagedIdentity.bicep \ + --parameters name="ccwLockerManagedIdentity" \ + --parameters location="$LOCATION" \ + --parameters storageAccountName="$STORAGE_ACCOUNT_NAME" \ + --parameters applyRoleAssignments="$APPLY_ROLE_ASSIGNMENTS" \ + --name "ccw-vmss-mi-deployment-${RESOURCE_GROUP}-${LOCATION}" + +echo Creating managed identity for the CycleCloud virtual machine in resource group "${RESOURCE_GROUP}" in location "${LOCATION}"... +if [ "$APPLY_ROLE_ASSIGNMENTS" = true ]; then + echo "Role assignments will be applied after the managed identity is created." +else + echo "Role assignments will NOT be applied as requested." +fi +az deployment group create \ + --resource-group "$RESOURCE_GROUP" \ + --template-file $(pwd)/bicep/vmManagedIdentity.bicep \ + --parameters location="$LOCATION" \ + --parameters applyRoleAssignments="$APPLY_ROLE_ASSIGNMENTS" \ + --name "ccw-vm-mi-deployment-${RESOURCE_GROUP}-${LOCATION}" \ No newline at end of file