WIP: Support for brownfield deployment with BYO managed identities and storage accounts

bicep/ccw.bicep

@@ -15,6 +15,7 @@ param adminUsername string
 param adminPassword string
 param adminSshPublicKey string
 param storedKey types.storedKey_t
+param storageAccount types.storageAccount_t
 param ccVMName string
 param ccVMSize string
 param resourceGroup string
@@ -113,7 +114,17 @@ module ccwBastion './bastion.bicep' = if (deploy_bastion) {
 }
 
 param cyclecloudBaseImage string = 'azurecyclecloud:azure-cyclecloud:cyclecloud8-gen2:8.7.320250909'
+var vmMiName = 'ccwCycleCloudVirtualMachineManagedIdentity'
+module ccwVirtualMachineManagedIdentity './vmManagedIdentity.bicep' = if (!infrastructureOnly && storageAccount.type == 'new') {
+  name: vmMiName
+  params: {
+    name: vmMiName
+    location: location
+    tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags)
+  }
+}
 
+var ccwVirtualMachineManagedIdentityId = !infrastructureOnly ? ( storageAccount.type == 'new' ? ccwVirtualMachineManagedIdentity!.outputs.managedIdentityId : storageAccount.vmManagedIdentityId) : ''
 module ccwVM './vm.bicep' = if (!infrastructureOnly) {
   name: 'ccwVM-cyclecloud'
   params: {
@@ -151,49 +162,60 @@ module ccwVM './vm.bicep' = if (!infrastructureOnly) {
         createOption: split(cyclecloudBaseImage, ':')[0] == 'azurecyclecloud' ? 'FromImage' : 'Empty'
       }
     ]
+    managedIdentityId: ccwVirtualMachineManagedIdentityId
   }
   dependsOn: [
     ccwNetwork
   ]
 }
 
-var miName = 'ccwLockerManagedIdentity'
-module ccwManagedIdentity 'mi.bicep' = if (!infrastructureOnly) {
-  name: miName
+module ccwNewStorageAccount './storage-new.bicep' = if (storageAccount.type == 'new') {
+  name: 'ccwNewStorageAccount'
   params: {
-    name: miName
     location: location
-    storageAccountName: ccwStorage.outputs.storageAccountName
-    tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags)
+    tags: getTags('Microsoft.Storage/storageAccounts', tags)
   }
 }
+var storageAccountName  = storageAccount.type == 'existing' ? split(storageAccount.storageAccountId, '/')[8] : ccwNewStorageAccount!.outputs.storageAccountName
 
-module ccwRoleAssignments './vmRoleAssignments.bicep' = if (!infrastructureOnly) {
-  name: 'ccwRoleFor-${ccVMName}-${location}'
-  scope: subscription()
+module ccwStorageNetworking './storage-networking.bicep' = {
+  name: 'ccwStorageAccountNetworking'
   params: {
-    roles: [
-      'Contributor'
-      'Storage Account Contributor'
-      'Storage Blob Data Contributor'
-    ]
-    principalId: ccwVM.outputs.principalId
+    location: location
+    saName: storageAccountName
+    tags: getTags('Microsoft.Storage/storageAccounts', tags)
+    subnetId: subnets.cyclecloud.id
+    storagePrivateDnsZone: storagePrivateDnsZone
   }
-  dependsOn: [
-    ccwVM
-  ]
 }
 
-module ccwStorage './storage.bicep' = {
-  name: 'ccwStorage'
+var vmssMiName = 'ccwLockerManagedIdentity'
+module ccwVMSSManagedIdentity 'vmssManagedIdentity.bicep' = if (!infrastructureOnly && storageAccount.type == 'new') {
+  name: vmssMiName
   params: {
+    name: vmssMiName
     location: location
-    tags: getTags('Microsoft.Storage/storageAccounts', tags)
-    saName: 'ccwstorage${uniqueString(az.resourceGroup().id)}'
-    subnetId: subnets.cyclecloud.id 
-    storagePrivateDnsZone: storagePrivateDnsZone
+    storageAccountName: storageAccountName
+    tags: getTags('Microsoft.ManagedIdentity/userAssignedIdentities', tags)
   }
 }
+var vmssManagedIdentityId = !infrastructureOnly ? ( storageAccount.type == 'new' ? ccwVMSSManagedIdentity!.outputs.managedIdentityId : storageAccount.vmssManagedIdentityId) : ''
+
+// module ccwRoleAssignments './vmRoleAssignments.bicep' = if (!infrastructureOnly) {
+//   name: 'ccwRoleFor-${ccVMName}-${location}'
+//   scope: subscription()
+//   params: {
+//     roles: [
+//       'Contributor'
+//       'Storage Account Contributor'
+//       'Storage Blob Data Contributor'
+//     ]
+//     principalId: ccwVM.outputs.principalId
+//   }
+//   dependsOn: [
+//     ccwVM
+//   ]
+// }
 
 var create_database = contains(slurmSettings, 'databaseAdminPassword')
 var db_name = 'ccw-mysqldb-${uniqueString(az.resourceGroup().id)}'
@@ -313,9 +335,10 @@ output filerInfoFinal types.filerInfo_t = {
   }
 }
 
-output cyclecloudPrincipalId string = infrastructureOnly ? '' : ccwVM.outputs.principalId
+output cyclecloudPrincipalId string = infrastructureOnly ? '' : ccwVM!.outputs.principalId
 
-output managedIdentityId string = infrastructureOnly ? '' : ccwManagedIdentity.outputs.managedIdentityId
+// MI for VMSS
+output managedIdentityId string = vmssManagedIdentityId
 
 // Automatically inject the ccw and pyxis cluster init specs
 
@@ -369,7 +392,7 @@ var clusterNameCleaned = join(clusterNameArrCleaned,'')
 
 output resourceGroup string = resourceGroup
 output location string = location
-output storageAccountName string = ccwStorage.outputs.storageAccountName
+output storageAccountName string = storageAccountName
 output clusterName string = clusterNameCleaned
 output publicKey string = publicKey
 output adminUsername string = adminUsername

bicep/mainTemplate.bicep

@@ -12,6 +12,7 @@ param storedKey types.storedKey_t = {id: 'foo', location: 'foo', name:'foo'}
 param ccVMName string
 param ccVMSize string
 param resourceGroup string
+param storageAccount types.storageAccount_t
 param sharedFilesystem types.sharedFilesystem_t
 param additionalFilesystem types.additionalFilesystem_t = { type: 'disabled' }
 param network types.vnet_t
@@ -62,6 +63,7 @@ module makeCCWresources 'ccw.bicep' = {
     adminUsername: adminUsername
     adminPassword: adminPassword
     adminSshPublicKey: adminSshPublicKey
+    storageAccount: storageAccount
     sharedFilesystem: sharedFilesystem
     additionalFilesystem: additionalFilesystem
     network: network

bicep/storage.bicep → bicep/storage-networking.bicep

@@ -12,24 +12,8 @@ var privateDnsZoneResourceGroup = split(privateDnsZoneId, '/')[4]
 var createVnetLink = storagePrivateDnsZone.type == 'existing' ? storagePrivateDnsZone.vnetLink : storagePrivateDnsZone.type == 'new'
 var vnetLinkScope = contains(storagePrivateDnsZone,'id') ? split(privateDnsZoneId, '/')[4] : az.resourceGroup().name
 
-resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' = {
+resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' existing = {
   name: saName
-  location: location
-  tags: tags
-  sku: {
-    name: 'Standard_LRS'
-  }
-  kind: 'StorageV2'
-  properties:{
-    accessTier: 'Hot'
-    minimumTlsVersion: 'TLS1_2'
-    allowSharedKeyAccess: false
-    publicNetworkAccess: 'Disabled'
-    allowBlobPublicAccess: false
-    networkAcls: {
-      defaultAction: 'Deny'
-      }      
-  }
 }
 
 var storageBlobPrivateEndpointName = 'ccwstorage-blob-pe'

bicep/storage-new.bicep

@@ -0,0 +1,28 @@
+targetScope = 'resourceGroup'
+import {tags_t} from './types.bicep'
+
+var storageAccountName = 'ccwstorage${uniqueString(az.resourceGroup().id)}'
+param location string
+param tags tags_t = {}
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2024-01-01' = {
+  name: storageAccountName
+  location: location
+  tags: tags
+  sku: {
+    name: 'Standard_LRS'
+  }
+  kind: 'StorageV2'
+  properties:{
+    accessTier: 'Hot'
+    minimumTlsVersion: 'TLS1_2'
+    allowSharedKeyAccess: false
+    publicNetworkAccess: 'Disabled'
+    allowBlobPublicAccess: false
+    networkAcls: {
+      defaultAction: 'Deny'
+      }      
+  }
+}
+
+output storageAccountName string = storageAccount.name

bicep/types.bicep

@@ -317,3 +317,18 @@ type cluster_init_t = github_cluster_init_t | prestaged_cluster_init_t
 
 @export()
 type cluster_init_param_t = cluster_init_t[]
+
+type storageAccount_new_t = {
+  type: 'new'
+}
+
+type storageAccount_existing_t = {
+  type: 'existing'
+  storageAccountId: string
+  vmManagedIdentityId: string
+  vmssManagedIdentityId: string
+}
+
+@export()
+@discriminator('type')
+type storageAccount_t = storageAccount_new_t | storageAccount_existing_t

bicep/vm.bicep

@@ -18,6 +18,7 @@ param adminSshPublicKey string
 param vmSize string
 param dataDisks array
 param osDiskSize int = 0 //TODO: add to UI
+param managedIdentityId string
 
 resource nic 'Microsoft.Network/networkInterfaces@2023-11-01' = {
   name: '${name}-nic'
@@ -39,7 +40,11 @@ resource nic 'Microsoft.Network/networkInterfaces@2023-11-01' = {
   }
 }
 
-resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-03-01' = {
+resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
+  name: split(managedIdentityId, '/')[8]
+}
+
+resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-11-01' = {
   name: name
   location: location
   tags: tags
@@ -49,7 +54,10 @@ resource virtualMachine 'Microsoft.Compute/virtualMachines@2024-03-01' = {
     name: split(image.plan,':')[2]
   } : null
   identity: {
-    type: 'SystemAssigned'
+    type: 'UserAssigned'
+    userAssignedIdentities: {
+      '${managedIdentityId}': {}
+    }
   }
   properties: {
     hardwareProfile: {
@@ -132,6 +140,6 @@ resource cse 'Microsoft.Compute/virtualMachines/extensions@2024-03-01' = {
 output fqdn string = '' //contains(vm, 'pip') && vm.pip ? publicIp.properties.dnsSettings.fqdn : ''
 output publicIp string = '' //contains(vm, 'pip') && vm.pip ? publicIp.properties.ipAddress : ''
 output privateIp string = nic.properties.ipConfigurations[0].properties.privateIPAddress
-output principalId string = virtualMachine.identity.principalId
+output principalId string = managedIdentity.properties.principalId
 //output privateIps array = [ for i in range(0, count): nic[i].properties.ipConfigurations[0].properties.privateIPAddress ]
 //output principalIds array = [ for i in range(0, count): virtualMachine[i].identity.principalId ]

bicep/vmManagedIdentity.bicep

@@ -0,0 +1,29 @@
+targetScope = 'resourceGroup'
+import {tags_t} from './types.bicep'
+
+param name string = 'ccwCycleCloudVirtualMachineManagedIdentity'
+param location string
+param applyRoleAssignments bool = true
+param tags tags_t = {}
+
+//create managed identity for CycleCloud VM
+resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
+  name: name
+  location: location
+  tags: tags
+}
+
+module ccwCycleCloudVirtualMachineRoleAssignments './vmManagedIdentityRoleAssignments.bicep' = if (applyRoleAssignments) {
+  name: 'ccwRoleForCycleCloudVirtualMachine-${location}'
+  scope: subscription()
+  params: {
+    roles: [
+      'Contributor'
+      'Storage Account Contributor'
+      'Storage Blob Data Contributor'
+    ]
+    principalId: managedIdentity.properties.principalId
+  }
+}
+
+output managedIdentityId string = managedIdentity.id

bicep/mi.bicep → bicep/vmssManagedIdentity.bicep

@@ -4,7 +4,8 @@ import {tags_t} from './types.bicep'
 param name string
 param location string
 param storageAccountName string
-param tags tags_t
+param applyRoleAssignments bool = true
+param tags tags_t = {}
 
 //create managed identity for VMSSs
 resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
@@ -13,7 +14,7 @@ resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-
   tags: tags
 }
 
-module ccwMIRoleAssignments './miRoleAssignments.bicep' = {
+module ccwLockerManagedIdentityRoleAssignments './vmssManagedIdentityRoleAssignments.bicep' = if (applyRoleAssignments) {
   name: 'ccwRoleForLockerManagedIdentity'
   params: {
     principalId: managedIdentity.properties.principalId