gb200 hub spoke

.gitignore

@@ -407,3 +407,5 @@ arm-ttk/
 
 # delete_roles.sh files
 util/.role_assignment_cleanup*
+bicep/hub/build/
+bicep/hub/params/*.json

bicep/anf.bicep

@@ -4,12 +4,12 @@ import {tags_t, availabilityZone_t} from './types.bicep'
 param name string
 param location string
 param tags tags_t
-param availabilityZone availabilityZone_t[]
+param availabilityZone availabilityZone_t[] = []
 param resourcePostfix string = uniqueString(resourceGroup().id)
 param subnetId string
 param serviceLevel string
 param sizeTiB int
-param defaultMountOptions string
+param defaultMountOptions string = 'rw,hard,rsize=262144,wsize=262144,vers=3,tcp,_netdev,nconnect=8'
 param infrastructureOnly bool = false
 var capacity = sizeTiB * 1024 * 1024 * 1024 * 1024
 

bicep/ccw.bicep

@@ -7,7 +7,10 @@ param insidersBuild bool
 
 param branch string
 param projectVersion string
-param pyxisProjectVersion string
+param monitoringProjectVersion string
+param monitoringIngestionEndpoint string
+param monitoringIdentityClientId string
+param hubMI string
 
 param adminUsername string
 @secure()
@@ -25,7 +28,9 @@ param clusterInitSpecs types.cluster_init_param_t
 param slurmSettings types.slurmSettings_t 
 param schedulerNode types.scheduler_t
 param loginNodes types.login_t
-param htc types.htc_t
+param d64d types.htc_t
+param d16d types.htc_t
+param m64 types.htc_t
 param hpc types.hpc_t
 param gpu types.hpc_t
 param tags types.resource_tags_t
@@ -103,7 +108,7 @@ output vnet types.networkOutput_t = union(
 var deploy_bastion = network.?bastion ?? false
 module ccwBastion './bastion.bicep' = if (deploy_bastion) {
   name: 'ccwBastion'
-  scope: create_new_vnet ? az.resourceGroup() : az.resourceGroup(split(network.?existing_vnet_id, '/')[4])
+  scope: az.resourceGroup()
   params: {
     location: location
     tags: getTags('Microsoft.Network/bastionHosts', tags)
@@ -202,7 +207,7 @@ module mySQLccw './mysql.bicep' = if (create_database) {
   params: {
     location: location
     tags: getTags('Microsoft.DBforMySQL/flexibleServers', tags)
-    Name: db_name
+    // Name: db_name
     adminUser: adminUsername
     adminPassword: databaseAdminPassword
     subnetId: subnets.database.id
@@ -315,25 +320,24 @@ output filerInfoFinal types.filerInfo_t = {
 output cyclecloudPrincipalId string = infrastructureOnly ? '' : ccwVM.outputs.principalId
 
 output managedIdentityId string = infrastructureOnly ? '' : ccwManagedIdentity.outputs.managedIdentityId
-
-// Automatically inject the ccw and pyxis cluster init specs
+// Automatically inject the ccw and monitoring cluster init specs
 
 var ccwClusterInitSpec = {
   type: 'gitHubReleaseURL'
   gitHubReleaseURL: uri('https://github.com/Azure/cyclecloud-slurm-workspace/releases/tag/', projectVersion)
   spec: 'default'
-  target: ['login', 'scheduler', 'htc', 'hpc', 'gpu', 'dynamic']
+  target: ['login', 'scheduler', 'd64d', 'd16d', 'm64', 'hpc', 'gpu', 'dynamic']
 }
 
-var pyxisClusterInitSpec = {
+var monitoringClusterInitSpec = {
   type: 'gitHubReleaseURL'
-  gitHubReleaseURL: uri('https://github.com/Azure/cyclecloud-pyxis/releases/tag/', pyxisProjectVersion)
+  gitHubReleaseURL: uri('https://github.com/Azure/cyclecloud-monitoring/releases/tag/', monitoringProjectVersion)
   spec: 'default'
-  target: ['login', 'scheduler', 'htc', 'hpc', 'gpu', 'dynamic']
+  target: ['login', 'scheduler', 'd64d', 'd16d', 'm64', 'hpc', 'gpu', 'dynamic']
 }
 
-// Projects <= 2025.02.06 have the pyxis logic embedded in the ccw cluster init spec
-var requiredClusterInitSpecs = [ccwClusterInitSpec, pyxisClusterInitSpec]
+// Use of azslurm 4.0 does not require pyxis
+var requiredClusterInitSpecs = [ccwClusterInitSpec, monitoringClusterInitSpec]
 
 output clusterInitSpecs types.cluster_init_param_t = union(requiredClusterInitSpecs, clusterInitSpecs)
 
@@ -344,12 +348,24 @@ output schedulerNode types.scheduler_t = schedulerNode
 output loginNodes types.login_t = loginNodes
 
 output partitions types.partitions_t = {
-  htc: union({
-    sku: htc.sku
-    maxNodes: htc.maxNodes
-    osImage: htc.osImage
-    useSpot: htc.?useSpot ?? false
-  }, contains(htc,'availabilityZone') ? { availabilityZone: htc.?availabilityZone } : {})
+  d64d: union({
+    sku: d64d.sku
+    maxNodes: d64d.maxNodes
+    osImage: d64d.osImage
+    useSpot: d64d.?useSpot ?? false
+  }, contains(d64d,'availabilityZone') ? { availabilityZone: d64d.?availabilityZone } : {})
+  d16d: union({
+    sku: d16d.sku
+    maxNodes: d16d.maxNodes
+    osImage: d16d.osImage
+    useSpot: d16d.?useSpot ?? false
+  }, contains(d16d,'availabilityZone') ? { availabilityZone: d16d.?availabilityZone } : {})
+  m64: union({
+    sku: m64.sku
+    maxNodes: m64.maxNodes
+    osImage: m64.osImage
+    useSpot: m64.?useSpot ?? false
+  }, contains(m64,'availabilityZone') ? { availabilityZone: m64.?availabilityZone } : {})
   hpc: hpc
   gpu: gpu
 }
@@ -389,7 +405,7 @@ output manualInstall bool = manualInstall
 output acceptMarketplaceTerms bool = acceptMarketplaceTerms
 
 output ood object = union(ood, {
-  version: '1.0.1'
+  version: '1.1.0'
   nic: deployOOD ? oodNIC.outputs.NICId : ''
   managedIdentity: deployOOD ? createOODMI ? oodNewManagedIdentity.id : ood.?appManagedIdentityId : ''
   clientId: deployOOD ? registerOODApp ? oodApp.outputs.oodClientAppId : ood.?appId : ''
@@ -402,9 +418,16 @@ output oodManualRegistration object = {
   fqdn: deployOOD ? oodNIC.outputs.privateIp : ''
 }
 
+output monitoring object = {
+  ingestionEndpoint: monitoringIngestionEndpoint
+  identityClientId: monitoringIdentityClientId
+}
+output hubMI string = hubMI
+
 output files object = {
   availability_zones_json: loadTextContent('./files-to-load/encoded/availability_zones.json.base64')
   create_cc_param_py: loadTextContent('./files-to-load/encoded/create_cc_param.py.base64')
   cyclecloud_install_py: loadTextContent('./files-to-load/encoded/cyclecloud_install.py.base64')
   initial_params_json: loadTextContent('./files-to-load/encoded/initial_params.json.base64')
+  slurm_txt: loadTextContent('./files-to-load/encoded/slurm.txt.base64')
 }

bicep/exports.bicep

@@ -5,4 +5,5 @@ var role_lookup = {
   'Storage Account Contributor': resourceId('microsoft.authorization/roleDefinitions', '17d1049b-9a84-46fb-8f53-869881c3d3ab')
   'Storage Blob Data Contributor': resourceId('microsoft.authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')
   'Storage Blob Data Reader': resourceId('microsoft.authorization/roleDefinitions', '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1')
+  'Monitoring Metrics Publisher': resourceId('microsoft.authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb')
 }

bicep/files-to-load/create_cc_param.py

@@ -30,15 +30,13 @@ def set_slurm_params(params, dbPassword, outputs):
         params['SubnetId'] = '/'.join([outputs['vnet']['value']['rg'], outputs['vnet']['value']['name'], outputs['vnet']['value']['computeSubnetName']])
 
     # Define Availability Zone
-    params['DefineNodesAvailabilityZone'] = any('availabilityZone' in zoneList for zoneList in [outputs['partitions']['value']['htc'], outputs['partitions']['value']['hpc'], outputs['partitions']['value']['gpu']])
-
-    #HTC
-    params['HTCMachineType'] = outputs['partitions']['value']['htc']['sku']
-    params['MaxHTCExecuteNodeCount'] = int(outputs['partitions']['value']['htc']['maxNodes'])
-    params['HTCImageName'] = outputs['partitions']['value']['htc']['osImage']
-    params['HTCUseLowPrio'] = outputs['partitions']['value']['htc']['useSpot']
-    params['HTCAvailabilityZone'] = outputs['partitions']['value']['htc']['availabilityZone'] if params['DefineNodesAvailabilityZone'] and 'availabilityZone' in outputs['partitions']['value']['htc'] else None
+    params['DefineNodesAvailabilityZone'] = any('availabilityZone' in zoneList for zoneList in [outputs['partitions']['value']['hpc'], outputs['partitions']['value']['gpu']])
 
+    for na in ['D64D', 'D16D', 'M64']:
+        params[f'{na}MachineType'] = outputs['partitions']['value'][na.lower()]['sku']
+        params[f'Max{na}NodeCount'] = int(outputs['partitions']['value'][na.lower()]['maxNodes'])
+        params[f'{na}ImageName'] = outputs['partitions']['value'][na.lower()]['osImage']
+
     #HPC
     params['HPCMachineType'] = outputs['partitions']['value']['hpc']['sku']
     params['MaxHPCExecuteNodeCount'] = int(outputs['partitions']['value']['hpc']['maxNodes'])
@@ -96,6 +94,13 @@ def set_slurm_params(params, dbPassword, outputs):
         params['AdditionalNFSMountOptions'] = outputs['filerInfoFinal']['value']['additional']['mountOptions']
         params['AdditionalNFSAddress'] = outputs['filerInfoFinal']['value']['additional']['ipAddress']
 
+    # Monitoring
+    params['MonitoringEnabled'] = outputs['monitoring']["value"]['ingestionEndpoint'] != ''
+    params['MonitoringIngestionEndpoint'] = outputs['monitoring']['value']['ingestionEndpoint']
+    params['MonitoringIdentityClientId'] = outputs['monitoring']['value']['identityClientId']
+
+    params['ManagedIdentity'] = outputs['hubMI']['value']
+
 
 def set_ood_params(params, outputs):
     slurm_params = get_json_dict('initial_params.json')
@@ -119,6 +124,7 @@ def set_ood_params(params, outputs):
     params['ood_entra_tenant_id'] = outputs['ood']['value'].get('tenantId')
     params['ood_nic'] = outputs['ood']['value'].get('nic')
 
+
 class ClusterInitSpec:
     def __init__(self, project: str, version: str, spec: str, targets: typing.List[str]):
         self.project = project
@@ -197,7 +203,9 @@ def main():
         "login": "LoginClusterInitSpecs",
         "gpu": "GPUClusterInitSpecs",
         "hpc": "HPCClusterInitSpecs",
-        "htc": "HTCClusterInitSpecs",
+        "d64d": "D64DClusterInitSpecs",
+        "d16d": "D16DClusterInitSpecs",
+        "m64": "M64ClusterInitSpecs",
         "scheduler": "SchedulerClusterInitSpecs",
         "dynamic": "DynamicClusterInitSpecs",
         "ood": "ClusterInitSpecs"