Add 3 new nodearrays and synchronize to latest standard spoke settings

Author: ryanhamel

bicep/ccw.bicep

@@ -28,8 +28,9 @@ param clusterInitSpecs types.cluster_init_param_t
 param slurmSettings types.slurmSettings_t 
 param schedulerNode types.scheduler_t
 param loginNodes types.login_t
-param htc types.htc_t
-param htc2 types.htc_t
+param d64d types.htc_t
+param d16d types.htc_t
+param m64 types.htc_t
 param hpc types.hpc_t
 param gpu types.hpc_t
 param tags types.resource_tags_t
@@ -325,14 +326,14 @@ var ccwClusterInitSpec = {
   type: 'gitHubReleaseURL'
   gitHubReleaseURL: uri('https://github.com/Azure/cyclecloud-slurm-workspace/releases/tag/', projectVersion)
   spec: 'default'
-  target: ['login', 'scheduler', 'htc', 'htc2', 'hpc', 'gpu', 'dynamic']
+  target: ['login', 'scheduler', 'd64d', 'd16d', 'm64', 'hpc', 'gpu', 'dynamic']
 }
 
 var monitoringClusterInitSpec = {
   type: 'gitHubReleaseURL'
   gitHubReleaseURL: uri('https://github.com/Azure/cyclecloud-monitoring/releases/tag/', monitoringProjectVersion)
   spec: 'default'
-  target: ['login', 'scheduler', 'htc', 'htc2', 'hpc', 'gpu', 'dynamic']
+  target: ['login', 'scheduler', 'd64d', 'd16d', 'm64', 'hpc', 'gpu', 'dynamic']
 }
 
 // Use of azslurm 4.0 does not require pyxis
@@ -347,19 +348,24 @@ output schedulerNode types.scheduler_t = schedulerNode
 output loginNodes types.login_t = loginNodes
 
 output partitions types.partitions_t = {
-  htc: union({
-    sku: htc.sku
-    maxNodes: htc.maxNodes
-    osImage: htc.osImage
-    useSpot: htc.?useSpot ?? false
-  }, contains(htc,'availabilityZone') ? { availabilityZone: htc.?availabilityZone } : {})
-  htc2: union({
-    sku: htc2.sku
-    maxNodes: htc2.maxNodes
-    osImage: htc2.osImage
-    useSpot: htc2.?useSpot ?? false
-    // we aren't using availability zones for now, ignore this for htc2 TODO
-  }, contains(htc,'availabilityZone') ? { availabilityZone: htc.?availabilityZone } : {})
+  d64d: union({
+    sku: d64d.sku
+    maxNodes: d64d.maxNodes
+    osImage: d64d.osImage
+    useSpot: d64d.?useSpot ?? false
+  }, contains(d64d,'availabilityZone') ? { availabilityZone: d64d.?availabilityZone } : {})
+  d16d: union({
+    sku: d16d.sku
+    maxNodes: d16d.maxNodes
+    osImage: d16d.osImage
+    useSpot: d16d.?useSpot ?? false
+  }, contains(d16d,'availabilityZone') ? { availabilityZone: d16d.?availabilityZone } : {})
+  m64: union({
+    sku: m64.sku
+    maxNodes: m64.maxNodes
+    osImage: m64.osImage
+    useSpot: m64.?useSpot ?? false
+  }, contains(m64,'availabilityZone') ? { availabilityZone: m64.?availabilityZone } : {})
   hpc: hpc
   gpu: gpu
 }

bicep/files-to-load/create_cc_param.py

@@ -30,22 +30,13 @@ def set_slurm_params(params, dbPassword, outputs):
         params['SubnetId'] = '/'.join([outputs['vnet']['value']['rg'], outputs['vnet']['value']['name'], outputs['vnet']['value']['computeSubnetName']])
 
     # Define Availability Zone
-    params['DefineNodesAvailabilityZone'] = any('availabilityZone' in zoneList for zoneList in [outputs['partitions']['value']['htc'], outputs['partitions']['value']['hpc'], outputs['partitions']['value']['gpu']])
-    
-    #HTC
-    params['HTCMachineType'] = outputs['partitions']['value']['htc']['sku']
-    params['MaxHTCExecuteNodeCount'] = int(outputs['partitions']['value']['htc']['maxNodes'])
-    params['HTCImageName'] = outputs['partitions']['value']['htc']['osImage']
-    params['HTCUseLowPrio'] = outputs['partitions']['value']['htc']['useSpot']
-    params['HTCAvailabilityZone'] = outputs['partitions']['value']['htc']['availabilityZone'] if params['DefineNodesAvailabilityZone'] and 'availabilityZone' in outputs['partitions']['value']['htc'] else None
-    
-    #HTC2
-    params['HTC2MachineType'] = outputs['partitions']['value']['htc2']['sku']
-    params['MaxHTC2ExecuteNodeCount'] = int(outputs['partitions']['value']['htc2']['maxNodes'])
-    params['HTC2ImageName'] = outputs['partitions']['value']['htc2']['osImage']
-    params['HTC2UseLowPrio'] = outputs['partitions']['value']['htc2']['useSpot']
-    params['HTC2AvailabilityZone'] = outputs['partitions']['value']['htc2']['availabilityZone'] if params['DefineNodesAvailabilityZone'] and 'availabilityZone' in outputs['partitions']['value']['htc'] else None
+    params['DefineNodesAvailabilityZone'] = any('availabilityZone' in zoneList for zoneList in [outputs['partitions']['value']['hpc'], outputs['partitions']['value']['gpu']])
 
+    for na in ['D64D', 'D16D', 'M64']:
+        params[f'{na}MachineType'] = outputs['partitions']['value'][na.lower()]['sku']
+        params[f'Max{na}NodeCount'] = int(outputs['partitions']['value'][na.lower()]['maxNodes'])
+        params[f'{na}ImageName'] = outputs['partitions']['value'][na.lower()]['osImage']
+
     #HPC
     params['HPCMachineType'] = outputs['partitions']['value']['hpc']['sku']
     params['MaxHPCExecuteNodeCount'] = int(outputs['partitions']['value']['hpc']['maxNodes'])
@@ -212,8 +203,9 @@ def main():
         "login": "LoginClusterInitSpecs",
         "gpu": "GPUClusterInitSpecs",
         "hpc": "HPCClusterInitSpecs",
-        "htc": "HTCClusterInitSpecs",
-        "htc2": "HTC2ClusterInitSpecs",
+        "d64d": "D64DClusterInitSpecs",
+        "d16d": "D16DClusterInitSpecs",
+        "m64": "M64ClusterInitSpecs",
         "scheduler": "SchedulerClusterInitSpecs",
         "dynamic": "DynamicClusterInitSpecs",
         "ood": "ClusterInitSpecs"

bicep/files-to-load/slurm.txt

@@ -229,44 +229,58 @@ Autoscale = $Autoscale
         slurm.hpc = true
         slurm.partition = hpc
 
-    [[nodearray htc]]
+    [[nodearray d64d]]
     Extends = nodearraybase
-    MachineType = $HTCMachineType
-    ImageName = $HTCImageName
-    MaxCount = $MaxHTCExecuteNodeCount
+    MachineType = $D64DMachineType
+    ImageName = $D64DImageName
+    MaxCount = $MaxD64DNodeCount
 
-    Interruptible = $HTCUseLowPrio
-    MaxPrice = $HTCSpotMaxPrice
-    AdditionalClusterInitSpecs = $HTCClusterInitSpecs
+    AdditionalClusterInitSpecs = $D64DClusterInitSpecs
 
         [[[configuration]]]
         slurm.hpc = false
-        slurm.partition = htc
+        slurm.partition = d64d
         # set pcpu = false for all hyperthreaded VMs
         slurm.use_pcpu = false
 
-    [[nodearray htc2]]
+    [[nodearray d16d]]
     Extends = nodearraybase
-    MachineType = $HTC2MachineType
-    ImageName = $HTC2ImageName
-    MaxCount = $MaxHTC2ExecuteNodeCount
+    MachineType = $D16DMachineType
+    ImageName = $D16DImageName
+    MaxCount = $MaxD16DNodeCount
 
-    Interruptible = $HTC2UseLowPrio
-    MaxPrice = $HTC2SpotMaxPrice
-    AdditionalClusterInitSpecs = $HTC2ClusterInitSpecs
+    AdditionalClusterInitSpecs = $D16DClusterInitSpecs
 
         [[[configuration]]]
         slurm.hpc = false
-        slurm.partition = htc2
+        slurm.partition = d16d
+        # set pcpu = false for all hyperthreaded VMs
+        slurm.use_pcpu = false
+    
+    [[nodearray m64]]
+    Extends = nodearraybase
+    MachineType = $M64MachineType
+    ImageName = $M64ImageName
+    MaxCount = $MaxM64NodeCount
+
+    AdditionalClusterInitSpecs = $M64ClusterInitSpecs
+
+        [[[configuration]]]
+        slurm.hpc = false
+        slurm.partition = m64
         # set pcpu = false for all hyperthreaded VMs
         slurm.use_pcpu = false
 
     [[nodearray gpu]]
         CloudInit="""#!/bin/bash
 sed -i 's/^SHELL.*/SHELL=\/bin\/bash/g' /etc/default/useradd
     """
+
+    # GB200: Peregrine disallows SinglePlacementGroup, but pkey requires SingleScaleset
     Azure.Overprovision = false
     Azure.MaxScaleSetSize = 1000
+    Azure.SingleScaleset = true
+    Azure.SinglePlacementGroup = false
 
     Extends = nodearraybase
     MachineType = $GPUMachineType
@@ -347,17 +361,23 @@ Order = 10
         ParameterType = Cloud.MachineType
         DefaultValue = Standard_F2s_v2
 
-        [[[parameter HTCMachineType]]]
-        Label = HTC VM Type
-        Description = The VM type for HTC execute nodes
+        [[[parameter D64DMachineType]]]
+        Label = D64D VM Type
+        Description = The VM type for D64D execute nodes
         ParameterType = Cloud.MachineType
-        DefaultValue = Standard_F2s_v2
+        DefaultValue = Standard_D64ds_v5
 
-        [[[parameter HTC2MachineType]]]
-        Label = HTC2 VM Type
-        Description = The VM type for HTC2 execute nodes
+        [[[parameter D16DMachineType]]]
+        Label = D16D VM Type
+        Description = The VM type for D16D execute nodes
         ParameterType = Cloud.MachineType
-        DefaultValue = Standard_F2s_v2
+        DefaultValue = Standard_D16ds_v5
+
+        [[[parameter M64MachineType]]]
+        Label = M64 VM Type
+        Description = The VM type for M64 execute nodes
+        ParameterType = Cloud.MachineType
+        DefaultValue = Standard_M64s_v2
 
         [[[parameter GPUMachineType]]]
         Label = GPU VM Type
@@ -382,17 +402,25 @@ Order = 10
         Widget.Plugin = pico.form.BooleanCheckBox
         Widget.Label = Start and stop execute instances automatically
 
-        [[[parameter MaxHTCExecuteNodeCount]]]
-        Label = Max HTC Nodes
-        Description = The total number of HTC execute nodes to start
+        [[[parameter MaxD64DNodeCount]]]
+        Label = Max D64D Nodes
+        Description = The total number of D64D execute nodes to start
         DefaultValue = 100
         Config.Plugin = pico.form.NumberTextBox
         Config.MinValue = 0
         Config.IntegerOnly = true
 
-        [[[parameter Max2HTCExecuteNodeCount]]]
-        Label = Max HTC2 Nodes
-        Description = The total number of HTC2 execute nodes to start
+        [[[parameter MaxD16DNodeCount]]]
+        Label = Max D16D Nodes
+        Description = The total number of D16D execute nodes to start
+        DefaultValue = 100
+        Config.Plugin = pico.form.NumberTextBox
+        Config.MinValue = 0
+        Config.IntegerOnly = true
+
+        [[[parameter MaxM64NodeCount]]]
+        Label = Max M64 Nodes
+        Description = The total number of M64 execute nodes to start
         DefaultValue = 100
         Config.Plugin = pico.form.NumberTextBox
         Config.MinValue = 0
@@ -430,36 +458,6 @@ Order = 10
         Config.MinValue = 1
         Config.IntegerOnly = true
 
-
-        [[[parameter HTCUseLowPrio]]]
-        Label = HTC Spot
-        DefaultValue = false
-        Widget.Plugin = pico.form.BooleanCheckBox
-        Widget.Label = Use Spot VMs for HTC execute hosts
-
-        [[[parameter HTC2UseLowPrio]]]
-        Label = HTC2 Spot
-        DefaultValue = false
-        Widget.Plugin = pico.form.BooleanCheckBox
-        Widget.Label = Use Spot VMs for HTC execute hosts
-
-        [[[parameter HTCSpotMaxPrice]]]
-        Label = Max Price HTC
-        DefaultValue = -1
-        Description = Max price for Spot VMs in USD (value of -1 will not evict based on price)
-        Config.Plugin = pico.form.NumberTextBox
-        Conditions.Excluded := HTCUseLowPrio isnt true
-        Config.MinValue = -1
-
-        [[[parameter HTC2SpotMaxPrice]]]
-        Label = Max Price HTC2
-        DefaultValue = -1
-        Description = Max price for Spot VMs in USD (value of -1 will not evict based on price)
-        Config.Plugin = pico.form.NumberTextBox
-        Conditions.Excluded := HTC2UseLowPrio isnt true
-        Config.MinValue = -1
-
-
         [[[parameter GPUUseLowPrio]]]
         Label = GPU Spot
         DefaultValue = false
@@ -845,6 +843,7 @@ Order = 20
         Label = Slurm Configuration
         Description = Any additional lines to add to slurm.conf
         ParameterType = Text
+        DefaultValue = "SuspendExcParts=gpu"
 
         [[[parameter configuration_slurm_launch_parameters]]]
         Label = Launch Parameters
@@ -905,20 +904,26 @@ Order = 20
         DefaultValue = almalinux8
         Config.Filter := Package in {"cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"}
 
-        [[[parameter HTCImageName]]]
-        Label = HTC OS
+        [[[parameter D64DImageName]]]
+        Label = D64D OS
         ParameterType = Cloud.Image
         Config.OS = linux
         DefaultValue = almalinux8
         Config.Filter := Package in {"cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"}
 
-        [[[parameter HTC2ImageName]]]
-        Label = HTC2 OS
+        [[[parameter D16DImageName]]]
+        Label = D16D OS
         ParameterType = Cloud.Image
         Config.OS = linux
         DefaultValue = almalinux8
         Config.Filter := Package in {"cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"}
 
+        [[[parameter M64ImageName]]]
+        Label = M64 OS
+        ParameterType = Cloud.Image
+        Config.OS = linux
+        DefaultValue = almalinux8
+        Config.Filter := Package in {"cycle.image.ubuntu20", "cycle.image.ubuntu22", "cycle.image.sles15-hpc", "almalinux8"}
 
         [[[parameter GPUImageName]]]
         Label = GPU OS
@@ -946,16 +951,22 @@ Order = 20
         Description = Cluster init specs to apply to Login nodes
         ParameterType = Cloud.ClusterInitSpecs
 
-        [[[parameter HTCClusterInitSpecs]]]
-        Label = HTC Cluster-Init
+        [[[parameter D64DClusterInitSpecs]]]
+        Label = D64D Cluster-Init
+        DefaultValue = =undefined
+        Description = Cluster init specs to apply to D64D execute nodes
+        ParameterType = Cloud.ClusterInitSpecs
+
+        [[[parameter D16DClusterInitSpecs]]]
+        Label = D16D Cluster-Init
         DefaultValue = =undefined
-        Description = Cluster init specs to apply to HTC execute nodes
+        Description = Cluster init specs to apply to D16D execute nodes
         ParameterType = Cloud.ClusterInitSpecs
 
-        [[[parameter HTC2ClusterInitSpecs]]]
-        Label = HTC2 Cluster-Init
+        [[[parameter M64ClusterInitSpecs]]]
+        Label = M64 Cluster-Init
         DefaultValue = =undefined
-        Description = Cluster init specs to apply to HTC2 execute nodes
+        Description = Cluster init specs to apply to M64 execute nodes
         ParameterType = Cloud.ClusterInitSpecs
 
         [[[parameter HPCClusterInitSpecs]]]

bicep/hub/deploy_spoke.sh

@@ -136,7 +136,7 @@ MOUNT_OPTIONS=$(jq -r '.mountOptions.value' outputs/hub-anf-outputs.json)
 replace_fields ".sharedFilesystem={ value: { type: \"nfs-existing\", ipAddress: \"$IP_ADDRESS\", exportPath: \"$EXPORT_PATH\", mountOptions: \"$MOUNT_OPTIONS\" } }"
 
 # new vnet 
-ADDRESS_SPACE="10.${SPOKE_NUMBER}.0.0/24"
+ADDRESS_SPACE="10.${SPOKE_NUMBER}.0.0/20"
 replace_fields ".network.value.addressSpace=\"$ADDRESS_SPACE\""
 
 # vnet to peer 
@@ -173,10 +173,9 @@ az deployment sub create \
     --parameters "$(pwd)/spoke_params.json" \
     --parameters location="$LOCATION" \
     --parameters resourceGroup="${SPOKE_RG_NAME}" \
-    --parameters ccVMName="ccw-${SPOKE_NUMBER}-cyclecloud-vm" \
-    --parameters clusterName="ccw-${SPOKE_NUMBER}" \
+    --parameters ccVMName="ccw${SPOKE_NUMBER}-cyclecloud-vm" \
+    --parameters clusterName="ccw${SPOKE_NUMBER}" \
     --parameters monitoringIngestionEndpoint="${MONITORING_INGESTION_ENDPOINT}" \
     --name "spoke-ccw-0${SPOKE_DEPLOYMENT_NAME}" \
     $WHATIF_FLAG
-