From 30c663e9c36cc1630101f0835154759e07caa85c Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 19 Mar 2025 22:26:26 +0100 Subject: [PATCH 01/24] Parse sinfo output to manage resources --- .../commands/slurm_manage_resources.py | 109 ++++++++++++++++++ coldfront/plugins/slurm/utils.py | 14 +++ 2 files changed, 123 insertions(+) create mode 100644 coldfront/plugins/slurm/management/commands/slurm_manage_resources.py diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py new file mode 100644 index 000000000..c0ac169e9 --- /dev/null +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -0,0 +1,109 @@ +import logging +import os +import re +from functools import reduce +from cProfile import Profile + +from django.core.management.base import BaseCommand, CommandError + +from coldfront.core.resource.models import ResourceType, ResourceAttribute, ResourceAttributeType, AttributeType, Resource +from coldfront.core.project.models import Project +from coldfront.plugins.slurm.utils import slurm_get_nodes_info +from django.utils.datetime_safe import datetime + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = 'Manage slurm resources from sinfo output' + + def get_output_from_file(self, file_path): + try: + keys = None + with open(file_path, 'r') as output_file: + for line in output_file: + if keys is None: + keys = re.sub(r'\s+', ' ', line).strip().lower().split(' ') + else: + values = re.sub(r'\s+', ' ', line).strip().split(' ') + yield dict(zip(keys, values)) + except FileNotFoundError: + print(f"File at {file_path} does not exist. Cant simulate output.") + except IOError as e: + print(f"An error occurred: {e}") + + + def add_arguments(self, parser): + parser.add_argument("-e", "--environment", help="Environment, use dev to simulate output") + parser.add_argument('--profile', action='store_true', default=False) + + def handle(self, *args, **options): + if options.get('profile', False): + profiler = Profile() + profiler.runcall(self._handle, *args, **options) + profiler.print_stats() + else: + self._handle(*args, **options) + + def _handle(self, *args, **options): + def calculate_gpu_count(gres_value): + if 'null' in gres_value: + return 0 + gpu_list = gres_value.split(',') + return reduce(lambda x, y: x + y,[int(gpu_info.split(':')[2].replace('(S','')) for gpu_info in gpu_list]) + + def calculate_cpu_count(row): + if row.get('S:C:T', None) is None: + return 0 + cpu_count = row.get('S:C:T').split(':')[1] + return int(cpu_count) + + def calculate_owner_value(project_list, row): + owner_name = '' + project_name_list = [project.title for project in project_list] + owner_lists = row.get('groups', '').split(',') + owner_project = [name_owner for name_owner in owner_lists if name_owner in project_name_list] + if len(owner_project) > 0: + return owner_project[0] + if {'cluster_users', 'slurm-admin'}.issubset(set(owner_lists)): + return'FASRC' + return owner_name + + env = options['environment'] or 'production' + if 'dev' in env: + output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo.txt')) + else: + output = slurm_get_nodes_info() + print(f'Running on {env} mode') + project_list = Project.objects.all() + compute_node, compute_node_created = ResourceType.objects.get_or_create(name='Compute Node', description='Compute Node') + partition_resource_type, partition_created = ResourceType.objects.get_or_create(name='Cluster Partition', description='Cluster Partition') + int_attribute_type = AttributeType.objects.get(name='Int') + text_attribute_type = AttributeType.objects.get(name='Text') + gpu_count_attribute_type, gpu_count_created = ResourceAttributeType.objects.get_or_create(name='GPU Count', defaults={'attribute_type': int_attribute_type}) + core_count_attribute_type, core_count_created = ResourceAttributeType.objects.get_or_create(name='Core Count', defaults={'attribute_type': int_attribute_type}) + features_attribute_type, features_created = ResourceAttributeType.objects.get_or_create(name='Features', defaults={'attribute_type': text_attribute_type}) + owner_attribute_type, owner_created = ResourceAttributeType.objects.get_or_create(name='Owner', defaults={'attribute_type': text_attribute_type}) + service_end_attribute_type, service_end_created = ResourceAttributeType.objects.get_or_create(name='ServiceEnd', defaults={'attribute_type': text_attribute_type}) + processed_resources = set() + bulk_process_resource_attribute = [] + bulk_update_resource = [] + for row in output: + new_resource, compute_node_created_created = Resource.objects.get_or_create(name=row['nodelist'], defaults={'is_allocatable':False, 'resource_type':compute_node}) + Resource.objects.get_or_create(name=row['partition'], defaults={'resource_type':partition_resource_type}) + bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres']))) + bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=core_count_attribute_type, resource=new_resource, value=calculate_cpu_count(row))) + bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=features_attribute_type, resource=new_resource, value=row.get('avail_features', '(null)'))) + bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=owner_attribute_type, resource=new_resource, value=calculate_owner_value(project_list, row))) + if new_resource.is_available is False: + bulk_update_resource.append(Resource(name=row['nodelist'], is_available=True)) + bulk_process_resource_attribute.append(ResourceAttribute(resource=new_resource, value=' ', resource_attribute_type=service_end_attribute_type)) + processed_resources.add(new_resource.name) + ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) + Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + bulk_process_resource_attribute = [] + bulk_update_resource = [] + for resource_to_delete in Resource.objects.exclude(name__in=list(processed_resources)).filter(is_available=True, resource_type=compute_node): + bulk_update_resource.append(Resource(name=resource_to_delete.name, is_available=False)) + bulk_process_resource_attribute.append(ResourceAttribute(resource=resource_to_delete, value=str(datetime.now()), resource_attribute_type=service_end_attribute_type)) + ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) + Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) \ No newline at end of file diff --git a/coldfront/plugins/slurm/utils.py b/coldfront/plugins/slurm/utils.py index 4cf3c4fa2..127600bee 100644 --- a/coldfront/plugins/slurm/utils.py +++ b/coldfront/plugins/slurm/utils.py @@ -1,4 +1,5 @@ import logging +import re import shlex import struct import subprocess @@ -37,6 +38,7 @@ SLURM_CMD_LIST_PARTITIONS = SLURM_SCONTROL_PATH + ' show partitions' SLURM_CMD_GET_USER_INFO = SLURM_SSHARE_PATH + ' -u {} -A {} -o Account,User,RawShares,NormShares,RawUsage,FairShare --parsable' +SLURM_CMD_SINFO_NODES = 'sinfo -N -r --format="%19N %23P %66G %5a %5c %8z %65f %50g"' logger = logging.getLogger(__name__) @@ -145,6 +147,18 @@ def slurm_block_account(cluster, account, noop=False): cmd = SLURM_CMD_BLOCK_ACCOUNT.format(shlex.quote(account), shlex.quote(cluster)) _run_slurm_cmd(cmd, noop=noop) +def slurm_get_nodes_info(): + def table_to_dict_list(table_info): + keys = re.sub(r'\s+', ' ', table_info[0]).strip().lower().split(' ') + values = [re.sub(r'\s+', ' ', row).strip().split(' ') for row in table_info[1:] if len(row)>0] + result_list = [dict(zip(keys, row)) for row in values if len(row)> 0] + return result_list + + cmd = SLURM_CMD_SINFO_NODES + nodes_info = _run_slurm_cmd(cmd, noop=False) + nodes_info = nodes_info.decode('utf-8').split('\n') + return table_to_dict_list(nodes_info) + def slurm_check_assoc(user, cluster, account): cmd = SLURM_CMD_CHECK_ASSOCIATION.format( shlex.quote(user), shlex.quote(cluster), shlex.quote(account) From 818867c96fd1c20be8cc8f099c393b65f07a2652 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Fri, 21 Mar 2025 23:00:43 +0100 Subject: [PATCH 02/24] Add resourceattrtypes to defauls command. Fix parsing of gpu count. Prevent bulk update error --- .../commands/add_resource_defaults.py | 2 +- .../commands/slurm_manage_resources.py | 38 +++++++++++-------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index 473dd3b6b..193201f7f 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -43,7 +43,7 @@ def handle(self, *args, **options): ('xdmod_resource', 'Text'), # ('eula', 'Text'), # ('OnDemand','Yes/No'), - # ('ServiceEnd', 'Date'), + ('ServiceEnd', 'Date'), # ('ServiceStart', 'Date'), ('slurm_cluster', 'Text'), ('slurm_specs', 'Attribute Expanded Text'), diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py index c0ac169e9..67efdc652 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -31,7 +31,6 @@ def get_output_from_file(self, file_path): except IOError as e: print(f"An error occurred: {e}") - def add_arguments(self, parser): parser.add_argument("-e", "--environment", help="Environment, use dev to simulate output") parser.add_argument('--profile', action='store_true', default=False) @@ -48,7 +47,7 @@ def _handle(self, *args, **options): def calculate_gpu_count(gres_value): if 'null' in gres_value: return 0 - gpu_list = gres_value.split(',') + gpu_list = gres_value.split('),') return reduce(lambda x, y: x + y,[int(gpu_info.split(':')[2].replace('(S','')) for gpu_info in gpu_list]) def calculate_cpu_count(row): @@ -77,13 +76,11 @@ def calculate_owner_value(project_list, row): project_list = Project.objects.all() compute_node, compute_node_created = ResourceType.objects.get_or_create(name='Compute Node', description='Compute Node') partition_resource_type, partition_created = ResourceType.objects.get_or_create(name='Cluster Partition', description='Cluster Partition') - int_attribute_type = AttributeType.objects.get(name='Int') - text_attribute_type = AttributeType.objects.get(name='Text') - gpu_count_attribute_type, gpu_count_created = ResourceAttributeType.objects.get_or_create(name='GPU Count', defaults={'attribute_type': int_attribute_type}) - core_count_attribute_type, core_count_created = ResourceAttributeType.objects.get_or_create(name='Core Count', defaults={'attribute_type': int_attribute_type}) - features_attribute_type, features_created = ResourceAttributeType.objects.get_or_create(name='Features', defaults={'attribute_type': text_attribute_type}) - owner_attribute_type, owner_created = ResourceAttributeType.objects.get_or_create(name='Owner', defaults={'attribute_type': text_attribute_type}) - service_end_attribute_type, service_end_created = ResourceAttributeType.objects.get_or_create(name='ServiceEnd', defaults={'attribute_type': text_attribute_type}) + gpu_count_attribute_type = ResourceAttributeType.objects.get(name='GPU Count') + core_count_attribute_type = ResourceAttributeType.objects.get(name='Core Count') + features_attribute_type = ResourceAttributeType.objects.get(name='Features') + owner_attribute_type = ResourceAttributeType.objects.get(name='Owner') + service_end_attribute_type = ResourceAttributeType.objects.get(name='ServiceEnd') processed_resources = set() bulk_process_resource_attribute = [] bulk_update_resource = [] @@ -95,15 +92,24 @@ def calculate_owner_value(project_list, row): bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=features_attribute_type, resource=new_resource, value=row.get('avail_features', '(null)'))) bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=owner_attribute_type, resource=new_resource, value=calculate_owner_value(project_list, row))) if new_resource.is_available is False: - bulk_update_resource.append(Resource(name=row['nodelist'], is_available=True)) - bulk_process_resource_attribute.append(ResourceAttribute(resource=new_resource, value=' ', resource_attribute_type=service_end_attribute_type)) + bulk_update_resource.append(Resource(name=row['nodelist'], is_available=True, resource_type=compute_node)) + bulk_process_resource_attribute.append(ResourceAttribute(resource=new_resource, value=None, resource_attribute_type=service_end_attribute_type)) processed_resources.add(new_resource.name) - ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) - Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + try: + ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) + Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + except Exception as e: + logger.error(f'Error processing resources info: {str(e)}') + raise bulk_process_resource_attribute = [] bulk_update_resource = [] for resource_to_delete in Resource.objects.exclude(name__in=list(processed_resources)).filter(is_available=True, resource_type=compute_node): - bulk_update_resource.append(Resource(name=resource_to_delete.name, is_available=False)) + resource_to_delete.is_available = False + bulk_update_resource.append(resource_to_delete) bulk_process_resource_attribute.append(ResourceAttribute(resource=resource_to_delete, value=str(datetime.now()), resource_attribute_type=service_end_attribute_type)) - ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) - Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) \ No newline at end of file + try: + ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) + Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + except Exception as e: + logger.error(f'Error cleaning up resources: {str(e)}') + raise \ No newline at end of file From f64bcd5dc6ff93018a2e324271e160b048e41d68 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 26 Mar 2025 13:13:40 +0100 Subject: [PATCH 03/24] Use bulk_update_with_history and bulk_create_with_history to generate change history --- .../commands/slurm_manage_resources.py | 110 ++++++++++++++---- 1 file changed, 87 insertions(+), 23 deletions(-) diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py index 67efdc652..af39ab3b7 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -5,6 +5,7 @@ from cProfile import Profile from django.core.management.base import BaseCommand, CommandError +from simple_history.utils import bulk_update_with_history, bulk_create_with_history from coldfront.core.resource.models import ResourceType, ResourceAttribute, ResourceAttributeType, AttributeType, Resource from coldfront.core.project.models import Project @@ -27,9 +28,9 @@ def get_output_from_file(self, file_path): values = re.sub(r'\s+', ' ', line).strip().split(' ') yield dict(zip(keys, values)) except FileNotFoundError: - print(f"File at {file_path} does not exist. Cant simulate output.") + logger.error(f"File at {file_path} does not exist. Cant simulate output.") except IOError as e: - print(f"An error occurred: {e}") + logger.error(f"An error occurred: {e}") def add_arguments(self, parser): parser.add_argument("-e", "--environment", help="Environment, use dev to simulate output") @@ -51,9 +52,9 @@ def calculate_gpu_count(gres_value): return reduce(lambda x, y: x + y,[int(gpu_info.split(':')[2].replace('(S','')) for gpu_info in gpu_list]) def calculate_cpu_count(row): - if row.get('S:C:T', None) is None: + if row.get('s:c:t', None) is None: return 0 - cpu_count = row.get('S:C:T').split(':')[1] + cpu_count = row.get('s:c:t').split(':')[1] return int(cpu_count) def calculate_owner_value(project_list, row): @@ -69,47 +70,110 @@ def calculate_owner_value(project_list, row): env = options['environment'] or 'production' if 'dev' in env: - output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo.txt')) + output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo_output.txt')) else: output = slurm_get_nodes_info() - print(f'Running on {env} mode') + logger.debug(f'Running on {env} mode') project_list = Project.objects.all() - compute_node, compute_node_created = ResourceType.objects.get_or_create(name='Compute Node', description='Compute Node') - partition_resource_type, partition_created = ResourceType.objects.get_or_create(name='Cluster Partition', description='Cluster Partition') + compute_node = ResourceType.objects.get(name='Compute Node') + attribute_type_name_list = ['GPU Count', 'Core Count', 'Features', 'Owner', 'ServiceEnd'] + partition_resource_type = ResourceType.objects.get(name='Cluster Partition') gpu_count_attribute_type = ResourceAttributeType.objects.get(name='GPU Count') core_count_attribute_type = ResourceAttributeType.objects.get(name='Core Count') features_attribute_type = ResourceAttributeType.objects.get(name='Features') owner_attribute_type = ResourceAttributeType.objects.get(name='Owner') service_end_attribute_type = ResourceAttributeType.objects.get(name='ServiceEnd') + existing_resource_attributes = list(ResourceAttribute.objects.filter( + resource_attribute_type__name__in=attribute_type_name_list, + resource__resource_type__name='Compute Node' + ).values_list('pk', 'resource__name', 'resource_attribute_type__name') + ) + existing_resource_attributes_check = [f'{resource_att[1]} {resource_att[2]}' for resource_att in existing_resource_attributes] + existing_resource_attributes_pk_map = {f'{resource_att[1]} {resource_att[2]}': resource_att[0] for resource_att in existing_resource_attributes} processed_resources = set() - bulk_process_resource_attribute = [] + bulk_update_resource_attribute = [] + bulk_create_resource_attribute = [] bulk_update_resource = [] + processed_resource_attribute = [] for row in output: new_resource, compute_node_created_created = Resource.objects.get_or_create(name=row['nodelist'], defaults={'is_allocatable':False, 'resource_type':compute_node}) Resource.objects.get_or_create(name=row['partition'], defaults={'resource_type':partition_resource_type}) - bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres']))) - bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=core_count_attribute_type, resource=new_resource, value=calculate_cpu_count(row))) - bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=features_attribute_type, resource=new_resource, value=row.get('avail_features', '(null)'))) - bulk_process_resource_attribute.append(ResourceAttribute(resource_attribute_type=owner_attribute_type, resource=new_resource, value=calculate_owner_value(project_list, row))) + + gpu_count = ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres'])) + gpu_count_key = f"{row['nodelist']} {gpu_count_attribute_type.name}" + if gpu_count_key in existing_resource_attributes_check: + gpu_count.pk = existing_resource_attributes_pk_map[gpu_count_key] + bulk_update_resource_attribute.append(gpu_count) + else: + if gpu_count_key not in processed_resource_attribute: + bulk_create_resource_attribute.append(gpu_count) + processed_resource_attribute.append(gpu_count_key) + + core_count = ResourceAttribute(resource_attribute_type=core_count_attribute_type, resource=new_resource, value=calculate_cpu_count(row)) + core_count_key = f"{row['nodelist']} {core_count_attribute_type.name}" + if core_count_key in existing_resource_attributes_check: + core_count.pk = existing_resource_attributes_pk_map[core_count_key] + bulk_update_resource_attribute.append(core_count) + else: + if core_count_key not in processed_resource_attribute: + bulk_create_resource_attribute.append(core_count) + processed_resource_attribute.append(core_count_key) + + features = ResourceAttribute(resource_attribute_type=features_attribute_type, resource=new_resource, value=row.get('avail_features', '(null)')) + features_key = f"{row['nodelist']} {features_attribute_type.name}" + if features_key in existing_resource_attributes_check: + features.pk = existing_resource_attributes_pk_map[features_key] + bulk_update_resource_attribute.append(features) + else: + if features_key not in processed_resource_attribute: + bulk_create_resource_attribute.append(features) + processed_resource_attribute.append(features_key) + + owner = ResourceAttribute(resource_attribute_type=owner_attribute_type, resource=new_resource, value=calculate_owner_value(project_list, row)) + owner_key = f"{row['nodelist']} {owner_attribute_type.name}" + if owner_key in existing_resource_attributes_check: + owner.pk = existing_resource_attributes_pk_map[owner_key] + bulk_update_resource_attribute.append(owner) + else: + if owner_key not in processed_resource_attribute: + bulk_create_resource_attribute.append(owner) + processed_resource_attribute.append(owner_key) + if new_resource.is_available is False: - bulk_update_resource.append(Resource(name=row['nodelist'], is_available=True, resource_type=compute_node)) - bulk_process_resource_attribute.append(ResourceAttribute(resource=new_resource, value=None, resource_attribute_type=service_end_attribute_type)) + new_resource.is_available = True + bulk_update_resource.append(new_resource) + service_end_pk = existing_resource_attributes_pk_map[f"{row['nodelist']} {service_end_attribute_type.name}"] + bulk_update_resource_attribute.append(ResourceAttribute(resource=new_resource, value=None, resource_attribute_type=service_end_attribute_type, pk=service_end_pk)) processed_resources.add(new_resource.name) try: - ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) - Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + logger.debug(f'Updating {len(bulk_update_resource_attribute)} ResourceAttribute records') + bulk_update_with_history(bulk_update_resource_attribute, ResourceAttribute, ['value'], batch_size=500, default_change_reason='slurm_manage_resource command') + logger.debug(f'Updating {len(bulk_update_resource)} Resource records') + bulk_update_with_history(bulk_update_resource, Resource, ['is_available'], batch_size=500, default_change_reason='slurm_manage_resource command') + logger.debug(f'Creating {len(bulk_create_resource_attribute)} ResourceAttribute records') + bulk_create_with_history(bulk_create_resource_attribute, ResourceAttribute, batch_size=500, default_change_reason='slurm_manage_resource command') except Exception as e: - logger.error(f'Error processing resources info: {str(e)}') + logger.debug(f'Error processing resources info: {str(e)}') raise - bulk_process_resource_attribute = [] + bulk_update_resource_attribute = [] + bulk_create_resource_attribute = [] bulk_update_resource = [] for resource_to_delete in Resource.objects.exclude(name__in=list(processed_resources)).filter(is_available=True, resource_type=compute_node): resource_to_delete.is_available = False bulk_update_resource.append(resource_to_delete) - bulk_process_resource_attribute.append(ResourceAttribute(resource=resource_to_delete, value=str(datetime.now()), resource_attribute_type=service_end_attribute_type)) + service_end = ResourceAttribute(resource=resource_to_delete, value=str(datetime.now()), resource_attribute_type=service_end_attribute_type) + if f"{resource_to_delete.name} {service_end_attribute_type.name}" in existing_resource_attributes_check: + service_end.pk = existing_resource_attributes_pk_map[f"{resource_to_delete.name} {service_end_attribute_type.name}"] + bulk_update_resource_attribute.append(service_end) + else: + bulk_create_resource_attribute.append(service_end) try: - ResourceAttribute.objects.bulk_create(bulk_process_resource_attribute, update_conflicts=True, unique_fields=[], update_fields=['value']) - Resource.objects.bulk_create(bulk_update_resource, update_conflicts=True, unique_fields=[], update_fields=['is_available']) + logger.debug(f'Decommissioning {len(bulk_update_resource)} Resource records') + bulk_update_with_history(bulk_update_resource, Resource, ['is_available'], batch_size=500, default_change_reason='slurm_manage_resource command') + logger.debug(f'Creating {len(bulk_create_resource_attribute)} ServiceEnd ResourceAttribute records') + bulk_create_with_history(bulk_create_resource_attribute, ResourceAttribute, batch_size=500, default_change_reason='slurm_manage_resource command') + logger.debug(f'Updating {len(bulk_update_resource_attribute)} ServiceEnd ResourceAttribute records') + bulk_update_with_history(bulk_update_resource_attribute, ResourceAttribute, ['value'], batch_size=500, default_change_reason='slurm_manage_resource command') except Exception as e: logger.error(f'Error cleaning up resources: {str(e)}') - raise \ No newline at end of file + raise From 7d9e4c746d7c594a88277de1c74501c46ab5cc48 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Fri, 28 Mar 2025 17:22:11 +0100 Subject: [PATCH 04/24] Add current cluster as parent resoruce for compute nodes --- coldfront/core/resource/models.py | 2 + .../commands/slurm_manage_resources.py | 55 +++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/coldfront/core/resource/models.py b/coldfront/core/resource/models.py index b37f2e7b3..5662415af 100644 --- a/coldfront/core/resource/models.py +++ b/coldfront/core/resource/models.py @@ -3,7 +3,9 @@ from django.db import models from django.conf import settings from django.contrib.auth.models import Group +from django.utils.translation import gettext_lazy as _ from django.core.exceptions import ValidationError, ObjectDoesNotExist +from model_utils.fields import AutoLastModifiedField from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py index af39ab3b7..629f4bf70 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -3,14 +3,14 @@ import re from functools import reduce from cProfile import Profile +from django.utils import timezone -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from simple_history.utils import bulk_update_with_history, bulk_create_with_history from coldfront.core.resource.models import ResourceType, ResourceAttribute, ResourceAttributeType, AttributeType, Resource from coldfront.core.project.models import Project from coldfront.plugins.slurm.utils import slurm_get_nodes_info -from django.utils.datetime_safe import datetime logger = logging.getLogger(__name__) @@ -68,13 +68,19 @@ def calculate_owner_value(project_list, row): return'FASRC' return owner_name + def get_cluster(): + return Resource.objects.get(resource_type__name='Cluster') + + env = options['environment'] or 'production' if 'dev' in env: - output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo_output.txt')) + output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo.txt')) else: output = slurm_get_nodes_info() logger.debug(f'Running on {env} mode') + modify_history_date = timezone.now() project_list = Project.objects.all() + current_cluster = get_cluster() compute_node = ResourceType.objects.get(name='Compute Node') attribute_type_name_list = ['GPU Count', 'Core Count', 'Features', 'Owner', 'ServiceEnd'] partition_resource_type = ResourceType.objects.get(name='Cluster Partition') @@ -96,7 +102,14 @@ def calculate_owner_value(project_list, row): bulk_update_resource = [] processed_resource_attribute = [] for row in output: - new_resource, compute_node_created_created = Resource.objects.get_or_create(name=row['nodelist'], defaults={'is_allocatable':False, 'resource_type':compute_node}) + new_resource, compute_node_created_created = Resource.objects.get_or_create( + name=row['nodelist'], + defaults={ + 'is_allocatable':False, + 'resource_type':compute_node, + 'parent_resource':current_cluster + } + ) Resource.objects.get_or_create(name=row['partition'], defaults={'resource_type':partition_resource_type}) gpu_count = ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres'])) @@ -143,13 +156,27 @@ def calculate_owner_value(project_list, row): new_resource.is_available = True bulk_update_resource.append(new_resource) service_end_pk = existing_resource_attributes_pk_map[f"{row['nodelist']} {service_end_attribute_type.name}"] - bulk_update_resource_attribute.append(ResourceAttribute(resource=new_resource, value=None, resource_attribute_type=service_end_attribute_type, pk=service_end_pk)) + bulk_update_resource_attribute.append( + ResourceAttribute( + resource=new_resource, value=None, + resource_attribute_type=service_end_attribute_type, + pk=service_end_pk, + modified=modify_history_date + ) + ) processed_resources.add(new_resource.name) try: logger.debug(f'Updating {len(bulk_update_resource_attribute)} ResourceAttribute records') - bulk_update_with_history(bulk_update_resource_attribute, ResourceAttribute, ['value'], batch_size=500, default_change_reason='slurm_manage_resource command') + bulk_update_with_history( + bulk_update_resource_attribute, ResourceAttribute, ['value'], + batch_size=500, default_change_reason='slurm_manage_resource command', + default_date=modify_history_date + ) logger.debug(f'Updating {len(bulk_update_resource)} Resource records') - bulk_update_with_history(bulk_update_resource, Resource, ['is_available'], batch_size=500, default_change_reason='slurm_manage_resource command') + bulk_update_with_history( + bulk_update_resource, Resource, ['is_available'], batch_size=500, + default_change_reason='slurm_manage_resource command', default_date=modify_history_date + ) logger.debug(f'Creating {len(bulk_create_resource_attribute)} ResourceAttribute records') bulk_create_with_history(bulk_create_resource_attribute, ResourceAttribute, batch_size=500, default_change_reason='slurm_manage_resource command') except Exception as e: @@ -161,19 +188,25 @@ def calculate_owner_value(project_list, row): for resource_to_delete in Resource.objects.exclude(name__in=list(processed_resources)).filter(is_available=True, resource_type=compute_node): resource_to_delete.is_available = False bulk_update_resource.append(resource_to_delete) - service_end = ResourceAttribute(resource=resource_to_delete, value=str(datetime.now()), resource_attribute_type=service_end_attribute_type) + service_end = ResourceAttribute(resource=resource_to_delete, value=modify_history_date, resource_attribute_type=service_end_attribute_type) if f"{resource_to_delete.name} {service_end_attribute_type.name}" in existing_resource_attributes_check: service_end.pk = existing_resource_attributes_pk_map[f"{resource_to_delete.name} {service_end_attribute_type.name}"] bulk_update_resource_attribute.append(service_end) else: bulk_create_resource_attribute.append(service_end) try: - logger.debug(f'Decommissioning {len(bulk_update_resource)} Resource records') - bulk_update_with_history(bulk_update_resource, Resource, ['is_available'], batch_size=500, default_change_reason='slurm_manage_resource command') + logger.debug(f'Decommissioning {bulk_update_resource} Resource records') + bulk_update_with_history( + bulk_update_resource, Resource, ['is_available'], batch_size=500, + default_change_reason='slurm_manage_resource command', default_date=modify_history_date + ) logger.debug(f'Creating {len(bulk_create_resource_attribute)} ServiceEnd ResourceAttribute records') bulk_create_with_history(bulk_create_resource_attribute, ResourceAttribute, batch_size=500, default_change_reason='slurm_manage_resource command') logger.debug(f'Updating {len(bulk_update_resource_attribute)} ServiceEnd ResourceAttribute records') - bulk_update_with_history(bulk_update_resource_attribute, ResourceAttribute, ['value'], batch_size=500, default_change_reason='slurm_manage_resource command') + bulk_update_with_history( + bulk_update_resource_attribute, ResourceAttribute, ['value'], batch_size=500, + default_change_reason='slurm_manage_resource command', default_date=modify_history_date + ) except Exception as e: logger.error(f'Error cleaning up resources: {str(e)}') raise From 7902069b5f2c1dbada82fd5e3cb5d7a501b87a15 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Thu, 17 Apr 2025 13:47:28 -0700 Subject: [PATCH 05/24] remove developer configs --- .../commands/slurm_manage_resources.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py index 629f4bf70..f4280707d 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -1,14 +1,12 @@ import logging -import os import re from functools import reduce -from cProfile import Profile from django.utils import timezone from django.core.management.base import BaseCommand from simple_history.utils import bulk_update_with_history, bulk_create_with_history -from coldfront.core.resource.models import ResourceType, ResourceAttribute, ResourceAttributeType, AttributeType, Resource +from coldfront.core.resource.models import ResourceType, ResourceAttribute, ResourceAttributeType, Resource from coldfront.core.project.models import Project from coldfront.plugins.slurm.utils import slurm_get_nodes_info @@ -34,17 +32,8 @@ def get_output_from_file(self, file_path): def add_arguments(self, parser): parser.add_argument("-e", "--environment", help="Environment, use dev to simulate output") - parser.add_argument('--profile', action='store_true', default=False) def handle(self, *args, **options): - if options.get('profile', False): - profiler = Profile() - profiler.runcall(self._handle, *args, **options) - profiler.print_stats() - else: - self._handle(*args, **options) - - def _handle(self, *args, **options): def calculate_gpu_count(gres_value): if 'null' in gres_value: return 0 @@ -68,19 +57,10 @@ def calculate_owner_value(project_list, row): return'FASRC' return owner_name - def get_cluster(): - return Resource.objects.get(resource_type__name='Cluster') - - - env = options['environment'] or 'production' - if 'dev' in env: - output = self.get_output_from_file(os.path.join(os.getcwd(), 'coldfront/plugins/slurm/management/commands/sinfo.txt')) - else: - output = slurm_get_nodes_info() - logger.debug(f'Running on {env} mode') + output = slurm_get_nodes_info() modify_history_date = timezone.now() project_list = Project.objects.all() - current_cluster = get_cluster() + current_cluster = Resource.objects.get(resource_type__name='Cluster') compute_node = ResourceType.objects.get(name='Compute Node') attribute_type_name_list = ['GPU Count', 'Core Count', 'Features', 'Owner', 'ServiceEnd'] partition_resource_type = ResourceType.objects.get(name='Cluster Partition') @@ -89,7 +69,8 @@ def get_cluster(): features_attribute_type = ResourceAttributeType.objects.get(name='Features') owner_attribute_type = ResourceAttributeType.objects.get(name='Owner') service_end_attribute_type = ResourceAttributeType.objects.get(name='ServiceEnd') - existing_resource_attributes = list(ResourceAttribute.objects.filter( + existing_resource_attributes = list( + ResourceAttribute.objects.filter( resource_attribute_type__name__in=attribute_type_name_list, resource__resource_type__name='Compute Node' ).values_list('pk', 'resource__name', 'resource_attribute_type__name') @@ -102,7 +83,7 @@ def get_cluster(): bulk_update_resource = [] processed_resource_attribute = [] for row in output: - new_resource, compute_node_created_created = Resource.objects.get_or_create( + new_resource, compute_node_created = Resource.objects.get_or_create( name=row['nodelist'], defaults={ 'is_allocatable':False, @@ -112,7 +93,7 @@ def get_cluster(): ) Resource.objects.get_or_create(name=row['partition'], defaults={'resource_type':partition_resource_type}) - gpu_count = ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres'])) + gpu_count = ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres'])) gpu_count_key = f"{row['nodelist']} {gpu_count_attribute_type.name}" if gpu_count_key in existing_resource_attributes_check: gpu_count.pk = existing_resource_attributes_pk_map[gpu_count_key] @@ -142,7 +123,11 @@ def get_cluster(): bulk_create_resource_attribute.append(features) processed_resource_attribute.append(features_key) - owner = ResourceAttribute(resource_attribute_type=owner_attribute_type, resource=new_resource, value=calculate_owner_value(project_list, row)) + owner = ResourceAttribute( + resource_attribute_type=owner_attribute_type, + resource=new_resource, + value=calculate_owner_value(project_list, row) + ) owner_key = f"{row['nodelist']} {owner_attribute_type.name}" if owner_key in existing_resource_attributes_check: owner.pk = existing_resource_attributes_pk_map[owner_key] From 61f4c49126c96c2aef01aca03fc85038c8ce26c8 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 17 Apr 2025 21:37:21 +0000 Subject: [PATCH 06/24] prevent creation of Products from Compute Node resources --- coldfront/plugins/ifx/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/coldfront/plugins/ifx/models.py b/coldfront/plugins/ifx/models.py index 100cfab64..a00e820a3 100644 --- a/coldfront/plugins/ifx/models.py +++ b/coldfront/plugins/ifx/models.py @@ -131,7 +131,7 @@ def resource_post_save(sender, instance, **kwargs): ''' Ensure that there is a Product for each Resource ''' - if not kwargs.get('raw') and not instance.resource_type.name in ["Storage Tier", "Cluster", "Cluster Partition"]: + if not kwargs.get('raw') and not instance.resource_type.name in ["Storage Tier", "Cluster", "Cluster Partition", "Compute Node"]: try: product_resource = ProductResource.objects.get(resource=instance) except ProductResource.DoesNotExist: @@ -147,6 +147,7 @@ def resource_post_save(sender, instance, **kwargs): fiine_product.pop('object_code_category') fiine_product['facility'] = facility fiine_product['billing_calculator'] = 'coldfront.plugins.ifx.calculator.NewColdfrontBillingCalculator' + fiine_product['billable'] = instance.requires_payment (product, created) = Product.objects.get_or_create(**fiine_product) product_resource = ProductResource.objects.create(product=product, resource=instance) @@ -273,4 +274,4 @@ class PreferredUsername(models.Model): preferred_username = models.CharField(max_length=255) def __str__(self): - return f'{self.ifxid} - {self.preferred_username}' \ No newline at end of file + return f'{self.ifxid} - {self.preferred_username}' From 6af3ce97b18fb0c93503737849f6c5c2e1a3e4d0 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Thu, 17 Apr 2025 21:38:24 +0000 Subject: [PATCH 07/24] change defaults for created resources, set cluster to fasrc --- .../commands/slurm_manage_resources.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py index f4280707d..6609f36a7 100644 --- a/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py +++ b/coldfront/plugins/slurm/management/commands/slurm_manage_resources.py @@ -60,7 +60,7 @@ def calculate_owner_value(project_list, row): output = slurm_get_nodes_info() modify_history_date = timezone.now() project_list = Project.objects.all() - current_cluster = Resource.objects.get(resource_type__name='Cluster') + current_cluster = Resource.objects.get(name='FASRC Cluster') compute_node = ResourceType.objects.get(name='Compute Node') attribute_type_name_list = ['GPU Count', 'Core Count', 'Features', 'Owner', 'ServiceEnd'] partition_resource_type = ResourceType.objects.get(name='Cluster Partition') @@ -86,12 +86,20 @@ def calculate_owner_value(project_list, row): new_resource, compute_node_created = Resource.objects.get_or_create( name=row['nodelist'], defaults={ - 'is_allocatable':False, - 'resource_type':compute_node, - 'parent_resource':current_cluster + 'is_allocatable': False, + 'resource_type': compute_node, + 'parent_resource': current_cluster, + 'description': f"{row['nodelist']} node on {current_cluster.name}", + } + ) + Resource.objects.get_or_create( + name=row['partition'], + resource_type=partition_resource_type, + defaults={ + 'parent_resource': current_cluster, + 'description': f"{row['partition']} partition on {current_cluster.name}", } ) - Resource.objects.get_or_create(name=row['partition'], defaults={'resource_type':partition_resource_type}) gpu_count = ResourceAttribute(resource_attribute_type=gpu_count_attribute_type, resource=new_resource, value=calculate_gpu_count(row['gres'])) gpu_count_key = f"{row['nodelist']} {gpu_count_attribute_type.name}" From 432a6288c32bc906fe4deffb53e05af1a089abaf Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Thu, 20 Mar 2025 20:28:10 +0100 Subject: [PATCH 08/24] Include owned resources on user home. Only show owner or access granted resources on resources listview. --- coldfront/core/portal/views.py | 12 +++++++----- coldfront/core/resource/views.py | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/coldfront/core/portal/views.py b/coldfront/core/portal/views.py index 2ec63233f..8bf0a2de8 100644 --- a/coldfront/core/portal/views.py +++ b/coldfront/core/portal/views.py @@ -15,7 +15,7 @@ ) from coldfront.core.project.models import Project from coldfront.core.publication.models import Publication -from coldfront.core.resource.models import Resource +from coldfront.core.resource.models import Resource, ResourceAttribute from coldfront.config.env import ENV from coldfront.core.department.models import Department, DepartmentMember from coldfront.core.utils.common import import_from_settings @@ -82,10 +82,12 @@ def home(request): department_list = Department.objects.filter( id__in=user_depts.values_list('organization_id') ) - - resource_list = Resource.objects.filter( - allowed_users=request.user) - + project_title_list = [project.title for project in project_list] + owned_resources = [attribute.resource.pk for attribute in ResourceAttribute.objects.filter( + resource_attribute_type__name='Owner', + value__in=project_title_list + )] + resource_list = Resource.objects.filter(Q(allowed_users=request.user) | Q(pk__in=owned_resources)).distinct() context['resource_list'] = resource_list context['department_list'] = department_list context['project_list'] = project_list diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index 17ed02ee0..7c640607f 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -28,6 +28,10 @@ from coldfront.plugins.slurm.utils import SlurmError +<<<<<<< HEAD +======= +from coldfront.core.project.models import ProjectUser, Project +>>>>>>> 8ed42192 (Include owned resources on user home. Only show owner or access granted resources on resources listview.) logger = logging.getLogger(__name__) @@ -304,6 +308,15 @@ def get_queryset(self): resource_search_form = ResourceSearchForm(self.request.GET) resources = Resource.objects.filter(is_available=True) + project_list = Project.objects.filter( + Q(status__name__in=['New', 'Active', ]) & ( + Q(pi=self.request.user) | ( + Q(projectuser__user=self.request.user) + & Q(projectuser__status__name='Active') + ) + ) + ).distinct().order_by('-created') + if order_by == 'name': direction = self.request.GET.get('direction') if direction == 'asc': @@ -363,7 +376,12 @@ def get_queryset(self): Q(resourceattribute__resource_attribute_type__name='Vendor') & Q(resourceattribute__value=data.get('vendor')) ) - return resources.distinct() + project_title_list = [project.title for project in project_list] + owned_resources = [attribute.resource.pk for attribute in ResourceAttribute.objects.filter( + resource_attribute_type__name='Owner', + value__in=project_title_list + )] + return resources.filter(Q(allowed_users=self.request.user) | Q(pk__in=owned_resources)).distinct() def get_context_data(self, **kwargs): context = super().get_context_data( From f56796cd7435af483450fa4bafbb078c20ffad4d Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 26 Mar 2025 17:52:37 +0100 Subject: [PATCH 09/24] Only excluded not owned Compute Nodes from resource listview --- coldfront/core/resource/views.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index 7c640607f..4531afd13 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -377,11 +377,12 @@ def get_queryset(self): Q(resourceattribute__value=data.get('vendor')) ) project_title_list = [project.title for project in project_list] - owned_resources = [attribute.resource.pk for attribute in ResourceAttribute.objects.filter( + not_owned_compute_nodes = [attribute.resource.pk for attribute in ResourceAttribute.objects.filter( resource_attribute_type__name='Owner', - value__in=project_title_list - )] - return resources.filter(Q(allowed_users=self.request.user) | Q(pk__in=owned_resources)).distinct() + resource__resource_type__name='Compute Node' + ).exclude(value__in=project_title_list)] + return resources.exclude(pk__in=not_owned_compute_nodes).distinct() + def get_context_data(self, **kwargs): context = super().get_context_data( From a3485520adb17a22dfc4241638a93f3caf504f04 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 26 Mar 2025 19:01:17 +0100 Subject: [PATCH 10/24] Add listview for retired resources, prevent modifications for retired resources --- .../templates/resource_archived_list.html | 63 +++++++++++ .../resource/templates/resource_detail.html | 6 +- .../resource/templates/resource_list.html | 10 ++ coldfront/core/resource/urls.py | 1 + coldfront/core/resource/views.py | 103 +++++++++++++++++- 5 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 coldfront/core/resource/templates/resource_archived_list.html diff --git a/coldfront/core/resource/templates/resource_archived_list.html b/coldfront/core/resource/templates/resource_archived_list.html new file mode 100644 index 000000000..d99996a91 --- /dev/null +++ b/coldfront/core/resource/templates/resource_archived_list.html @@ -0,0 +1,63 @@ +{% extends "list_view.html" %} + +{% block title %} +Project List +{% endblock %} + +{% block page_title%}Archived Resources{% endblock %} + +{% block presearch %} + +{% endblock %} + + +{% block list_title %}Resource{{count|pluralize}}: {{count}}{% endblock %} + +{% block table_contents %} + + + + ID + Sort ID asc + Sort ID desc + + + Resource Name + Sort Resource Name asc + Sort Resource Name desc + + + Parent Resource + Sort Parent Resource asc + Sort Parent Resource desc + + + Resource Type + Sort Resource Type asc + Sort Resource Type desc + + + + + {% for resource in item_list %} + + {{ resource.id }} + {{ resource }} + {{ resource.parent_resource }} + {{ resource.resource_type.name }} + + {% endfor %} + +{% endblock %} + +{% block activelink %} +$("#navbar-project-menu").addClass("active"); +$("#navbar-resource").addClass("active"); + +{% endblock %} diff --git a/coldfront/core/resource/templates/resource_detail.html b/coldfront/core/resource/templates/resource_detail.html index 69d077b99..7b5503e76 100644 --- a/coldfront/core/resource/templates/resource_detail.html +++ b/coldfront/core/resource/templates/resource_detail.html @@ -12,7 +12,11 @@ {% block content %} - +{% if resource.is_available == False %} + +{% endif %}

Resource Detail


diff --git a/coldfront/core/resource/templates/resource_list.html b/coldfront/core/resource/templates/resource_list.html index da74e6f96..e5cbc78bc 100644 --- a/coldfront/core/resource/templates/resource_list.html +++ b/coldfront/core/resource/templates/resource_list.html @@ -3,6 +3,15 @@ {% block title %}Resource List{% endblock %} {% block page_title %}Resources{% endblock %} +{% block presearch %} + +{% endblock %} {% block list_title %}Resource{{count|pluralize}}: {{count}}{% endblock %} @@ -46,4 +55,5 @@ {% block activelink %} $("#navbar-project-menu").addClass("active"); $("#navbar-resource").addClass("active"); + {% endblock %} diff --git a/coldfront/core/resource/urls.py b/coldfront/core/resource/urls.py index afb501f81..30ec4fad7 100644 --- a/coldfront/core/resource/urls.py +++ b/coldfront/core/resource/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path('', resource_views.ResourceListView.as_view(), name='resource-list'), + path('archived/', resource_views.ResourceArchivedListView.as_view(), name='resource-archived-list'), path('/', resource_views.ResourceDetailView.as_view(), name='resource-detail'), path('/resourceattribute/add', diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index 4531afd13..d057b462e 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -28,10 +28,8 @@ from coldfront.plugins.slurm.utils import SlurmError -<<<<<<< HEAD -======= from coldfront.core.project.models import ProjectUser, Project ->>>>>>> 8ed42192 (Include owned resources on user home. Only show owner or access granted resources on resources listview.) +from pandas.io.clipboard import is_available logger = logging.getLogger(__name__) @@ -197,6 +195,18 @@ def test_func(self): messages.error( self.request, 'You do not have permission to add resource attributes.') + def dispatch(self, request, *args, **kwargs): + resource_obj = get_object_or_404(Resource, pk=self.kwargs.get('pk')) + err = None + if resource_obj.is_available is False: + err = 'You cannot add resource attributes to retired allocations.' + if err: + messages.error(request, err) + return HttpResponseRedirect( + reverse('resource-detail', kwargs={'pk': resource_obj.pk}) + ) + return super().dispatch(request, *args, **kwargs) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) pk = self.kwargs.get('pk') @@ -231,6 +241,18 @@ def test_func(self): messages.error( self.request, 'You do not have permission to delete resource attributes.') + def dispatch(self, request, *args, **kwargs): + resource_obj = get_object_or_404(Resource, pk=self.kwargs.get('pk')) + err = None + if resource_obj.is_available is False: + err = 'You cannot delete resource attributes from retired allocations.' + if err: + messages.error(request, err) + return HttpResponseRedirect( + reverse('resource-detail', kwargs={'pk': resource_obj.pk}) + ) + return super().dispatch(request, *args, **kwargs) + def get(self, request, *args, **kwargs): pk = self.kwargs.get('pk') resource_obj = get_object_or_404(Resource, pk=pk) @@ -381,15 +403,86 @@ def get_queryset(self): resource_attribute_type__name='Owner', resource__resource_type__name='Compute Node' ).exclude(value__in=project_title_list)] + if self.request.user.is_superuser: + return resources.distinct() return resources.exclude(pk__in=not_owned_compute_nodes).distinct() - def get_context_data(self, **kwargs): context = super().get_context_data( SearchFormClass=ResourceSearchForm, **kwargs) return context +class ResourceArchivedListView(ResourceListView): + template_name = 'resource_archived_list.html' + + def get_queryset(self): + + order_by = self.return_order() + resource_search_form = ResourceSearchForm(self.request.GET) + + if order_by == 'name': + direction = self.request.GET.get('direction') + if direction == 'asc': + resources = Resource.objects.all().order_by(Lower('name')) + elif direction == 'des': + resources = (Resource.objects.all().order_by(Lower('name')).reverse()) + else: + resources = Resource.objects.all().order_by(order_by) + else: + resources = Resource.objects.all().order_by(order_by) + if resource_search_form.is_valid(): + data = resource_search_form.cleaned_data + + if data.get('show_allocatable_resources'): + resources = resources.filter(is_allocatable=True) + if data.get('resource_name'): + resources = resources.filter( + name__icontains=data.get('resource_name') + ) + if data.get('resource_type'): + resources = resources.filter( + resource_type=data.get('resource_type') + ) + + if data.get('model'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='Model') & + Q(resourceattribute__value=data.get('model')) + ) + if data.get('serialNumber'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='SerialNumber') & + Q(resourceattribute__value=data.get('serialNumber')) + ) + if data.get('installDate'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='InstallDate') & + Q(resourceattribute__value=data.get('installDate').strftime('%m/%d/%Y')) + ) + if data.get('serviceStart'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type_name='ServiceStart') & + Q(resourceattribute__value=data.get('serviceStart').strftime('%m/%d/%Y')) + ) + if data.get('serviceEnd'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='ServiceEnd') & + Q(resourceattribute__value=data.get('serviceEnd').strftime('%m/%d/%Y')) + ) + if data.get('warrantyExpirationDate'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='WarrantyExpirationDate') & + Q(resourceattribute__value=data.get('warrantyExpirationDate').strftime('%m/%d/%Y')) + ) + if data.get('vendor'): + resources = resources.filter( + Q(resourceattribute__resource_attribute_type__name='Vendor') & + Q(resourceattribute__value=data.get('vendor')) + ) + return resources.exclude(is_available=True).distinct() + + class ResourceAllocationsEditView(LoginRequiredMixin, UserPassesTestMixin, TemplateView): template_name = 'resource_allocations_edit.html' @@ -407,6 +500,8 @@ def dispatch(self, request, *args, **kwargs): err = None if 'Storage' in resource_obj.resource_type.name: err = 'You cannot bulk-edit storage allocations.' + if resource_obj.is_available is False: + err = 'You cannot edit retired allocations.' if err: messages.error(request, err) return HttpResponseRedirect( From 057d381d2b408a1ec398fc68d7fa5d90a9e86716 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 9 Apr 2025 16:20:37 +0200 Subject: [PATCH 11/24] Tests --- coldfront/core/resource/tests.py | 65 ++++++++++++- coldfront/core/test_helpers/factories.py | 24 +++++ .../test_data/test_fixtures/resource.json | 97 +++++++++++++++++++ 3 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 coldfront/core/test_helpers/test_data/test_fixtures/resource.json diff --git a/coldfront/core/resource/tests.py b/coldfront/core/resource/tests.py index b364d45a4..8bc1219ae 100644 --- a/coldfront/core/resource/tests.py +++ b/coldfront/core/resource/tests.py @@ -1,12 +1,20 @@ +from boto3 import resource +from django.db.models import Q from django.test import TestCase from coldfront.core.test_helpers import utils -from coldfront.core.test_helpers.factories import setup_models +from coldfront.core.test_helpers.factories import setup_models, AttributeTypeFactory, ProjectFactory, ResourceFactory, ResourceTypeFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory +from coldfront.core.project.models import Project +from coldfront.core.resource.models import AttributeType, ResourceType UTIL_FIXTURES = [ "coldfront/core/test_helpers/test_data/test_fixtures/ifx.json", ] +RESOURCE_FIXTURES = [ + "coldfront/core/test_helpers/test_data/test_fixtures/resource.json", +] + BACKEND = "django.contrib.auth.backends.ModelBackend" @@ -27,11 +35,63 @@ def resource_access_tstbase(self, url): utils.test_logged_out_redirect_to_login(self, url) utils.test_user_can_access(self, self.admin_user, url) # admin can access + class ResourceListViewTest(ResourceViewBaseTest): """Tests for ResourceListView""" + fixtures = RESOURCE_FIXTURES def setUp(self): - self.client.force_login(self.admin_user, backend=BACKEND) + self.client.force_login(self.pi_user, backend=BACKEND) + self.url = f'/resource/' + + def test_only_user_managed_compute_nodes_show(self): + ProjectFactory(pi=self.pi_user, title="managed_lab") + ProjectFactory(pi=self.admin_user, title="admin_lab") + text_attribute_type = AttributeType.objects.get(name="Text") + managed_resource = ResourceFactory(name="managed_lab", resource_type__name='Compute Node') + admin_resource = ResourceFactory(name="admin_lab", resource_type__name='Compute Node') + owner_resourcer_attr_type = ResourceAttributeTypeFactory(name="Owner", attribute_type=text_attribute_type) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", resource=managed_resource) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="admin_lab", resource=admin_resource) + utils.page_contains_for_user(self, self.pi_user, self.url, 'managed_lab') + utils.page_does_not_contain_for_user(self, self.pi_user, self.url, 'admin_lab') + utils.page_contains_for_user(self, self.admin_user, self.url, 'admin_lab') + utils.page_contains_for_user(self, self.admin_user, self.url, 'managed_lab') + + def test_retired_resources_filter_shows(self): + utils.page_contains_for_user(self, self.pi_user, self.url, 'View retired resources') + utils.page_contains_for_user(self, self.admin_user, self.url, 'View retired resources') + + +class ResourceArchivedListViewTest(ResourceViewBaseTest): + """Tests for ResourceArchivedListView""" + fixtures = RESOURCE_FIXTURES + + def setUp(self): + self.client.force_login(self.pi_user, backend=BACKEND) + self.url = f'/resource/archived/' + + def test_archive_resources_show(self): + ResourceFactory(name="archived_resource", resource_type__name='Compute Node', is_available=False) + ResourceFactory(name="active_resource", resource_type__name='Compute Node') + utils.page_contains_for_user(self, self.pi_user, self.url, 'archived_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, self.url, 'active_resource') + utils.page_contains_for_user(self, self.admin_user, self.url, 'archived_resource') + utils.page_does_not_contain_for_user(self, self.admin_user, self.url, 'active_resource') + + def test_can_filter_by_name(self): + AttributeType.objects.get(name="Text") + ResourceFactory(name="archived_resource", resource_type__name='Compute Node', is_available=False) + ResourceFactory(name="archived_resource2", resource_type__name='Compute Node', is_available=False) + ResourceFactory(name="active_resource", resource_type__name='Compute Node') + search_url = f'{self.url}?resource_name=archived_resource' + utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'archived_resource2') + utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'active_resource') + search_url = f'{self.url}?resource_name=archived_resource2' + utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource2') + utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'archived_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'active_resource') class ClusterResourceDetailViewTest(ResourceViewBaseTest): @@ -146,6 +206,7 @@ def test_resource_attribute_create_access(self): utils.test_user_cannot_access(self, self.resource_allowed_user, self.url) utils.test_user_cannot_access(self, self.pi_user, self.url) + class ClusterResourceAttributeDeleteViewTest(ResourceViewBaseTest): """Tests for ResourceAttributeDeleteView""" diff --git a/coldfront/core/test_helpers/factories.py b/coldfront/core/test_helpers/factories.py index 98c54928d..7d922ead9 100644 --- a/coldfront/core/test_helpers/factories.py +++ b/coldfront/core/test_helpers/factories.py @@ -36,6 +36,8 @@ from coldfront.core.grant.models import GrantFundingAgency, GrantStatusChoice from coldfront.core.publication.models import PublicationSource +from coldfront.core.allocation.models import AttributeType +from coldfront.core.resource.models import ResourceAttribute, ResourceAttributeType ### Default values and Faker provider setup ### @@ -212,7 +214,29 @@ class Meta: description = factory.Faker('sentence') resource_type = SubFactory(ResourceTypeFactory) +class AttributeTypeFactory(DjangoModelFactory): + class Meta: + model = AttributeType + + name = factory.Faker("word") + +class ResourceAttributeTypeFactory(DjangoModelFactory): + class Meta: + model = ResourceAttributeType + + attribute_type = factory.SubFactory(AttributeTypeFactory) + name = factory.Faker("word") + is_required = factory.Faker("boolean") + is_unique_per_resource = factory.Faker("boolean") + is_value_unique = factory.Faker("boolean") + +class ResourceAttributeFactory(DjangoModelFactory): + class Meta: + model = ResourceAttribute + django_get_or_create = ('resource', 'resource_attribute_type') + value = 'admin_lab' + resource = factory.SubFactory(ResourceFactory) ### Allocation factories ### diff --git a/coldfront/core/test_helpers/test_data/test_fixtures/resource.json b/coldfront/core/test_helpers/test_data/test_fixtures/resource.json new file mode 100644 index 000000000..23ad32b7f --- /dev/null +++ b/coldfront/core/test_helpers/test_data/test_fixtures/resource.json @@ -0,0 +1,97 @@ +[ + + { + "model": "resource.attributetype", + "pk": 1, + "fields": { + "created": "2024-10-10T01:18:54.259Z", + "modified": "2024-10-10T01:18:54.259Z", + "name": "Active/Inactive" + } + }, + { + "model": "resource.attributetype", + "pk": 2, + "fields": { + "created": "2024-10-10T01:18:54.266Z", + "modified": "2024-10-10T01:18:54.266Z", + "name": "Date" + } + }, + { + "model": "resource.attributetype", + "pk": 3, + "fields": { + "created": "2024-10-10T01:18:54.277Z", + "modified": "2024-10-10T01:18:54.277Z", + "name": "Int" + } + }, + + { + "model": "resource.attributetype", + "pk": 5, + "fields": { + "created": "2024-10-10T01:18:54.290Z", + "modified": "2024-10-10T01:18:54.290Z", + "name": "Text" + } + }, + { + "model": "resource.attributetype", + "pk": 7, + "fields": { + "created": "2024-10-10T01:18:54.300Z", + "modified": "2024-10-10T01:18:54.300Z", + "name": "Attribute Expanded Text" + } + }, + { + "model": "resource.attributetype", + "pk": 8, + "fields": { + "created": "2024-10-10T01:18:54.305Z", + "modified": "2024-10-10T01:18:54.305Z", + "name": "Float" + } + }, + { + "model": "resource.resourceattributetype", + "pk": 10, + "fields": { + "created": "2024-10-11T00:26:36.739Z", + "modified": "2024-10-11T00:26:36.739Z", + "attribute_type": 5, + "name": "slurm_cluster", + "is_required": false, + "is_unique_per_resource": false, + "is_value_unique": false + } + }, + { + "model": "resource.resourceattributetype", + "pk": 15, + "fields": { + "created": "2025-03-21T21:50:07.174Z", + "modified": "2025-03-21T21:50:07.174Z", + "attribute_type": 5, + "name": "Owner", + "is_required": false, + "is_unique_per_resource": false, + "is_value_unique": false + } + }, + { + "model": "resource.resourceattributetype", + "pk": 16, + "fields": { + "created": "2025-03-21T21:50:07.177Z", + "modified": "2025-03-21T21:50:07.177Z", + "attribute_type": 2, + "name": "ServiceEnd", + "is_required": false, + "is_unique_per_resource": false, + "is_value_unique": false + } + } +] From 07ed0aacf839991938437b9d388b7c8a383ee0f4 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 9 Apr 2025 17:47:19 +0200 Subject: [PATCH 12/24] Filter our archived resources from user home page managed resources. Add more tests --- coldfront/core/portal/tests.py | 47 +++++++++++++++++++++++++++++++- coldfront/core/portal/views.py | 3 +- coldfront/core/resource/tests.py | 9 ++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/coldfront/core/portal/tests.py b/coldfront/core/portal/tests.py index 69796f91d..0cb1db807 100644 --- a/coldfront/core/portal/tests.py +++ b/coldfront/core/portal/tests.py @@ -2,9 +2,14 @@ from coldfront.core.test_helpers import utils from coldfront.core.test_helpers.factories import setup_models from coldfront.core.allocation.models import AllocationChangeRequest, AllocationChangeStatusChoice +from coldfront.core.test_helpers.factories import setup_models, AttributeTypeFactory, ProjectFactory, ResourceFactory, ResourceTypeFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory +from coldfront.core.project.models import Project +from coldfront.core.resource.models import AttributeType, ResourceType UTIL_FIXTURES = ['coldfront/core/test_helpers/test_data/test_fixtures/ifx.json'] - +RESOURCE_FIXTURES = [ + "coldfront/core/test_helpers/test_data/test_fixtures/resource.json", +] class PortalViewTest(TestCase): """Base class for portal view tests @@ -29,6 +34,7 @@ def test_center_summary(self): class HomePageTest(PortalViewTest): + fixtures = RESOURCE_FIXTURES def test_pi_home_page(self): """check that the pi home page displays properly with the existing database @@ -85,3 +91,42 @@ def test_home_page_projects_display(self): # allocationuser not belonging to project cannot see project response = utils.login_and_get_page(self.client, self.nonproj_allocationuser, '') self.assertEqual(response.context['project_list'].count(), 0) + + def test_home_page_managed_resources_display(self): + """check that managed resources display properly on the home page + """ + ProjectFactory(pi=self.pi_user, title="managed_lab") + ProjectFactory(pi=self.admin_user, title="admin_lab") + text_attribute_type = AttributeType.objects.get(name="Text") + managed_resource = ResourceFactory(name="managed_lab", resource_type__name='Compute Node') + managed_resource2 = ResourceFactory(name="managed_lab2", resource_type__name='Compute Node') + admin_resource = ResourceFactory(name="admin_lab", resource_type__name='Compute Node') + owner_resourcer_attr_type = ResourceAttributeTypeFactory(name="Owner", attribute_type=text_attribute_type) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", + resource=managed_resource) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", + resource=managed_resource2) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="admin_lab", + resource=admin_resource) + utils.page_contains_for_user(self, self.pi_user, '', 'Managed Resources') + utils.page_contains_for_user(self, self.admin_user, '', 'Managed Resources') + utils.page_contains_for_user(self, self.pi_user, '', 'managed_lab') + utils.page_contains_for_user(self, self.pi_user, '', 'managed_lab2') + utils.page_does_not_contain_for_user(self, self.pi_user, '', 'admin_lab') + utils.page_contains_for_user(self, self.admin_user, '', 'admin_lab') + utils.page_does_not_contain_for_user(self, self.admin_user, '', 'managed_lab') + utils.page_does_not_contain_for_user(self, self.admin_user, '', 'managed_lab2') + + def test_home_page_archive_resources_dont_show(self): + ProjectFactory(pi=self.pi_user, title="managed_lab") + text_attribute_type = AttributeType.objects.get(name="Text") + owner_resourcer_attr_type = ResourceAttributeTypeFactory(name="Owner", attribute_type=text_attribute_type) + archived_resource = ResourceFactory(name="archived_resource", resource_type__name='Compute Node', is_available=False) + archived_resource2 = ResourceFactory(name="archived_resource2", resource_type__name='Compute Node', is_available=False) + active_resource = ResourceFactory(name="active_resource", resource_type__name='Compute Node') + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", resource=archived_resource) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", resource=archived_resource2) + ResourceAttributeFactory(resource_attribute_type=owner_resourcer_attr_type, value="managed_lab", resource=active_resource) + utils.page_contains_for_user(self, self.pi_user, '', 'active_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, '', 'archived_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, '', 'archived_resource2') diff --git a/coldfront/core/portal/views.py b/coldfront/core/portal/views.py index 8bf0a2de8..c847753fd 100644 --- a/coldfront/core/portal/views.py +++ b/coldfront/core/portal/views.py @@ -19,6 +19,7 @@ from coldfront.config.env import ENV from coldfront.core.department.models import Department, DepartmentMember from coldfront.core.utils.common import import_from_settings +from pandas.io.clipboard import is_available if ENV.bool('PLUGIN_SFTOCF', default=False): from coldfront.plugins.sftocf.utils import StarFishRedash, STARFISH_SERVER @@ -87,7 +88,7 @@ def home(request): resource_attribute_type__name='Owner', value__in=project_title_list )] - resource_list = Resource.objects.filter(Q(allowed_users=request.user) | Q(pk__in=owned_resources)).distinct() + resource_list = Resource.objects.filter(Q(allowed_users=request.user) | Q(pk__in=owned_resources)).filter(is_available=True).distinct() context['resource_list'] = resource_list context['department_list'] = department_list context['project_list'] = project_list diff --git a/coldfront/core/resource/tests.py b/coldfront/core/resource/tests.py index 8bc1219ae..c6e291e3b 100644 --- a/coldfront/core/resource/tests.py +++ b/coldfront/core/resource/tests.py @@ -62,6 +62,15 @@ def test_retired_resources_filter_shows(self): utils.page_contains_for_user(self, self.pi_user, self.url, 'View retired resources') utils.page_contains_for_user(self, self.admin_user, self.url, 'View retired resources') + def test_archive_resources_dont_show(self): + ResourceFactory(name="archived_resource", resource_type__name='Compute Node', is_available=False) + ResourceFactory(name="archived_resource2", resource_type__name='Compute Node', is_available=False) + ResourceFactory(name="active_resource", resource_type__name='Compute Node') + utils.page_contains_for_user(self, self.pi_user, self.url, 'active_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, self.url, 'archived_resource') + utils.page_does_not_contain_for_user(self, self.pi_user, self.url, 'archived_resource2') + + class ResourceArchivedListViewTest(ResourceViewBaseTest): """Tests for ResourceArchivedListView""" From 4be2e3bae6c07ba9d63ef31f494e0be330c107a2 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 13:03:06 -0700 Subject: [PATCH 13/24] add owner property to resource model --- coldfront/core/allocation/models.py | 1 - coldfront/core/resource/models.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/coldfront/core/allocation/models.py b/coldfront/core/allocation/models.py index 045be6e40..4af3b7be5 100644 --- a/coldfront/core/allocation/models.py +++ b/coldfront/core/allocation/models.py @@ -10,7 +10,6 @@ from django.db.models import Q from django.utils.html import mark_safe from django.utils.module_loading import import_string -from django.contrib.auth import get_user_model from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords diff --git a/coldfront/core/resource/models.py b/coldfront/core/resource/models.py index 5662415af..8167c233e 100644 --- a/coldfront/core/resource/models.py +++ b/coldfront/core/resource/models.py @@ -195,6 +195,18 @@ def used_percentage(self): return round(100 - (self.free_capacity / self.capacity * 100), 2) return None + @property + def owner(self): + """ + Returns: + str: the status of the resource + """ + try: + return ResourceAttribute.objects.get(resource=self, resource_attribute_type__attribute='Owner').value + except ObjectDoesNotExist: + return None + + @property def status(self): """ From e649d44a1188ff790f1f41ac436d1b59873d4c62 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 13:11:31 -0700 Subject: [PATCH 14/24] fix resourceattributetype lookups --- coldfront/core/resource/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/coldfront/core/resource/models.py b/coldfront/core/resource/models.py index 8167c233e..629a2fb49 100644 --- a/coldfront/core/resource/models.py +++ b/coldfront/core/resource/models.py @@ -202,7 +202,7 @@ def owner(self): str: the status of the resource """ try: - return ResourceAttribute.objects.get(resource=self, resource_attribute_type__attribute='Owner').value + return ResourceAttribute.objects.get(resource=self, resource_attribute_type__name='Owner').value except ObjectDoesNotExist: return None @@ -213,8 +213,10 @@ def status(self): Returns: str: the status of the resource """ - - return ResourceAttribute.objects.get(resource=self, resource_attribute_type__attribute='Status').value + try: + return ResourceAttribute.objects.get(resource=self, resource_attribute_type__name='Status').value + except ObjectDoesNotExist: + return None @property def expiry(self): From c220cab46caf7e03ca6f20412403505a64a5798d Mon Sep 17 00:00:00 2001 From: claire-peters Date: Fri, 18 Apr 2025 21:19:45 +0000 Subject: [PATCH 15/24] show owner, link to project if corresponding project exists --- .../core/resource/templates/resource_detail.html | 14 ++++++++++++++ coldfront/core/resource/views.py | 14 +++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/coldfront/core/resource/templates/resource_detail.html b/coldfront/core/resource/templates/resource_detail.html index 7b5503e76..f3a021246 100644 --- a/coldfront/core/resource/templates/resource_detail.html +++ b/coldfront/core/resource/templates/resource_detail.html @@ -77,6 +77,20 @@

Resource Information

{% endif %} + {% if owner != None %} + + Owner: + + {% if owner|stringformat:"s" == owner %} + {{ owner }} + {% else %} + + {{ resource.owner }} + + {% endif %} + + + {% endif %}
diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index d057b462e..d2eb4532d 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -14,6 +14,7 @@ from django.urls import reverse from django.views.generic import TemplateView from django.views.generic.edit import CreateView +from django.core.exceptions import ObjectDoesNotExist from coldfront.core.utils.views import ColdfrontListView from coldfront.core.resource.models import Resource, ResourceAttribute @@ -23,13 +24,12 @@ ResourceAttributeDeleteForm, ResourceAllocationUpdateForm, ) -from coldfront.core.allocation.models import AllocationStatusChoice, AllocationAttributeType, AllocationAttribute +from coldfront.core.allocation.models import AllocationAttributeType, AllocationAttribute from coldfront.core.allocation.signals import allocation_raw_share_edit from coldfront.plugins.slurm.utils import SlurmError -from coldfront.core.project.models import ProjectUser, Project -from pandas.io.clipboard import is_available +from coldfront.core.project.models import Project logger = logging.getLogger(__name__) @@ -172,6 +172,14 @@ def get_context_data(self, **kwargs): allocation_total['usage'] += allocation.usage context['allocation_total'] = allocation_total + owner = resource_obj.owner + if owner != None: + try: + owner = Project.objects.get(title=owner).pk + except ObjectDoesNotExist: + owner = owner + context['owner'] = owner + context['allocations'] = allocations context['resource'] = resource_obj context['attributes'] = attributes From 5387f3607a575b9eda9134b8fa7264091fbf3cc4 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:14:24 -0700 Subject: [PATCH 16/24] remove unused imports, revise resource test object creation --- coldfront/core/portal/tests.py | 5 +- coldfront/core/portal/views.py | 1 - coldfront/core/resource/models.py | 1 - coldfront/core/resource/tests.py | 13 +-- coldfront/core/test_helpers/factories.py | 29 ++++-- .../test_data/test_fixtures/resource.json | 97 ------------------- 6 files changed, 25 insertions(+), 121 deletions(-) delete mode 100644 coldfront/core/test_helpers/test_data/test_fixtures/resource.json diff --git a/coldfront/core/portal/tests.py b/coldfront/core/portal/tests.py index 0cb1db807..31410fb4a 100644 --- a/coldfront/core/portal/tests.py +++ b/coldfront/core/portal/tests.py @@ -2,9 +2,8 @@ from coldfront.core.test_helpers import utils from coldfront.core.test_helpers.factories import setup_models from coldfront.core.allocation.models import AllocationChangeRequest, AllocationChangeStatusChoice -from coldfront.core.test_helpers.factories import setup_models, AttributeTypeFactory, ProjectFactory, ResourceFactory, ResourceTypeFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory -from coldfront.core.project.models import Project -from coldfront.core.resource.models import AttributeType, ResourceType +from coldfront.core.test_helpers.factories import setup_models, ProjectFactory, ResourceFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory +from coldfront.core.resource.models import AttributeType UTIL_FIXTURES = ['coldfront/core/test_helpers/test_data/test_fixtures/ifx.json'] RESOURCE_FIXTURES = [ diff --git a/coldfront/core/portal/views.py b/coldfront/core/portal/views.py index c847753fd..a4c42baf0 100644 --- a/coldfront/core/portal/views.py +++ b/coldfront/core/portal/views.py @@ -19,7 +19,6 @@ from coldfront.config.env import ENV from coldfront.core.department.models import Department, DepartmentMember from coldfront.core.utils.common import import_from_settings -from pandas.io.clipboard import is_available if ENV.bool('PLUGIN_SFTOCF', default=False): from coldfront.plugins.sftocf.utils import StarFishRedash, STARFISH_SERVER diff --git a/coldfront/core/resource/models.py b/coldfront/core/resource/models.py index 629a2fb49..7788bea30 100644 --- a/coldfront/core/resource/models.py +++ b/coldfront/core/resource/models.py @@ -5,7 +5,6 @@ from django.contrib.auth.models import Group from django.utils.translation import gettext_lazy as _ from django.core.exceptions import ValidationError, ObjectDoesNotExist -from model_utils.fields import AutoLastModifiedField from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords diff --git a/coldfront/core/resource/tests.py b/coldfront/core/resource/tests.py index c6e291e3b..e38108df8 100644 --- a/coldfront/core/resource/tests.py +++ b/coldfront/core/resource/tests.py @@ -1,20 +1,13 @@ -from boto3 import resource -from django.db.models import Q from django.test import TestCase from coldfront.core.test_helpers import utils -from coldfront.core.test_helpers.factories import setup_models, AttributeTypeFactory, ProjectFactory, ResourceFactory, ResourceTypeFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory -from coldfront.core.project.models import Project -from coldfront.core.resource.models import AttributeType, ResourceType +from coldfront.core.test_helpers.factories import setup_models, ProjectFactory, ResourceFactory, ResourceAttributeTypeFactory, ResourceAttributeFactory +from coldfront.core.resource.models import AttributeType UTIL_FIXTURES = [ "coldfront/core/test_helpers/test_data/test_fixtures/ifx.json", ] -RESOURCE_FIXTURES = [ - "coldfront/core/test_helpers/test_data/test_fixtures/resource.json", -] - BACKEND = "django.contrib.auth.backends.ModelBackend" @@ -38,7 +31,6 @@ def resource_access_tstbase(self, url): class ResourceListViewTest(ResourceViewBaseTest): """Tests for ResourceListView""" - fixtures = RESOURCE_FIXTURES def setUp(self): self.client.force_login(self.pi_user, backend=BACKEND) @@ -74,7 +66,6 @@ def test_archive_resources_dont_show(self): class ResourceArchivedListViewTest(ResourceViewBaseTest): """Tests for ResourceArchivedListView""" - fixtures = RESOURCE_FIXTURES def setUp(self): self.client.force_login(self.pi_user, backend=BACKEND) diff --git a/coldfront/core/test_helpers/factories.py b/coldfront/core/test_helpers/factories.py index 7d922ead9..82dc142f9 100644 --- a/coldfront/core/test_helpers/factories.py +++ b/coldfront/core/test_helpers/factories.py @@ -1,14 +1,16 @@ import factory -from django.contrib.auth.models import User from django.contrib.auth import get_user_model from factory import SubFactory -from factory.fuzzy import FuzzyChoice from factory.django import DjangoModelFactory from faker import Faker from faker.providers import BaseProvider, DynamicProvider from coldfront.core.field_of_science.models import FieldOfScience -from coldfront.core.resource.models import ResourceType, Resource +from coldfront.core.resource.models import ( + ResourceType, + Resource, + AttributeType as RAttributeType +) from coldfront.core.project.models import ( Project, ProjectUser, @@ -214,17 +216,17 @@ class Meta: description = factory.Faker('sentence') resource_type = SubFactory(ResourceTypeFactory) -class AttributeTypeFactory(DjangoModelFactory): +class RAttributeTypeFactory(DjangoModelFactory): class Meta: - model = AttributeType - - name = factory.Faker("word") + model = PAttributeType + # django_get_or_create = ('name',) + name = factory.Faker('Int') class ResourceAttributeTypeFactory(DjangoModelFactory): class Meta: model = ResourceAttributeType - attribute_type = factory.SubFactory(AttributeTypeFactory) + attribute_type = factory.SubFactory(RAttributeTypeFactory) name = factory.Faker("word") is_required = factory.Faker("boolean") is_unique_per_resource = factory.Faker("boolean") @@ -370,6 +372,17 @@ def setup_models(test_case): AllocationUserStatusChoiceFactory(name=status_choice) for status in ['Pending', 'Approved', 'Denied']: AllocationChangeStatusChoiceFactory(name=status) + for attribute_type in ['Active/Inactive', 'Date', 'Int', 'Text', 'Attribute Expanded Text', 'Float']: + RAttributeTypeFactory(name=attribute_type) + for resource_attribute_type, attribute_type in ( + ('slurm_cluster', 'Text'), + ('Owner', 'Text'), + ('ServiceEnd', 'Date'), + ): + ResourceAttributeTypeFactory( + name=resource_attribute_type, + attribute_type=RAttributeType.objects.get(name=attribute_type), + ) for resource_type in ['Storage', 'Cluster', 'Cluster Partition', 'Compute Node']: ResourceTypeFactory(name=resource_type) for resource, r_id, rtype in [ diff --git a/coldfront/core/test_helpers/test_data/test_fixtures/resource.json b/coldfront/core/test_helpers/test_data/test_fixtures/resource.json deleted file mode 100644 index 23ad32b7f..000000000 --- a/coldfront/core/test_helpers/test_data/test_fixtures/resource.json +++ /dev/null @@ -1,97 +0,0 @@ -[ - - { - "model": "resource.attributetype", - "pk": 1, - "fields": { - "created": "2024-10-10T01:18:54.259Z", - "modified": "2024-10-10T01:18:54.259Z", - "name": "Active/Inactive" - } - }, - { - "model": "resource.attributetype", - "pk": 2, - "fields": { - "created": "2024-10-10T01:18:54.266Z", - "modified": "2024-10-10T01:18:54.266Z", - "name": "Date" - } - }, - { - "model": "resource.attributetype", - "pk": 3, - "fields": { - "created": "2024-10-10T01:18:54.277Z", - "modified": "2024-10-10T01:18:54.277Z", - "name": "Int" - } - }, - - { - "model": "resource.attributetype", - "pk": 5, - "fields": { - "created": "2024-10-10T01:18:54.290Z", - "modified": "2024-10-10T01:18:54.290Z", - "name": "Text" - } - }, - { - "model": "resource.attributetype", - "pk": 7, - "fields": { - "created": "2024-10-10T01:18:54.300Z", - "modified": "2024-10-10T01:18:54.300Z", - "name": "Attribute Expanded Text" - } - }, - { - "model": "resource.attributetype", - "pk": 8, - "fields": { - "created": "2024-10-10T01:18:54.305Z", - "modified": "2024-10-10T01:18:54.305Z", - "name": "Float" - } - }, - { - "model": "resource.resourceattributetype", - "pk": 10, - "fields": { - "created": "2024-10-11T00:26:36.739Z", - "modified": "2024-10-11T00:26:36.739Z", - "attribute_type": 5, - "name": "slurm_cluster", - "is_required": false, - "is_unique_per_resource": false, - "is_value_unique": false - } - }, - { - "model": "resource.resourceattributetype", - "pk": 15, - "fields": { - "created": "2025-03-21T21:50:07.174Z", - "modified": "2025-03-21T21:50:07.174Z", - "attribute_type": 5, - "name": "Owner", - "is_required": false, - "is_unique_per_resource": false, - "is_value_unique": false - } - }, - { - "model": "resource.resourceattributetype", - "pk": 16, - "fields": { - "created": "2025-03-21T21:50:07.177Z", - "modified": "2025-03-21T21:50:07.177Z", - "attribute_type": 2, - "name": "ServiceEnd", - "is_required": false, - "is_unique_per_resource": false, - "is_value_unique": false - } - } -] From 65f200e9b912b3477b70bc2a4bbcff1cfe670e01 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:22:48 -0700 Subject: [PATCH 17/24] typo --- coldfront/core/test_helpers/factories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/test_helpers/factories.py b/coldfront/core/test_helpers/factories.py index 82dc142f9..f75cf86f2 100644 --- a/coldfront/core/test_helpers/factories.py +++ b/coldfront/core/test_helpers/factories.py @@ -218,7 +218,7 @@ class Meta: class RAttributeTypeFactory(DjangoModelFactory): class Meta: - model = PAttributeType + model = RAttributeType # django_get_or_create = ('name',) name = factory.Faker('Int') From a64e79b0d759c3e7f1486eee39d8eb320f95d3fe Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:27:25 -0700 Subject: [PATCH 18/24] remove RESOURCE_FIXTURES from portal tests --- coldfront/core/portal/tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/coldfront/core/portal/tests.py b/coldfront/core/portal/tests.py index 31410fb4a..971c19b46 100644 --- a/coldfront/core/portal/tests.py +++ b/coldfront/core/portal/tests.py @@ -6,9 +6,6 @@ from coldfront.core.resource.models import AttributeType UTIL_FIXTURES = ['coldfront/core/test_helpers/test_data/test_fixtures/ifx.json'] -RESOURCE_FIXTURES = [ - "coldfront/core/test_helpers/test_data/test_fixtures/resource.json", -] class PortalViewTest(TestCase): """Base class for portal view tests @@ -33,7 +30,6 @@ def test_center_summary(self): class HomePageTest(PortalViewTest): - fixtures = RESOURCE_FIXTURES def test_pi_home_page(self): """check that the pi home page displays properly with the existing database From 1876beccce07c89589653cea7839c893a2ca6cf0 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:32:25 -0700 Subject: [PATCH 19/24] fix resource search test --- coldfront/core/resource/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/resource/tests.py b/coldfront/core/resource/tests.py index e38108df8..89fdb1bdb 100644 --- a/coldfront/core/resource/tests.py +++ b/coldfront/core/resource/tests.py @@ -86,7 +86,7 @@ def test_can_filter_by_name(self): ResourceFactory(name="active_resource", resource_type__name='Compute Node') search_url = f'{self.url}?resource_name=archived_resource' utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource') - utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'archived_resource2') + utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource2') utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'active_resource') search_url = f'{self.url}?resource_name=archived_resource2' utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource2') From 0efc28cf4ad76d0d2b583de6f86ffec5208395e3 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 18 Apr 2025 16:39:32 -0700 Subject: [PATCH 20/24] fix resource search test --- coldfront/core/resource/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/coldfront/core/resource/tests.py b/coldfront/core/resource/tests.py index 89fdb1bdb..0fdcde19a 100644 --- a/coldfront/core/resource/tests.py +++ b/coldfront/core/resource/tests.py @@ -90,7 +90,6 @@ def test_can_filter_by_name(self): utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'active_resource') search_url = f'{self.url}?resource_name=archived_resource2' utils.page_contains_for_user(self, self.pi_user, search_url, 'archived_resource2') - utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'archived_resource') utils.page_does_not_contain_for_user(self, self.pi_user, search_url, 'active_resource') From 5356bdffd7aca163b0e90923e9a5926b01904187 Mon Sep 17 00:00:00 2001 From: Antonio Rodriguez Date: Wed, 9 Apr 2025 14:12:45 +0200 Subject: [PATCH 21/24] Remove if statement --- coldfront/core/resource/forms.py | 5 + .../resource/templates/resource_detail.html | 8 ++ .../resource_resourceattribute_edit.html | 101 ++++++++++++++++++ coldfront/core/resource/urls.py | 2 + coldfront/core/resource/views.py | 86 +++++++++++++++ 5 files changed, 202 insertions(+) create mode 100644 coldfront/core/resource/templates/resource_resourceattribute_edit.html diff --git a/coldfront/core/resource/forms.py b/coldfront/core/resource/forms.py index 559c9e991..9a1a2268b 100644 --- a/coldfront/core/resource/forms.py +++ b/coldfront/core/resource/forms.py @@ -58,6 +58,11 @@ def __init__(self, *args, **kwargs): self.fields['resource_attribute_type'].queryset = self.fields['resource_attribute_type'].queryset.order_by(Lower('name')) +class ResourceAttributeUpdateForm(forms.Form): + resource_attribute_type_name = forms.CharField(max_length=250, required=False, disabled=True) + value = forms.CharField(max_length=350, required=True) + + class ResourceAllocationUpdateForm(forms.Form): allocation_pk = forms.IntegerField(required=False) project = forms.CharField(max_length=250, required=False, disabled=True) diff --git a/coldfront/core/resource/templates/resource_detail.html b/coldfront/core/resource/templates/resource_detail.html index f3a021246..24249690b 100644 --- a/coldfront/core/resource/templates/resource_detail.html +++ b/coldfront/core/resource/templates/resource_detail.html @@ -28,6 +28,11 @@

Resource Information

+ {% if request.user.is_superuser and not owner_set%} + + {% endif %} {% csrf_token %}
@@ -118,6 +123,9 @@

Resou Add Resource Attribute + + Edit Resource Attributes + Delete Resource Attributes diff --git a/coldfront/core/resource/templates/resource_resourceattribute_edit.html b/coldfront/core/resource/templates/resource_resourceattribute_edit.html new file mode 100644 index 000000000..96a761bab --- /dev/null +++ b/coldfront/core/resource/templates/resource_resourceattribute_edit.html @@ -0,0 +1,101 @@ +{% extends "common/base.html" %} +{% load crispy_forms_tags %} +{% load static %} +{% load django_tables2 %} +{% load mathfilters %} +{% load common_tags %} +{% load fasrc_tags %} + +{% block title %}Edit Resource Attributes for {% endblock %} + +{% block content %} +
+
+

Edit Resource Attributes for {{resource.name}} {{resource.resource_type.name}}

+
+
+ +
+
+
+
+
+ {{ formset.management_form }} + {% csrf_token %} +

+ + + + + + + + + {% for form in formset.forms %} + + + + + {% if form.errors %} +
{{ form.errors }}
+ {% endif %} + + {% endfor %} + + + + +
#AttributeValue
{{ forloop.counter }} {{form.resource_attribute_type_name.value}} {{form.value}}
+ + {% if request.user.is_superuser %} + + {% endif %} +
+
+ + + + + + + + + + +{% endblock %} + diff --git a/coldfront/core/resource/urls.py b/coldfront/core/resource/urls.py index 30ec4fad7..4adbbaf81 100644 --- a/coldfront/core/resource/urls.py +++ b/coldfront/core/resource/urls.py @@ -14,4 +14,6 @@ resource_views.ResourceAttributeDeleteView.as_view(), name='resource-attribute-delete'), path('/resourceallocations/edit', resource_views.ResourceAllocationsEditView.as_view(), name='resource-allocations-edit'), + path('/resourceattributes/edit', resource_views.ResourceAttributeEditView.as_view(), + name='resource-attributes-edit'), ] \ No newline at end of file diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index d2eb4532d..c663e082f 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -22,6 +22,7 @@ ResourceAttributeCreateForm, ResourceSearchForm, ResourceAttributeDeleteForm, + ResourceAttributeUpdateForm, ResourceAllocationUpdateForm, ) from coldfront.core.allocation.models import AllocationAttributeType, AllocationAttribute @@ -179,6 +180,12 @@ def get_context_data(self, **kwargs): except ObjectDoesNotExist: owner = owner context['owner'] = owner + if 'Compute Node' in resource_obj.resource_type.name: + owner_attribute_list = list(filter(lambda attribute: attribute.resource_attribute_type.name.lower() in ['owner'], attributes)) + owner_attribute_value = '' + if len(owner_attribute_list) == 1: + owner_attribute_value = owner_attribute_list[0].value + context['owner_set'] = bool(owner_attribute_value) context['allocations'] = allocations context['resource'] = resource_obj @@ -239,6 +246,85 @@ def get_success_url(self): return reverse('resource-detail', kwargs={'pk': self.kwargs.get('pk')}) +class ResourceAttributeEditView(LoginRequiredMixin, UserPassesTestMixin, TemplateView): + template_name = 'resource_resourceattribute_edit.html' + + def test_func(self): + """UserPassesTestMixin Tests""" + if self.request.user.is_superuser: + return True + err = 'You do not have permission to edit resource attributes.' + messages.error(self.request, err) + return False + + def get_formset_initial_data(self, resource_attributes): + edit_attributes_formset_initial_data = [] + if resource_attributes: + for attribute in resource_attributes: + edit_attributes_formset_initial_data.append( + { + 'resource_attribute_type_name': attribute.resource_attribute_type.name, + 'value': attribute.value, + } + ) + return edit_attributes_formset_initial_data + + def get_context_data(self, resource_obj): + context = {} + resource_attributes = resource_obj.resourceattribute_set.all() + if resource_attributes: + ResourceAttributeUpdateFormSet = formset_factory( + ResourceAttributeUpdateForm, + max_num=len(resource_attributes), + extra=0 + ) + edit_attributes_formset_initial_data = self.get_formset_initial_data(resource_attributes) + formset = ResourceAttributeUpdateFormSet( + initial=edit_attributes_formset_initial_data, + prefix='attributesform' + ) + context['formset'] = formset + context['resource'] = resource_obj + return context + + def get(self, request, *args, **kwargs): + resource_obj = get_object_or_404(Resource, pk=self.kwargs.get('pk')) + context = self.get_context_data(resource_obj) + return render(request, self.template_name, context) + + def post(self, request, *args, **kwargs): + pk = self.kwargs.get('pk') + resource_obj = get_object_or_404(Resource, pk=pk) + resource_attributes = resource_obj.resourceattribute_set.all() + edit_attributes_formset_initial_data = self.get_formset_initial_data(resource_attributes) + logger.warning(f'initial_data {edit_attributes_formset_initial_data}') + + ResourceAttributeUpdateFormSet = formset_factory( + ResourceAttributeUpdateForm, + max_num=len(resource_attributes), + extra=0 + ) + formset = ResourceAttributeUpdateFormSet( + request.POST, + initial=edit_attributes_formset_initial_data, + prefix='attributesform' + ) + if formset.is_valid(): + for form in formset.forms: + attribute_name = form.cleaned_data.get('resource_attribute_type_name') + attribute_value = form.cleaned_data.get('value') + resource_attribute = [attribute for attribute in resource_attributes if attribute.resource_attribute_type.name == attribute_name][0] + resource_attribute.value = attribute_value + resource_attribute.save() + messages.success(request, 'Resource attributes update complete.') + return HttpResponseRedirect(reverse('resource-attributes-edit', kwargs={'pk': pk})) + else: + messages.error(request, 'Errors encountered, changes not saved. Check the form for details') + context = self.get_context_data(resource_obj) + context['formset'] = formset + return render(request, self.template_name, context) + + class ResourceAttributeDeleteView(LoginRequiredMixin, UserPassesTestMixin, TemplateView): template_name = 'resource_resourceattribute_delete.html' From 741b9063f367c0f1ff70143bb3c285f70fc1b4ca Mon Sep 17 00:00:00 2001 From: Aaron Kitzmiller Date: Fri, 18 Apr 2025 12:40:22 -0400 Subject: [PATCH 22/24] Don't make products out of Resources that won't be charged anything. Lots of partitions, etc. are going to be setup and they would clutter the product list Add some logging including admin emails for errors with ifx and ifxbilling Slightly better error handling for fiine errors --- coldfront/config/logging.py | 23 +++++++++++++++-- coldfront/plugins/ifx/models.py | 46 +++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/coldfront/config/logging.py b/coldfront/config/logging.py index f71cac929..5609e1c92 100644 --- a/coldfront/config/logging.py +++ b/coldfront/config/logging.py @@ -15,6 +15,14 @@ LOGGING = { 'version': 1, 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + }, + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue' + }, + }, 'formatters': { 'key-events': { "()": "django.utils.log.ServerFormatter", @@ -31,6 +39,12 @@ 'console': { 'class': 'logging.StreamHandler', }, + 'console_debug': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + }, 'django-q': { 'class': 'logging.handlers.TimedRotatingFileHandler', 'filename': 'logs/django-q.log', @@ -47,6 +61,11 @@ 'formatter': 'key-events', 'level': 'INFO', }, + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler', + }, # 'file': { # 'class': 'logging.FileHandler', # 'filename': '/tmp/debug.log', @@ -66,11 +85,11 @@ 'handlers': ['django-q', 'key-events'], }, 'ifx': { - 'handlers': ['console', 'key-events'], + 'handlers': ['console', 'key-events', 'console_debug', 'mail_admins'], 'level': 'INFO', }, 'ifxbilling': { - 'handlers': ['console', 'key-events'], + 'handlers': ['console', 'key-events', 'console_debug', 'mail_admins'], 'level': 'INFO', }, 'coldfront': { diff --git a/coldfront/plugins/ifx/models.py b/coldfront/plugins/ifx/models.py index a00e820a3..f4bb5d40d 100644 --- a/coldfront/plugins/ifx/models.py +++ b/coldfront/plugins/ifx/models.py @@ -2,6 +2,7 @@ Models for ifxbilling plugin ''' import logging +import json from decimal import Decimal from datetime import datetime from django.utils import timezone @@ -10,15 +11,16 @@ from django.db.models.signals import post_save from django.dispatch import receiver from django.conf import settings +from fiine.client import API as FiineAPI +from fiine.client import ApiException from coldfront.core.allocation.models import AllocationUser from coldfront.core.resource.models import Resource from coldfront.core.project.models import Project from ifxbilling.models import ProductUsage, Product, Facility from ifxbilling.fiine import create_new_product from ifxuser.models import Organization -from fiine.client import API as FiineAPI -logger = logging.getLogger(__name__) +logger = logging.getLogger('ifx') class ProjectOrganization(models.Model): @@ -131,25 +133,37 @@ def resource_post_save(sender, instance, **kwargs): ''' Ensure that there is a Product for each Resource ''' - if not kwargs.get('raw') and not instance.resource_type.name in ["Storage Tier", "Cluster", "Cluster Partition", "Compute Node"]: + if not kwargs.get('raw') and not instance.resource_type.name in ["Storage Tier", "Cluster", "Cluster Partition", "Compute Node"] and instance.requires_payment: try: product_resource = ProductResource.objects.get(resource=instance) except ProductResource.DoesNotExist: # Need to create a Product and ProductResource for this Resource products = [] - if not settings.FIINELESS: - products = FiineAPI.listProducts(product_name=instance.name) - facility = Facility.objects.get(name='Research Computing Storage') - if not products: - product = create_new_product(product_name=instance.name, product_description=instance.name, facility=facility) - else: - fiine_product = products[0].to_dict() - fiine_product.pop('object_code_category') - fiine_product['facility'] = facility - fiine_product['billing_calculator'] = 'coldfront.plugins.ifx.calculator.NewColdfrontBillingCalculator' - fiine_product['billable'] = instance.requires_payment - (product, created) = Product.objects.get_or_create(**fiine_product) - product_resource = ProductResource.objects.create(product=product, resource=instance) + try: + if not settings.FIINELESS: + products = FiineAPI.listProducts(product_name=instance.name) + facility = Facility.objects.get(name='Research Computing Storage') + if not products: + product = create_new_product(product_name=instance.name, product_description=instance.name, facility=facility) + else: + fiine_product = products[0].to_dict() + fiine_product.pop('object_code_category') + fiine_product['facility'] = facility + fiine_product['billing_calculator'] = 'coldfront.plugins.ifx.calculator.NewColdfrontBillingCalculator' + (product, created) = Product.objects.get_or_create(**fiine_product) + product_resource = ProductResource.objects.create(product=product, resource=instance) + except ApiException as e: + if e.status == 400: + msg = json.loads(e.body) + if 'Duplicate' in str(e): + msg = 'Duplicate entry in Fiine' + if e.status == 401: + msg = 'Unauthorized' + logger.error(f'Error creating Product for Resource {instance}: {msg}') + raise Exception(msg) from e + except Exception as e: + logger.error(f'Error creating Product for Resource {instance}: {e}') + raise class SuUser(get_user_model()): ''' From c85b4511b6f7c2fbfe4a140b71c8d5bc3e48332b Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 22 Apr 2025 21:45:35 +0000 Subject: [PATCH 23/24] change owner identification in resourcedetailview --- coldfront/core/resource/templates/resource_detail.html | 2 +- coldfront/core/resource/views.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/coldfront/core/resource/templates/resource_detail.html b/coldfront/core/resource/templates/resource_detail.html index 24249690b..216449f32 100644 --- a/coldfront/core/resource/templates/resource_detail.html +++ b/coldfront/core/resource/templates/resource_detail.html @@ -28,7 +28,7 @@

Resource Information

- {% if request.user.is_superuser and not owner_set%} + {% if request.user.is_superuser and owner != None and owner == '' %} diff --git a/coldfront/core/resource/views.py b/coldfront/core/resource/views.py index c663e082f..c9bfaa4f9 100644 --- a/coldfront/core/resource/views.py +++ b/coldfront/core/resource/views.py @@ -180,12 +180,6 @@ def get_context_data(self, **kwargs): except ObjectDoesNotExist: owner = owner context['owner'] = owner - if 'Compute Node' in resource_obj.resource_type.name: - owner_attribute_list = list(filter(lambda attribute: attribute.resource_attribute_type.name.lower() in ['owner'], attributes)) - owner_attribute_value = '' - if len(owner_attribute_list) == 1: - owner_attribute_value = owner_attribute_list[0].value - context['owner_set'] = bool(owner_attribute_value) context['allocations'] = allocations context['resource'] = resource_obj From 51b1f7c7bc457b7ff2c028fa7c7c9349e3cd8950 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:34:44 -0700 Subject: [PATCH 24/24] add task for slurm_manage_resources --- .../core/utils/management/commands/add_scheduled_tasks.py | 2 +- coldfront/plugins/slurm/tasks.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/coldfront/core/utils/management/commands/add_scheduled_tasks.py b/coldfront/core/utils/management/commands/add_scheduled_tasks.py index a2ab2f5b7..31a4f3865 100644 --- a/coldfront/core/utils/management/commands/add_scheduled_tasks.py +++ b/coldfront/core/utils/management/commands/add_scheduled_tasks.py @@ -29,7 +29,7 @@ def handle(self, *args, **options): 'sftocf': ['pull_sf_push_cf', 'update_zones'], 'lfs': ['pull_lfs_filesystem_stats'], 'ldap': ['update_group_membership_ldap', 'id_add_projects'], - 'slurm': ['slurm_sync'], + 'slurm': ['slurm_sync', 'slurm_manage_resources'], 'xdmod': ['xdmod_usage'], } scheduled = [task.func for task in Schedule.objects.all()] diff --git a/coldfront/plugins/slurm/tasks.py b/coldfront/plugins/slurm/tasks.py index 317933d96..53e1bc8d5 100644 --- a/coldfront/plugins/slurm/tasks.py +++ b/coldfront/plugins/slurm/tasks.py @@ -4,3 +4,8 @@ def slurm_sync(): """ID and add new slurm allocations from ADGroup and ADUser data """ call_command('slurm_sync') + +def slurm_manage_resources(): + """ID and add new slurm allocations from ADGroup and ADUser data + """ + call_command('slurm_manage_resources')