Merge branch 'sinfo_resource_management' of https://github.com/theam/fasrc-coldfront into sinfo_resource_management

Author: antonio-rodriguez-tam

.github/workflows/django.yml

@@ -58,7 +58,7 @@ jobs:
         docker run -d --rm --name coldfront -v `pwd`:/usr/src/app \
         -e PLUGIN_SFTOCF=True -e SFUSER=${{ secrets.SFUSER }} -e SFPASS=${{ secrets.SFPASS }} \
         -e PLUGIN_IFX=True -e PLUGIN_FASRC=True -e NEO4JP=${{ secrets.NEO4JP }} \
-        -e PLUGIN_API=True -e PLUGIN_LDAP=True \
+        -e PLUGIN_API=True -e PLUGIN_LDAP=True -e PLUGIN_LFS=True \
         -e AUTH_LDAP_SERVER_URI=${{ secrets.AUTH_LDAP_SERVER_URI }} \
         -e AUTH_LDAP_BIND_DN=${{ secrets.AUTH_LDAP_BIND_DN }} \
         -e AUTH_LDAP_BIND_PASSWORD=${{ secrets.AUTH_LDAP_BIND_PASSWORD }} \

coldfront/config/plugins/lfs.py

@@ -0,0 +1,8 @@
+from coldfront.config.env import ENV
+from coldfront.config.logging import LOGGING
+from coldfront.config.base import INSTALLED_APPS
+
+INSTALLED_APPS += ['coldfront.plugins.lfs']
+
+LFS_HOST = ENV.str('LFS_HOST', default='localhost')
+LFS_PORT = ENV.int('LFS_PORT', default=50051)

coldfront/config/settings.py

@@ -28,6 +28,7 @@
     'PLUGIN_LDAP_USER_SEARCH': 'plugins/ldap_user_search.py',
     'PLUGIN_API': 'plugins/api.py',
     'PLUGIN_LDAP': 'plugins/ldap_fasrc.py',
+    'PLUGIN_LFS': 'plugins/lfs.py',
     'PLUGIN_SFTOCF': 'plugins/sftocf.py',
     'PLUGIN_FASRC': 'plugins/fasrc.py',
     'PLUGIN_FASRC_MONITORING': 'plugins/fasrc_monitoring.py',

coldfront/core/allocation/templates/allocation/allocation_add_users.html

@@ -155,11 +155,12 @@ <h2>Add users to allocation for project: {{allocation.project.title}}</h2>
     }
     if (q.indexOf('\n') >= 0) {
       $("#id_id_search_by_0_1").prop("checked", true);
+      q = q.replace(/(?:\r\n|\r|\n)/g, ' ')
     }
 
     var pk = "{{ pk }}"
     $.ajax({
-        url : "{% url 'allocation-add-users' allocation.pk %}?search="+q, // the endpoint
+        url : encodeURI("{% url 'allocation-add-users' allocation.pk %}?search="+q), // the endpoint
         type : "GET", // http method
         // handle a successful response
         success : function(data) {

coldfront/core/allocation/tests/test_views_cluster.py

@@ -1,6 +1,7 @@
 """View tests for cluster allocations"""
 import logging
 
+import urllib.parse
 from django.db.models import Count
 from django.test import TestCase
 from django.urls import reverse
@@ -155,6 +156,26 @@ def test_allocationaddusersview_access(self):
         utils.test_user_can_access(self, self.pi_user, self.url)
         utils.test_user_cannot_access(self, self.proj_allocationuser, self.url)
 
+    def test_allocation_non_project_user_space_separated_search(self):
+        username_list = ['iberlin', 'gvanrossum']
+        space_separated_search = " ".join(username_list)
+        self.client.force_login(self.admin_user, backend="django.contrib.auth.backends.ModelBackend")
+        url = f'/allocation/{self.cluster_allocation.pk}/add-users?search={space_separated_search}'
+        space_separated_search_response = str(self.client.get(url).content)
+        self.assertTrue('Found 2 matchs.' in space_separated_search_response)
+        self.assertTrue('<td>iberlin</td>' in space_separated_search_response)
+        self.assertTrue('<td>gvanrossum</td>' in space_separated_search_response)
+
+    def test_allocation_non_project_user_new_line_separated_search(self):
+        username_list = ['iberlin', 'gvanrossum']
+        new_line_separated_search = urllib.parse.quote_plus("\n".join(username_list))
+        self.client.force_login(self.admin_user, backend="django.contrib.auth.backends.ModelBackend")
+        url = f'/allocation/{self.cluster_allocation.pk}/add-users?search={new_line_separated_search}'
+        new_line_separated_search_response = str(self.client.get(url).content)
+        self.assertTrue('<td>iberlin</td>' in new_line_separated_search_response)
+        self.assertTrue('<td>gvanrossum</td>' in new_line_separated_search_response)
+        self.assertTrue('Found 2 matchs.' in new_line_separated_search_response)
+
 
 class ClusterAllocationEditUsersViewTest(ClusterAllocationViewBaseTest):
     """Tests for the AllocationEditUsersView"""

coldfront/core/allocation/views.py

@@ -7,6 +7,7 @@
 
 from io import BytesIO
 from lib2to3.fixes.fix_input import context
+import requests
 
 from xhtml2pdf import pisa
 
@@ -869,15 +870,26 @@ def get_users_to_add(self, allocation_obj):
         return users_to_add
 
     def non_project_users_search(self, allocation_obj, search_term):
-        like_filter = (Q(username__icontains=search_term) or Q(first_name__icontains=search_term) or Q(
-            last_name__icontains=search_term) or Q(title_icontains=search_term))
-        non_project_users = (
-            self.get_non_project_users_to_add(allocation_obj, return_all=True)
-            .filter(like_filter)
-            .exclude(project=allocation_obj.project)
-            .values('username', 'first_name', 'last_name', 'email')
-        )
-        return non_project_users
+        user_list = []
+        search_term = requests.utils.unquote(search_term)
+        user_search = search_term.split(" ") if " " in search_term else search_term.splitlines()
+        for username in user_search:
+            like_filter = (
+                Q(username__icontains=username)
+                or Q(first_name__icontains=username)
+                or Q(last_name__icontains=username)
+                or Q(title_icontains=username)
+            )
+
+            non_project_users = (
+                self.get_non_project_users_to_add(allocation_obj, return_all=True)
+                .filter(like_filter)
+                .exclude(project=allocation_obj.project)
+                .values('username', 'first_name', 'last_name', 'email')
+            )
+            user_list += non_project_users
+
+        return user_list
 
     def search_non_project_users(self, allocation_obj, search_term, request):
         found_non_project_users = self.non_project_users_search(allocation_obj, search_term)

coldfront/core/portal/views.py

@@ -269,7 +269,11 @@ def allocation_by_fos(request):
 def allocation_summary(request):
 
     allocation_resources = [
-        allocation.get_parent_resource.parent_resource if allocation.get_parent_resource.parent_resource else allocation.get_parent_resource for allocation in Allocation.objects.filter(status__name='Active')]
+        allocation.get_parent_resource.parent_resource
+        if allocation.get_parent_resource.parent_resource
+        else allocation.get_parent_resource
+        for allocation in Allocation.objects.filter(status__name='Active').distinct()
+    ]
 
     allocations_count_by_resource = dict(Counter(allocation_resources))
 

coldfront/core/project/templates/project/project_detail.html

@@ -128,7 +128,7 @@ <h4>Storage &nbsp;
                   <td>{{ allocation.get_parent_resource.name }}</td>
                   <td>{{ allocation.path }}</td>
                   <td>{{ allocation.allocationuser_set.count }}</td>
-                  <td>{{ allocation.size|floatformat:1 }} {{ allocation.get_parent_resource.unit }}</td>
+                  <td>{{ allocation.size|floatformat:1 }} {{ allocation.get_parent_resource.quantity_label }}</td>
                   <td>{{ allocation.usage|floatformat:1 }}</td>
                   <td>
                     {% if allocation.requires_payment %}
@@ -198,7 +198,7 @@ <h4>Cluster</h4>
                   <td>{{ allocation.get_parent_resource.name }}</td>
                   <td>{{ allocation.allocationuser_set.count }}</td>
                   <td>
-                    {{ allocation.size|floatformat:1 }} {{ allocation.get_parent_resource.unit }}
+                    {{ allocation.size|floatformat:1 }} {{ allocation.get_parent_resource.quantity_label }}
                   </td>
                   <td>
                     <a href="{% url 'allocation-detail' allocation.pk %}">

coldfront/core/resource/management/commands/add_resource_defaults.py

@@ -22,14 +22,18 @@ def handle(self, *args, **options):
             AttributeType.objects.get_or_create(name=attribute_type)
 
         for resource_attribute_type, attribute_type in (
+            #FASRC
             ('capacity_tb', 'Float'),
             ('free_tb', 'Float'),
             ('used_tb', 'Float'),
             ('file_count', 'Int'),
             ('allocated_tb', 'Float'),
             ('ChangeableAllocations', 'Yes/No'),
-            ('Core Count', 'Int'),
+            ('CPU Count', 'Int'),
             ('GPU Count', 'Int'),
+            ('Features', 'Text'),
+            # UBCCR
+            ('Core Count', 'Int'),
             # ('expiry_time', 'Int'),
             # ('fee_applies', 'Yes/No'),
             ('Node Count', 'Int'),
@@ -70,7 +74,8 @@ def handle(self, *args, **options):
             # ('Storage', 'NAS storage'),
         ):
             ResourceType.objects.get_or_create(
-                name=resource_type, description=description
+                name=resource_type,
+                defaults={'description': description},
             )
 
 

coldfront/core/resource/templates/resource_detail.html

@@ -79,7 +79,7 @@ <h3><i class="fas fa-list" aria-hidden="true"></i> Resource Information</h3>
 </div>
 
 <!-- Start Storage Report -->
-{% if resource.resource_type.name == "Storage" %}
+{% if resource.resource_type.name == "Storage" and user_is_manager %}
   <div class="card mb-3">
     <div class="card-header">
       <h3 class="d-inline"><i class="fas fa-info-circle" aria-hidden="true"></i> Storage Report</h3>
@@ -91,7 +91,7 @@ <h3 class="d-inline"><i class="fas fa-info-circle" aria-hidden="true"></i> Stora
 {% endif %}
 
 <!-- Start Resource Attributes -->
-{% if attributes or request.user.is_superuser %}
+{% if user_is_manager %}
   <div class="card mb-3">
     <div class="card-header">
       <h3 class="d-inline"><i class="fas fa-info-circle" aria-hidden="true"></i> Resource Attributes</h3>
@@ -170,7 +170,7 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Ad
 </div>
 
 <!-- Start Resource Allocations -->
-{% if user_is_manager and 'Cluster' in resource.resource_type.name %}
+{% if user_is_manager %}
   <div class="card mb-3">
     <div class="card-header">
       <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Allocations</h3>
@@ -179,11 +179,13 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
       {# {% if user_sync_dt %}#}
       {# <span class="float-right">Last Sync: {{user_sync_dt}}</span>#}
       {# {% endif %}#}
-      <div class="float-right">
-        <a class="btn btn-danger" href="{% url 'resource-allocations-edit' resource.pk %}" role="button">
-          <i class="fas fa-edit" aria-hidden="true"></i> Edit Resource Allocations
-        </a>
-      </div>
+      {% if 'Cluster' in resource.resource_type.name %}
+        <div class="float-right">
+          <a class="btn btn-danger" href="{% url 'resource-allocations-edit' resource.pk %}" role="button">
+            <i class="fas fa-edit" aria-hidden="true"></i> Edit Resource Allocations
+          </a>
+        </div>
+      {% endif %}
     </div>
     <div class="card-body">
       <div class="table-responsive">
@@ -194,22 +196,28 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
               <tr>
                 <th scope="col">Project</th>
                 <th scope="col">Users</th>
-                <th scope="col">CPU Hours</th>
-                <th scope="col">Percent Usage</th>
-                <th scope="col">RawShare</th>
-                <th scope="col">EffectvUsage <a class="info-button" title="EffectvUsage" data-toggle="popover" data-trigger="click"
-                    data-content="The fraction of the cluster the account has been granted. For more information, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."><i class="fas fa-info-circle"
-                    aria-hidden="true"></i></a>
-                </th>
-                <th scope="col">NormShares <a class="info-button" title="NormShares" data-toggle="popover" data-trigger="click"
-                      data-content="Calculated fairshare per user. For more information on this number, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."
-		      ><i class="fas fa-info-circle" aria-hidden="true"></i></a>
-                </th>
-                <th scope="col">FairShare <a class="info-button" title="FairShare" data-toggle="popover" data-trigger="click"
-                     data-content="User fairshare, calculated by the equation 2^(-EffectvUsage/NormShares). For more information on fairshare calculation, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."
-		     ><i class="fas fa-info-circle" aria-hidden="true"></i>
-                  </a>
-                </th>
+                {% if 'Cluster' in resource.resource_type.name %}
+                  <th scope="col">CPU Hours</th>
+                  <th scope="col">Percent Usage</th>
+                  <th scope="col">RawShare</th>
+                  <th scope="col">EffectvUsage <a class="info-button" title="EffectvUsage" data-toggle="popover" data-trigger="click"
+                      data-content="The fraction of the cluster the account has been granted. For more information, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."><i class="fas fa-info-circle"
+                      aria-hidden="true"></i></a>
+                  </th>
+                  <th scope="col">NormShares <a class="info-button" title="NormShares" data-toggle="popover" data-trigger="click"
+                        data-content="Calculated fairshare per user. For more information on this number, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."
+		      ><i cl  ass="fas fa-info-circle" aria-hidden="true"></i></a>
+                  </th>
+                  <th scope="col">FairShare <a class="info-button" title="FairShare" data-toggle="popover" data-trigger="click"
+                       data-content="User fairshare, calculated by the equation 2^(-EffectvUsage/NormShares). For more information on fairshare calculation, go <a href='https://docs.rc.fas.harvard.edu/kb/fairshare/#articleTOC_4' title='fairshare calculation'>here</a>."
+		     ><i cla  ss="fas fa-info-circle" aria-hidden="true"></i>
+                    </a>
+                  </th>
+                {% elif resource.resource_type.name == 'Storage' %}
+                  <th scope="col">Path</th>
+                  <th scope="col">Quota ({{ resource.quantity_label }})</th>
+                  <th scope="col">Used ({{ resource.quantity_label }})</th>
+                {% endif %}
               </tr>
             </thead>
 
@@ -222,28 +230,44 @@ <h3 class="d-inline"><i class="fas fa-users" aria-hidden="true"></i> Resource Al
                     </a>
                   </td>
                   <td>{{ allocation.user_count }}</td>
-                  <td data-sort="{{allocation.usage}}" name="usage">
-                    {% if allocation.usage is None %}
-                      0
-                    {% else %}
-                      {{ allocation.usage|floatformat:1 }}
-                    {% endif %}
-                  </td>
-                  <td data-sort="{{allocation.usage}}" name="usage_pct">
-                    {% if allocation.usage is None or allocation.usage == 0 %}
-                      0%
-                    {% else %}
-                      {{allocation.usage|div:total_hours|mul:100|floatformat:2 }}%
-                    {% endif %}
-                  </td>
-                  <td>{{ allocation.rawshares}}</td>
-                  <td>{{ allocation.effectvusage }}</td>
-                  <td>{{ allocation.normshares }}</td>
-                  <td>{{ allocation.fairshare }}</td>
-
+                  {% if 'Cluster' in resource.resource_type.name %}
+                    <td data-sort="{{allocation.usage}}" name="usage">
+                      {% if allocation.usage is None %}
+                        0
+                      {% else %}
+                        {{ allocation.usage|floatformat:1 }}
+                      {% endif %}
+                    </td>
+                    <td data-sort="{{allocation.usage}}" name="usage_pct">
+                      {% if allocation.usage is None or allocation.usage == 0 %}
+                        0%
+                      {% else %}
+                        {{allocation.usage|div:total_hours|mul:100|floatformat:2 }}%
+                      {% endif %}
+                    </td>
+                    <td>{{ allocation.rawshares}}</td>
+                    <td>{{ allocation.effectvusage }}</td>
+                    <td>{{ allocation.normshares }}</td>
+                    <td>{{ allocation.fairshare }}</td>
+                  {% elif resource.resource_type.name == 'Storage' %}
+                    <td>{{ allocation.path }}</td>
+                    <td>{{ allocation.size|floatformat:2 }}</td>
+                    <td>{{ allocation.usage|floatformat:2 }}</td>
+                  {% endif %}
                 </tr>
               {% endfor %}
             </tbody>
+            {% if resource.resource_type.name == 'Storage' %}
+              <tfoot>
+                <tr style="background-color:#C2C2C2;font-weight:bold">
+                    <td>Total Tracked Storage</td>
+                    <td></td>
+                    <td></td>
+                    <td>{{ allocation_total.size|floatformat:1}}</td>
+                    <td>{{ allocation_total.usage|floatformat:1}}</td>
+                </tr>
+              </tfoot>
+            {% endif %}
           </table>
         </div>