Merge pull request #203 from hubmapconsortium/test-release

v2.0.12 release

Author: yuanzhou

VERSION

@@ -1 +1 @@
-2.0.11
+2.0.12

src/app.py

@@ -780,9 +780,10 @@ def create_entity(entity_type):
     except schema_errors.InvalidNormalizedEntityTypeException as e:
         bad_request_error(f"Invalid entity type provided: {entity_type}")
 
-    # Execute validators defined in schema yaml before entity creation
+    # Execute entity level validator defined in schema yaml before entity creation
+    # Currently on Dataset and Upload creation require application header
     try:
-        schema_manager.execute_entity_level_validators('before_entity_create_validator', normalized_entity_type, request.headers)
+        schema_manager.execute_entity_level_validator('before_entity_create_validator', normalized_entity_type, request.headers)
     except schema_errors.MissingApplicationHeaderException as e: 
         bad_request_error(e)  
     except schema_errors.InvalidApplicationHeaderException as e: 
@@ -972,13 +973,8 @@ def update_entity(id):
     # Normalize user provided entity_type
     normalized_entity_type = schema_manager.normalize_entity_type(entity_dict['entity_type'])
 
-    # Execute validators defined in schema yaml before entity update
-    try:
-        schema_manager.execute_entity_level_validators('before_entity_update_validator', normalized_entity_type, request.headers)
-    except schema_errors.MissingApplicationHeaderException as e: 
-        bad_request_error(e)  
-    except schema_errors.InvalidApplicationHeaderException as e: 
-        bad_request_error(e)
+    # Note, we don't support entity level validators on entity update via PUT
+    # Only entity create via POST is supported at the entity level
 
     # Validate request json against the yaml schema
     # Pass in the entity_dict for missing required key check, this is different from creating new entity
@@ -988,17 +984,14 @@ def update_entity(id):
         # No need to log the validation errors
         bad_request_error(str(e))
 
-    # Currently only Dataset.status has such proerty level validator
-    # Execute validators defined in schema yaml before entity property update
+    # Execute property level validators defined in schema yaml before entity property update
     try:
-        schema_manager.execute_property_level_validators('before_property_update_validators', normalized_entity_type, request.headers, json_data_dict)
-    except schema_errors.MissingApplicationHeaderException as e: 
-        bad_request_error(e)  
-    except schema_errors.InvalidApplicationHeaderException as e: 
+        schema_manager.execute_property_level_validators('before_property_update_validators', normalized_entity_type, request.headers, entity_dict, json_data_dict)
+    except (schema_errors.MissingApplicationHeaderException, 
+            schema_errors.InvalidApplicationHeaderException, 
+            KeyError, 
+            ValueError) as e: 
         bad_request_error(e)
-    except ValueError as e:
-        bad_request_error(e)
-
 
     # Sample, Dataset, and Upload: additional validation, update entity, after_update_trigger
     # Collection and Donor: update entity

src/app_neo4j_queries.py

@@ -195,7 +195,7 @@ def get_ancestor_organs(neo4j_driver, entity_uuid):
              # apoc.coll.toSet() reruns a set containing unique nodes
              f"RETURN apoc.coll.toSet(COLLECT(organ)) AS {record_field_name}")
 
-    logger.debug("======create_entity() query======")
+    logger.debug("======get_ancestor_organs() query======")
     logger.debug(query)
 
     with neo4j_driver.session() as session:

src/schema/provenance_schema.yaml

@@ -1,13 +1,21 @@
 ############################################# Rules #############################################
-# type: data type of the property, one of the following: string|integer|boolean|list|json_string
-# generated: whether the property is auto genearated or user provided, default to false
-# required_on_create: whether the property is required from user reqeust JSON for entity creation via POST
-# immutable: whether the property can NOT be updated once being created, default to false
-# transient: whether the property to persist in database or not, default to false
-# exposed: whether the property gets returned to the user or not, default to true
-# trigger types: before_create_trigger|after_create_trigger|before_update_trigger|after_update_trigger|on_read_trigger, one property can have none (default) or more than one triggers
+# Entity properties:
+# - type: data type of the property, one of the following: string|integer|boolean|list|json_string
+# - generated: whether the property is auto genearated or user provided, default to false
+# - required_on_create: whether the property is required from user reqeust JSON for entity creation via POST
+# - immutable: whether the property can NOT be updated once being created, default to false
+# - transient: whether the property to persist in database or not, default to false
+# - exposed: whether the property gets returned to the user or not, default to true
+# - trigger types: before_create_trigger|after_create_trigger|before_update_trigger|after_update_trigger|on_read_trigger, one property can have none (default) or more than one triggers
+# - updated_peripherally: a temporary measure to correctly handle any attributes which are potentially updated by multiple triggers
 
-# Entity creation via http POST request, either standalone or derived:
+# Entity level validator: 
+# - types: before_entity_create_validator, a single validation method needed for creating or updating the entity
+
+# Property level validators: 
+# - types: before_property_update_validators, a list of validation methods
+
+# Entity creation via http POST request:
 # - Use `generated: true` to mark a property as to be auto generated by the program instead of from user input JSON
 # - If a property is marked as `generated: true`, either no trigger method needed (E.g. Donor.image_files) or a `before_create_trigger` can be used to generate the value
 # - If a property has `before_create_trigger`, it can't be specified in client request JSON
@@ -27,7 +35,6 @@
 # - If a property has `on_read_trigger`, it must be `transient: true`, meaning it's not stored in neo4j and only available during response
 # - If a property is `transient: true`, it can have `on_read_trigger` or not have one (when `exposed: false`)
 
-
 ############################################# Schema #############################################
 # Shared properties across ACTIVITIES and ENTITIES
 shared_properties: &shared_properties
@@ -98,17 +105,11 @@ doi_properties: &doi_properties
     # So we can't define it as `immutable: true`
     type: string
     description: "The doi of the registered entity. e.g. 10.35079/hbm289.pcbm.487"
-    # Only allowed applications can update this property via PUT
-    before_property_update_validators: 
-      - validate_application_header_before_property_update
   doi_url:
     # This property gets set via a PUT by ingest-api after the DOI is created via DataCite
     # So we can't define it as `immutable: true`
     type: string
     description: "The url from the doi registry for this entity. e.g. https://doi.org/10.35079/hbm289.pcbm.487"
-    # Only allowed applications can update this property via PUT
-    before_property_update_validators: 
-      - validate_application_header_before_property_update
   # Currently cretors and contacts are only available for public Collections
   creators:
     type: list
@@ -228,7 +229,6 @@ ENTITIES:
         required_on_create: true # Only required for create via POST, not update via PUT
         description: "True if the data contains any human genetic sequence information."
       status:
-        # Only allowed applications can update this property via PUT
         before_property_update_validators: 
           - validate_application_header_before_property_update
           - validate_dataset_status_value
@@ -615,8 +615,6 @@ ENTITIES:
   Upload:
     # Only allowed applications can create new Upload via POST
     before_entity_create_validator: validate_application_header_before_entity_create
-    # Only allowed applications can update an existing Upload via PUT
-    before_entity_update_validator: validate_application_header_before_entity_update
     # Upload requires an ancestor of Lab, and and has no allowed decesndents 
     derivation:
       source: false
@@ -633,6 +631,7 @@ ENTITIES:
         description: "Title of the datasets, a sentance or less"
       status:
         before_property_update_validators: 
+          - validate_application_header_before_property_update
           - validate_upload_status_value
         type: string
         generated: true

src/schema/schema_manager.py

@@ -690,18 +690,20 @@ def validate_json_data_against_schema(json_data_dict, normalized_entity_type, ex
 
 
 """
-Execute the entity level validators defined in the schema yaml 
+Execute the entity level validator of the given type defined in the schema yaml 
+before entity creation via POST or entity update via PUT
+Only one validator defined per given validator type, no need to support multiple validators
 
 Parameters
 ----------
 validator_type : str
-    One of the validator types: before_entity_create_validator, before_entity_update_validator
+    One of the validator types: before_entity_create_validator
 normalized_entity_type : str
     One of the normalized entity types defined in the schema yaml: Donor, Sample, Dataset, Upload
 request_headers: Flask request.headers object, behaves like a dict
     The instance of Flask request.headers passed in from application request
 """
-def execute_entity_level_validators(validator_type, normalized_entity_type, request_headers):
+def execute_entity_level_validator(validator_type, normalized_entity_type, request_headers):
     global _schema
 
     # A bit validation
@@ -733,19 +735,22 @@ def execute_entity_level_validators(validator_type, normalized_entity_type, requ
 
 """
 Execute the property level validators defined in the schema yaml 
+before property update via PUT
 
 Parameters
 ----------
 validator_type : str
-    For now only: before_property_update_validators
+    For now only: before_property_update_validators (support multiple validators)
 normalized_entity_type : str
     One of the normalized entity types defined in the schema yaml: Donor, Sample, Dataset, Upload
 request_headers: Flask request.headers object, behaves like a dict
     The instance of Flask request.headers passed in from application request
-request_json_data : dict
+existing_data_dict : dict
+    A dictionary that contains all existing entity properties
+new_data_dict : dict
     The json data in request body, already after the regular validations
 """
-def execute_property_level_validators(validator_type, normalized_entity_type, request_headers, request_json_data):
+def execute_property_level_validators(validator_type, normalized_entity_type, request_headers, existing_data_dict, new_data_dict):
     global _schema
 
     schema_section = None
@@ -758,7 +763,7 @@ def execute_property_level_validators(validator_type, normalized_entity_type, re
 
     for key in properties:
         # Only run the validators for keys present in the request json
-        if (key in request_json_data) and (validator_type in properties[key]):
+        if (key in new_data_dict) and (validator_type in properties[key]):
             # Get a list of defined validators on this property
             validators_list = properties[key][validator_type]
             # Run each validator defined on this property
@@ -769,7 +774,7 @@ def execute_property_level_validators(validator_type, normalized_entity_type, re
 
                     logger.debug(f"To run {validator_type}: {validator_method_name} defined for entity {normalized_entity_type} on property {key}")
 
-                    validator_method_to_call(key, normalized_entity_type, request_headers, request_json_data)
+                    validator_method_to_call(key, normalized_entity_type, request_headers, existing_data_dict, new_data_dict)
                 except schema_errors.MissingApplicationHeaderException as e: 
                     raise schema_errors.MissingApplicationHeaderException(e) 
                 except schema_errors.InvalidApplicationHeaderException as e: 
@@ -782,128 +787,6 @@ def execute_property_level_validators(validator_type, normalized_entity_type, re
                     logger.exception(msg)
 
 
-"""
-Get a list of application applications (other than users) that can 
-create new entity or update the existing entity of the given type
-
-Parameters
-----------
-normalized_entity_type : str
-    One of the normalized entity types: Dataset, Upload
-action : str
-    applications_allowed_on_entity_create or applications_allowed_on_entity_update
-
-Returns
--------
-list
-    A list of applications (normlized with lowercase)
-"""
-def get_entity_level_allowed_applications(normalize_entity_type, action):
-    global _schema
-
-    applications = []
-    entity = _schema['ENTITIES'][normalize_entity_type]
-    key = action
-
-    # When not specified, both users and applications can create this entity
-    if (key in entity) and isinstance(entity[key], list):
-        # Lowercase all applications in the list via a list comprehension
-        applications = [application.lower() for application in entity[key]]
-
-    return applications
-
-"""
-Get a list of applications (other than users) that can 
-update the existing entity property of the given property key
-
-Parameters
-----------
-normalized_entity_type : str
-    Dataset (Dataset.status is the only property requires this for now)
-property_key : str
-    The target property key that requires allowed applications to update
-
-Returns
--------
-list
-    A list of applications (normlized with lowercase)
-"""
-def get_property_level_allowed_applications_on_update_only(normalize_entity_type, property_key):
-    global _schema
-
-    applications = []
-    properties = _schema['ENTITIES'][normalize_entity_type]['properties']
-    action = 'applications_allowed_on_property_update'
-
-    # When not specified, both users and applications can update this entity
-    if (property_key in properties) and (action in properties[property_key]) and isinstance(properties[property_key][action], list):
-        # Lowercase all applications in the list via a list comprehension
-        applications = [application.lower() for application in properties[property_key][action]]
-
-    return applications
-
-"""
-Check if the application allowed to create or update this entity
-
-Parameters
-----------
-normalized_entity_type : str
-    One of the normalized entity types: Dataset, Upload
-request_headers: Flask request.headers object, behaves like a dict
-    The instance of Flask request.headers passed in from application request
-action : str
-    applications_allowed_on_entity_create or applications_allowed_on_entity_update
-"""
-def validate_entity_level_application(normalized_entity_type, request_headers, action):
-    # Get the list of applications allowed to create or update this entity
-    # Returns empty list if no restrictions, meaning both users and aplications can create or update
-    applications_allowed = get_entity_level_allowed_applications(normalized_entity_type, action)
-
-    # When application required
-    if applications_allowed:
-        # HTTP header names are case-insensitive
-        # request_headers.get('X-Hubmap-Application') returns None is the header doesn't exist
-        if not request_headers.get('X-Hubmap-Application'):
-            msg = "Unbale to proceed due to missing X-Hubmap-Application header from request"
-            raise schema_errors.MissingApplicationHeaderException(msg)
-
-        # Use lowercase for comparing the application header value against the yaml
-        if request_headers.get('X-Hubmap-Application').lower() not in applications_allowed:
-            msg = f"Unable to proceed due to invalid X-Hubmap-Application header value: {request_headers.get('X-Hubmap-Application')}"
-            raise schema_errors.InvalidApplicationHeaderException(msg)
-
-
-"""
-Check if the application allowed to update this entity property
-
-Parameters
-----------
-normalized_entity_type : str
-    Dataset (Dataset.status is the only property requires this for now)
-property_key : str
-    The target property key that requires allowed applications to update
-request_headers: Flask request.headers object, behaves like a dict
-    The instance of Flask request.headers passed in from application request
-"""
-def validate_property_level_application_on_update_only(normalized_entity_type, property_key, request_headers):
-    # Get the list of applications allowed to update this entity property, e.g., Dataset.status
-    # Returns empty list if no restrictions, meaning both users and aplications can update
-    applications_allowed = get_property_level_allowed_applications_on_update_only(normalized_entity_type, property_key)
-
-    # When application required
-    if applications_allowed:
-        # HTTP header names are case-insensitive
-        # request_headers.get('X-Hubmap-Application') returns None is the header doesn't exist
-        if not request_headers.get('X-Hubmap-Application'):
-            msg = f"Unable to update {property_key} due to missing X-Hubmap-Application header from request"
-            raise schema_errors.MissingApplicationHeaderException(msg)
-
-        # Use lowercase for comparing the application header value against the yaml
-        if request_headers.get('X-Hubmap-Application').lower() not in applications_allowed:
-            msg = f"Unable to update {property_key} due to invalid application header value: {request_headers.get('X-Hubmap-Application')}"
-            raise schema_errors.InvalidApplicationHeaderException(msg)
-
-
 """
 Get a list of entity types that can be used as derivation source in the schmea yaml
 
@@ -989,10 +872,10 @@ def validate_trigger_type(trigger_type):
 Parameters
 ----------
 validator_type : str
-    One of the validator types: before_create_validator, before_update_validator
+    One of the validator types: before_entity_create_validator
 """
 def validate_entity_level_validator_type(validator_type):
-    accepted_validator_types = ['before_entity_create_validator', 'before_entity_update_validator']
+    accepted_validator_types = ['before_entity_create_validator']
     separator = ', '
 
     if validator_type.lower() not in accepted_validator_types:
@@ -1007,7 +890,7 @@ def validate_entity_level_validator_type(validator_type):
 Parameters
 ----------
 validator_type : str
-    One of the validator types: before_create_validator, before_update_validator
+    One of the validator types: before_property_update_validators
 """
 def validate_property_level_validator_type(validator_type):
     accepted_validator_types = ['before_property_update_validators']

src/schema/schema_triggers.py

@@ -537,9 +537,6 @@ def update_file_descriptions(property_key, normalized_type, user_token, existing
             raise KeyError(f"Missing '{property_key}' key in 'generated_dict' during calling 'update_file_descriptions()' trigger method.")            
         existing_files_list = generated_dict[property_key]
 
-
-
-
     file_info_by_uuid_dict = {}
 
     for file_info in existing_files_list: