From 12835e9c520f7705fdc2e459660b29d67463cbf4 Mon Sep 17 00:00:00 2001 From: xiluan Date: Mon, 17 Apr 2023 10:08:07 -0700 Subject: [PATCH 1/7] added applied_to, and it's unit tests, refactoring rules parsing with unidentified entity --- moesifdjango/governance_rules.py | 45 ++++++++-- moesifdjango/middleware.py | 35 ++++---- moesifdjango/moesif_gov.py | 67 +++++++------- moesifdjango/tests/__init__.py | 0 moesifdjango/tests/test_apply_gov_rules.py | 100 +++++++++++++++++++++ setup.py | 2 +- 6 files changed, 188 insertions(+), 61 deletions(-) create mode 100644 moesifdjango/tests/__init__.py create mode 100644 moesifdjango/tests/test_apply_gov_rules.py diff --git a/moesifdjango/governance_rules.py b/moesifdjango/governance_rules.py index 133c838..2be2ef4 100644 --- a/moesifdjango/governance_rules.py +++ b/moesifdjango/governance_rules.py @@ -1,13 +1,27 @@ import json from moesifapi import APIException +from enum import Enum + + +class AppliedTo(Enum): + MATCHING = 'matching' + NOT_MATCHING = 'not_matching' + + +class RuleType(Enum): + USER = 'user' + COMPANY = 'company' + REGEX = 'regex' class GovernanceRulesCacher: def __init__(self, api_client): self.api_client = api_client - self.user_rules = {} - self.company_rules = {} + self.applied_to_identified_user_rules = {} + self.applied_to_identified_company_rules = {} + self.applied_to_unidentified_user_rules = {} + self.applied_to_unidentified_company_rules = {} self.regex_rules = {} def get_governance_rules_from_client(self, DEBUG): @@ -32,10 +46,16 @@ def generate_rules_caching(self, DEBUG): governance_rules = self.get_governance_rules_from_client(DEBUG) if not governance_rules: return None, None, None - rule_types = ['regex', 'user', 'company'] + rule_types = [RuleType.REGEX.value, RuleType.USER.value, RuleType.COMPANY.value] rules_type_mapping = {} for rule_type in rule_types: - rules_type_mapping[rule_type] = {} + if rule_type == RuleType.REGEX.value: + rules_type_mapping[rule_type] = {} + rules_type_mapping[rule_type][False] = {} + else: + rules_type_mapping[rule_type] = {} + rules_type_mapping[rule_type][True] = {} + rules_type_mapping[rule_type][False] = {} for rule in governance_rules: rule_id = rule['_id'] @@ -43,14 +63,21 @@ def generate_rules_caching(self, DEBUG): rule_type = rule['type'] if rule_type in rule_types: - rules_type_mapping[rule_type][rule_id] = rule + applied_to_unidentified = rule.get('applied_to_unidentified', False) + rules_type_mapping[rule_type][applied_to_unidentified][rule_id] = rule else: print('[moesif] Get parsed rule type {} is not valid'.format(rule['type'])) - self.user_rules = rules_type_mapping['user'] - self.company_rules = rules_type_mapping['company'] - self.regex_rules = rules_type_mapping['regex'] + self.applied_to_identified_user_rules = rules_type_mapping[RuleType.USER.value][False] + self.applied_to_unidentified_user_rules = rules_type_mapping[RuleType.USER.value][True] + self.applied_to_identified_company_rules = rules_type_mapping[RuleType.COMPANY.value][False] + self.applied_to_unidentified_company_rules = rules_type_mapping[RuleType.COMPANY.value][True] + # regex rule will not apply to unidentified or identified, currently, + # we will consider that the applied_to_unidentified always set to False + self.regex_rules = rules_type_mapping[RuleType.REGEX.value][False] except Exception as e: print("[moesif] Error when parsing rules response: ", e) - return self.user_rules, self.company_rules, self.regex_rules + return self.applied_to_identified_user_rules, self.applied_to_unidentified_user_rules, \ + self.applied_to_identified_company_rules, self.applied_to_unidentified_company_rules, \ + self.regex_rules diff --git a/moesifdjango/middleware.py b/moesifdjango/middleware.py index 9aaf950..9fb3c09 100644 --- a/moesifdjango/middleware.py +++ b/moesifdjango/middleware.py @@ -79,8 +79,9 @@ def __init__(self, get_response): self.entity_rules = self.gov_rule_helper.fetch_entity_rules_from_app_config(self.config, self.DEBUG) self.gov_rules_cacher = GovernanceRulesCacher(self.api_client) - self.user_governance_rules, self.company_governance_rules, self.regex_governance_rules \ - = self.gov_rules_cacher.generate_rules_caching(self.DEBUG) + self.identified_user_governance_rules, self.unidentified_user_governance_rules, \ + self.identified_company_governance_rules, self.unidentified_company_governance_rules, \ + self.regex_governance_rules = self.gov_rules_cacher.generate_rules_caching(self.DEBUG) self.sampling_percentage = 100 self.config_etag = None @@ -136,7 +137,9 @@ def event_listener(self, event): if response_rules_etag: if not self.rules_etag or self.rules_etag != response_rules_etag: self.rules_etag = response_rules_etag - self.user_governance_rules, self.company_governance_rules, self.regex_governance_rules \ + self.identified_user_governance_rules, self.unidentified_user_governance_rules,\ + self.identified_company_governance_rules, self.unidentified_company_governance_rules,\ + self.regex_governance_rules \ = self.gov_rules_cacher.generate_rules_caching(self.DEBUG) # Function to schedule send event job in async @@ -210,7 +213,8 @@ def __call__(self, request): self.middleware_settings) # Prepare Request Body - req_body, req_body_transfer_encoding = self.logger_helper.prepare_request_body(request, req_headers, self.LOG_BODY, + req_body, req_body_transfer_encoding = self.logger_helper.prepare_request_body(request, req_headers, + self.LOG_BODY, self.middleware_settings) # Fetch Ip Address ip_address = self.client_ip.get_client_ip(request) @@ -229,11 +233,12 @@ def __call__(self, request): rsp_headers = self.logger_helper.parse_response_headers(response, self.middleware_settings) # Prepare Response Body - rsp_body, rsp_body_transfer_encoding = self.logger_helper.prepare_response_body(response, rsp_headers, self.LOG_BODY, + rsp_body, rsp_body_transfer_encoding = self.logger_helper.prepare_response_body(response, rsp_headers, + self.LOG_BODY, self.middleware_settings) # Prepare Event Request Model - event_req = self.event_mapper.to_request(req_time, uri,request.method, self.api_version, ip_address, + event_req = self.event_mapper.to_request(req_time, uri, request.method, self.api_version, ip_address, req_headers, req_body, req_body_transfer_encoding) # Prepare Event Response Model @@ -258,15 +263,15 @@ def __call__(self, request): # Mask Event Model event_model = self.logger_helper.mask_event(event_model, self.middleware_settings, self.DEBUG) - updated_Response = self.gov_rule_helper.govern_request(event_model, - user_id, - company_id, - req_body_transfer_encoding, # could be json or base64 - self.entity_rules, - self.user_governance_rules, - self.company_governance_rules, - self.regex_governance_rules, - self.DEBUG) + updated_Response = self.gov_rule_helper.apply_governance_rules(event_model, + user_id, + company_id, + req_body_transfer_encoding, # could be json or base64 + self.entity_rules, + self.identified_user_governance_rules, + self.identified_company_governance_rules, + self.regex_governance_rules, + self.DEBUG) if updated_Response: response.content = self.parse_body.encode_response_body(updated_Response.block_response_body) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index 20f943e..bc3cda4 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -6,8 +6,9 @@ from .block_response_buffer import BlockResponseBufferList from .event_mapper import * from .governance_rule_response import GovernanceRuleBlockResponse +from .governance_rules import AppliedTo, RuleType -REVERSED_PRIORITY_RULES_ORDER = ['regex', 'company', 'user'] +REVERSED_PRIORITY_RULES_ORDER = [RuleType.REGEX.value, RuleType.COMPANY.value, RuleType.USER.value] class MoesifGovRuleHelper: @@ -176,20 +177,6 @@ def check_request_with_regex_match(self, gr_regex_configs_list, request_mapping_ # If regex conditions are not matched, return default sample rate (nil) and do not block request (false) return False - def check_event_matched_with_governance_rules(self, gr_regex_configs, request_mapping_for_regex_config, - ready_for_body_request): - """ - check if the request config mapping governance rule regex conditions - :param gr_regex_configs: - :param request_mapping_for_regex_config: - :param ready_for_body_request: - :return: - """ - matched = self.check_request_with_regex_match(gr_regex_configs, - request_mapping_for_regex_config, - ready_for_body_request) - - return matched @classmethod def get_req_content_type(cls, request): @@ -338,19 +325,12 @@ def block_request_based_on_entity_governance_rule(self, entity_id) continue - gr_regex_configs = {} - if "regex_config" in governance_rule and governance_rule["regex_config"]: - gr_regex_configs = governance_rule["regex_config"] + should_block = self.check_event_should_blocked_by_rule(governance_rule, request_mapping_for_regex_config, ready_for_body_request) - matched = not gr_regex_configs or self.check_event_matched_with_governance_rules( - gr_regex_configs, - request_mapping_for_regex_config, - ready_for_body_request) - - if not matched: + if not should_block: if DEBUG: print( - "[moesif] Skipped blocking request as governance rule {} regex conditions does not match".format( + "[moesif] Skipped blocking request because it's not satisfied with governance rule {}".format( rule_id)) continue @@ -366,6 +346,23 @@ def block_request_based_on_entity_governance_rule(self, return response_buffer + def check_event_should_blocked_by_rule(self, governance_rule, + request_mapping_for_regex_config, + ready_for_body_request): + applied_to = governance_rule.get('applied_to', AppliedTo.MATCHING.value) + + gr_regex_configs = {} + if "regex_config" in governance_rule and governance_rule["regex_config"]: + gr_regex_configs = governance_rule["regex_config"] + + matched = self.check_request_with_regex_match( + gr_regex_configs, + request_mapping_for_regex_config, + ready_for_body_request) + + return (matched and applied_to == AppliedTo.MATCHING.value) \ + or (not matched and applied_to == AppliedTo.NOT_MATCHING.value) + def get_rules_id_if_governance_rule_matched(self, regex_governance_rules, event, ready_for_body_request): """ find the regex governance rules what matched with request, and return the governance rules id @@ -518,17 +515,15 @@ def check_if_request_blocked(cls, response_buffers): return True return False - def govern_request(self, - event, - user_id, company_id, - req_body_transfer_encoding, - entity_rules, - user_governance_rules, - company_governance_rules, - regex_governance_rules, - DEBUG): - user_id_entity = user_id - company_id_entity = company_id + def apply_governance_rules(self, + event, + user_id_entity, company_id_entity, + req_body_transfer_encoding, + entity_rules, + user_governance_rules, + company_governance_rules, + regex_governance_rules, + DEBUG): ready_for_body_request = self.ok_request_body_regex_rule(event.request, req_body_transfer_encoding) request_mapping_for_regex_config = self.prepare_request_config_based_on_regex_config(event, diff --git a/moesifdjango/tests/__init__.py b/moesifdjango/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/moesifdjango/tests/test_apply_gov_rules.py b/moesifdjango/tests/test_apply_gov_rules.py new file mode 100644 index 0000000..0a302c0 --- /dev/null +++ b/moesifdjango/tests/test_apply_gov_rules.py @@ -0,0 +1,100 @@ +import unittest +from ..moesif_gov import * + + +class GovRulesTestCase(unittest.TestCase): + def setUp(self): + self.request = EventRequestModel( + body="LS0tLS0tV2ViS2l0Rm9ybUJvdW5kYXJ5blNmSzhhVUUwRUYxSUdudw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJjc3JmbWlkZGxld2FyZXRva2VuIg0KDQp1WEs3Qm5sMTFOUHJKNGFlVWthbGU2eDZ0TXdPQnk1SWlZRDQxRzR5cEhmZFhBRTZRZGNMNnFXV0ZsTTdXMjJIDQotLS0tLS1XZWJLaXRGb3JtQm91bmRhcnluU2ZLOGFVRTBFRjFJR253DQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InVzZXJuYW1lIg0KDQp1c2VyIDU5DQotLS0tLS1XZWJLaXRGb3JtQm91bmRhcnluU2ZLOGFVRTBFRjFJR253DQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImVtYWlsIg0KDQoNCi0tLS0tLVdlYktpdEZvcm1Cb3VuZGFyeW5TZks4YVVFMEVGMUlHbnctLQ0K", + headers={'CONTENT-LENGTH': '408', + 'CONTENT-TYPE': 'multipart/form-data; boundary=----WebKitFormBoundarynSfK8aUE0EF1IGnw', + 'HOST': '127.0.0.1:8000', 'CONNECTION': 'keep-alive', 'CACHE-CONTROL': 'max-age=0', + 'SEC-CH-UA': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"', + 'SEC-CH-UA-MOBILE': '?0', 'SEC-CH-UA-PLATFORM': '"macOS"', 'UPGRADE-INSECURE-REQUESTS': '1', + 'ORIGIN': 'http://127.0.0.1:8000', + 'USER-AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'ACCEPT': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'SEC-FETCH-SITE': 'same-origin', 'SEC-FETCH-MODE': 'navigate', 'SEC-FETCH-USER': '?1', + 'SEC-FETCH-DEST': 'document', 'REFERER': 'http://127.0.0.1:8000/users/', + 'ACCEPT-ENCODING': 'gzip, deflate, br', 'ACCEPT-LANGUAGE': 'en-US,en;q=0.9', + 'COOKIE': 'csrftoken=Ys3hZTsC7OL7HacTyYvgAsaRbhXoPF7GMtWepcb9vIbTVGGLuRxGsMzHnQdHa94F', + 'X-MOESIF-TRANSACTION-ID': 'd1965385-708f-485d-8370-b02ab334f6ef'}, + ip_address='127.0.0.1', + time='2023-04-12T20:33:41.047', + transfer_encoding='base64', + uri='http://127.0.0.1:8000/users/', + verb='POST' + ) + self.gov_helper = MoesifGovRuleHelper() + self.gov_rule = { + "_id": "642f4fcea6ca1c38705d660d", + "created_at": "2023-04-06T23:03:42.130", + "org_id": "125:14", + "app_id": "768:64", + "name": "Gov new schema test", + "block": True, + "type": "user", + "applied_to": "matching", + "applied_to_unidentified": True, + "variables": [ + { + "name": "0", + "path": "user_id" + } + ], + "regex_config": [ + { + "conditions": [ + { + "path": "request.route", + "value": ".*user.*" + } + ] + }, + { + "conditions": [ + { + "path": "request.route", + "value": ".*789.*" + } + ] + } + ], + "response": { + "status": 305, + "headers": {}, + "body": { + "msg": "Blocked by Gov Rule on [DEV 125:14-768:64]", + "user_id": "{{0}}" + } + } + } + + self.request_mapping_for_regex_config = \ + { + 'request.verb': 'GET', + 'request.route': 'http://127.0.0.1:8000/users/', + 'request.ip_address': '127.0.0.1', + 'response.status': 200 + } + + self.ready_for_body_request = True + + ob = MoesifGovRuleHelper() + + def test_something(self): + print(self.request) + self.assertEqual(True, True) # add assertion here + + def test_check_event_should_blocked_by_rule_should_block(self): + block = self.gov_helper.check_event_should_blocked_by_rule(self.gov_rule, self.request_mapping_for_regex_config, self.ready_for_body_request) + self.assertEqual(block, True) + + def test_check_event_should_blocked_by_rule_should_not_block(self): + self.request_mapping_for_regex_config['request.route'] = 'http://127.0.0.1:8000/groups/' + block = self.gov_helper.check_event_should_blocked_by_rule(self.gov_rule, self.request_mapping_for_regex_config, self.ready_for_body_request) + self.assertEqual(block, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup.py b/setup.py index 49a5a45..3542bb1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='2.3.1', + version='2.4.0', description='Moesif Middleware for Python Django', long_description=long_description, From 0b10349faaeefe534f9e6fad1090c98a1c281a65 Mon Sep 17 00:00:00 2001 From: xiluan Date: Tue, 18 Apr 2023 13:55:39 -0700 Subject: [PATCH 2/7] add applied_to_unidentified entity --- moesifdjango/block_response_buffer.py | 5 +- moesifdjango/middleware.py | 2 + moesifdjango/moesif_gov.py | 217 +++++++++++++++++--------- 3 files changed, 151 insertions(+), 73 deletions(-) diff --git a/moesifdjango/block_response_buffer.py b/moesifdjango/block_response_buffer.py index aacee9c..c947bf1 100644 --- a/moesifdjango/block_response_buffer.py +++ b/moesifdjango/block_response_buffer.py @@ -1,10 +1,11 @@ +from .governance_rules import RuleType from .governance_rule_response import GovernanceRuleBlockResponse class BlockResponseBufferList: - def __init__(self): + def __init__(self, rule_type=RuleType.REGEX.value): self.responses = [] - self.rule_type = None + self.rule_type = rule_type self.blocked = False def update(self, block, updated_gr_status, updated_gr_headers, updated_gr_body): diff --git a/moesifdjango/middleware.py b/moesifdjango/middleware.py index 9fb3c09..3f68091 100644 --- a/moesifdjango/middleware.py +++ b/moesifdjango/middleware.py @@ -269,7 +269,9 @@ def __call__(self, request): req_body_transfer_encoding, # could be json or base64 self.entity_rules, self.identified_user_governance_rules, + self.unidentified_user_governance_rules, self.identified_company_governance_rules, + self.unidentified_company_governance_rules, self.regex_governance_rules, self.DEBUG) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index bc3cda4..c496784 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -11,6 +11,22 @@ REVERSED_PRIORITY_RULES_ORDER = [RuleType.REGEX.value, RuleType.COMPANY.value, RuleType.USER.value] +def merge_block_response(block_response_buffer_list_one: BlockResponseBufferList, + block_response_buffer_list_two: BlockResponseBufferList): + if block_response_buffer_list_one.rule_type != block_response_buffer_list_two.rule_type: + print('[moesif] Error when merging block response buffer list, [] and [] rule_type are not matching' + .format(block_response_buffer_list_one.rule_type, block_response_buffer_list_two.rule_type)) + return None + merged_block_response_buffer_list = block_response_buffer_list_one + merged_block_response_buffer_list.blocked = block_response_buffer_list_one.blocked \ + or block_response_buffer_list_two.blocked + + for response in block_response_buffer_list_two.responses: + merged_block_response_buffer_list.responses.append(response) + + return merged_block_response_buffer_list + + class MoesifGovRuleHelper: def __init__(self): pass @@ -59,7 +75,16 @@ def get_entity_governance_rule_and_check_block(cls, entity_rules, governance_rul return block_entity_rules @classmethod - def fetch_governance_rule_response_details(cls, governance_rule): + def fetch_governance_rule_response_details(cls, governance_rule, DEBUG): + if 'response' not in governance_rule \ + or 'status' not in governance_rule['response'] \ + or 'headers' not in governance_rule['response']: + if DEBUG: + print( + '[moesif] Skipped blocking request as response is not set for the governance rule {} with regex config'.format( + governance_rule)) + return None, None, None + # Response status status = governance_rule['response']['status'] # Response headers @@ -235,14 +260,17 @@ def check_if_condition_for_request_body_field(cls, condition): return True - def get_updated_response_with_matched_rules(self, governance_rule, rule_and_values): + def get_updated_response_with_matched_entity_rules(self, governance_rule, rule_and_values, DEBUG): """ - get updated response if the governance is blocked checked and matched with request + get updated response if the entity governance rule is blocked checked, and event matched with request + utilized by both identified and unidentified :param governance_rule: :param rule_and_values: :return: """ - gr_status, gr_header, gr_body = self.fetch_governance_rule_response_details(governance_rule) + gr_status, gr_header, gr_body = self.fetch_governance_rule_response_details(governance_rule, DEBUG) + if not gr_status: + return None, None, None # Updated governance rule headers updated_gr_headers = {} @@ -270,26 +298,28 @@ def get_updated_response_with_matched_rules(self, governance_rule, rule_and_valu return gr_status, updated_gr_headers, updated_gr_body - def block_request_based_on_entity_governance_rule(self, - request_mapping_for_regex_config, - ready_for_body_request, - governance_rules, - entity_rules, - rule_entity_type, - entity_id, - DEBUG): + def block_request_based_on_entity_governance_rule_identified(self, + request_mapping_for_regex_config, + ready_for_body_request, + governance_rules, + entity_rules, + rule_entity_type, + entity_id, + rule_type, + DEBUG): """ Check if need to block request based on the governance rule of the entity associated with the request + :param rule_type: the type of BlockResponseBufferList (user, company or regex) :param request_mapping_for_regex_config: :param ready_for_body_request: :param governance_rules: :param entity_rules: - :param rule_entity_type: - :param entity_id: + :param rule_entity_type: the type of rules from config (user_rules, company_rules or regex_rules) + :param entity_id: user_id or company_id from event :param DEBUG: :return: object of updated response status, headers and body, if criteria is matched and block is true, otherwise return None """ - response_buffer = BlockResponseBufferList() + response_buffer = BlockResponseBufferList(rule_type) entity_id_rules_mapping = None @@ -335,8 +365,8 @@ def block_request_based_on_entity_governance_rule(self, continue # update response status, headers and body if one block rule matched - updated_gr_status, updated_gr_headers, updated_gr_body = self.get_updated_response_with_matched_rules( - governance_rule, rule_and_values) + updated_gr_status, updated_gr_headers, updated_gr_body = self.get_updated_response_with_matched_entity_rules( + governance_rule, rule_and_values, DEBUG) block = governance_rule.get('block', False) response_buffer.update(block, updated_gr_status, updated_gr_headers, updated_gr_body) @@ -363,45 +393,47 @@ def check_event_should_blocked_by_rule(self, governance_rule, return (matched and applied_to == AppliedTo.MATCHING.value) \ or (not matched and applied_to == AppliedTo.NOT_MATCHING.value) - def get_rules_id_if_governance_rule_matched(self, regex_governance_rules, event, ready_for_body_request): + def get_rules_id_if_governance_rule_matched(self, governance_rules, request_mapping_for_regex_config, + ready_for_body_request): """ find the regex governance rules what matched with request, and return the governance rules id - :param regex_governance_rules: - :param event: + :param request_mapping_for_regex_config: request config generated from event, for regex config matching in gov rules + :param governance_rules: :param ready_for_body_request: - :return: + :return: list of gov rule ids that event matched """ matched_rules_id = [] - request_config_mapping = self.prepare_request_config_based_on_regex_config(event, ready_for_body_request) - for id, rule in regex_governance_rules.items(): + for id, rule in governance_rules.items(): if 'regex_config' not in rule or not rule['regex_config']: continue regex_configs = rule['regex_config'] - matched = self.check_request_with_regex_match(regex_configs, request_config_mapping, + matched = self.check_request_with_regex_match(regex_configs, request_mapping_for_regex_config, ready_for_body_request) if matched: - try: - matched_rules_id.append(rule['_id']) - except KeyError as ke: - print('[moesif] Error when fetching matched regex governance rule ', ke) - + matched_rules_id.append(id) return matched_rules_id - def block_request_based_on_governance_rule_regex_config(self, event, ready_for_body_request, regex_governance_rules, - DEBUG): + def block_request_based_on_regex_or_unidentified_entity_governance_rule(self, + request_mapping_for_regex_config, + ready_for_body_request, + regex_governance_rules, + rule_type, + DEBUG): """ Check if need to block request based on the governance rule regex config associated with the request - :param event: + :param rule_type: + :param request_mapping_for_regex_config: :param ready_for_body_request: :param regex_governance_rules: :param DEBUG: :return: """ - response_buffer = BlockResponseBufferList() - matched_rules_id = self.get_rules_id_if_governance_rule_matched(regex_governance_rules, event, + response_buffer = BlockResponseBufferList(rule_type) + matched_rules_id = self.get_rules_id_if_governance_rule_matched(regex_governance_rules, + request_mapping_for_regex_config, ready_for_body_request) if not matched_rules_id: @@ -414,18 +446,10 @@ def block_request_based_on_governance_rule_regex_config(self, event, ready_for_b if DEBUG: print( '[moesif] Skipped blocking request as rule {} is not found'.format(rule_id)) - continue - - if 'response' not in governance_rule \ - or 'status' not in governance_rule['response'] \ - or 'headers' not in governance_rule['response']: - if DEBUG: - print( - '[moesif] Skipped blocking request as response is not set for the governance rule with regex config') continue block = governance_rule.get('block', False) - gr_status, gr_header, gr_body = self.fetch_governance_rule_response_details(governance_rule) + gr_status, gr_header, gr_body = self.fetch_governance_rule_response_details(governance_rule, DEBUG) response_buffer.update(block, gr_status, gr_header, gr_body) if DEBUG: @@ -520,8 +544,10 @@ def apply_governance_rules(self, user_id_entity, company_id_entity, req_body_transfer_encoding, entity_rules, - user_governance_rules, - company_governance_rules, + identified_user_governance_rules, + unidentified_user_governance_rules, + identified_company_governance_rules, + unidentified_company_governance_rules, regex_governance_rules, DEBUG): @@ -529,56 +555,105 @@ def apply_governance_rules(self, request_mapping_for_regex_config = self.prepare_request_config_based_on_regex_config(event, ready_for_body_request) - response_buffers = {} + response_buffers = { + RuleType.REGEX.value: BlockResponseBufferList(RuleType.REGEX.value), + RuleType.COMPANY.value: BlockResponseBufferList(RuleType.COMPANY.value), + RuleType.USER.value: BlockResponseBufferList(RuleType.USER.value) + } if regex_governance_rules: - regex_response_buffer = self.block_request_based_on_governance_rule_regex_config(event, - ready_for_body_request, - regex_governance_rules, - DEBUG) + regex_response_buffer = self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + request_mapping_for_regex_config, + ready_for_body_request, + regex_governance_rules, + RuleType.REGEX.value, + DEBUG) if not regex_response_buffer.blocked: if DEBUG: print('[moesif] No matching with the request from regex rules') - response_buffers['regex'] = regex_response_buffer + response_buffers[RuleType.REGEX.value] = regex_response_buffer else: if DEBUG: print('[moesif] No regex rules') - if company_id_entity and company_governance_rules: - company_response_buffer = self.block_request_based_on_entity_governance_rule( + if unidentified_company_governance_rules: + unidentified_company_response_buffer = \ + self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + request_mapping_for_regex_config, + ready_for_body_request, + unidentified_company_governance_rules, + RuleType.COMPANY.value, + DEBUG + ) + if not unidentified_company_response_buffer.blocked: + if DEBUG: + print('[moesif] No matching with the request from unidentified company rules') + else: + response_buffers[RuleType.COMPANY.value] = unidentified_company_response_buffer + else: + if DEBUG: + print('[moesif] no unidentified company governance rules') + + if company_id_entity and identified_company_governance_rules: + company_response_buffer = self.block_request_based_on_entity_governance_rule_identified( request_mapping_for_regex_config, ready_for_body_request, - company_governance_rules, + identified_company_governance_rules, entity_rules, 'company_rules', company_id_entity, + RuleType.COMPANY.value, DEBUG) if not company_response_buffer.blocked: if DEBUG: print('[moesif] No blocking from company: ', company_id_entity) - - response_buffers['company'] = company_response_buffer + else: + response_buffers[RuleType.COMPANY.value] = merge_block_response( + response_buffers.get(RuleType.COMPANY.value, BlockResponseBufferList()), + company_response_buffer) else: if DEBUG: - print('[moesif] company_id is not valid or no governance rules for the company') - - if user_id_entity and user_governance_rules: - user_response_buffer = self.block_request_based_on_entity_governance_rule(request_mapping_for_regex_config, - ready_for_body_request, - user_governance_rules, - entity_rules, - 'user_rules', - user_id_entity, - DEBUG) - - if not user_response_buffer.blocked: + print('[moesif] company_id is not valid or no identified company governance rules') + + if unidentified_user_governance_rules: + unidentified_company_response_buffer = \ + self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + request_mapping_for_regex_config, + ready_for_body_request, + unidentified_user_governance_rules, + RuleType.USER.value, + DEBUG + ) + if not unidentified_company_response_buffer.blocked: if DEBUG: - print('[moesif] No blocking from user: ', user_id_entity) + print('[moesif] No matching with the request from unidentified user rules') + else: + response_buffers[RuleType.USER.value] = unidentified_company_response_buffer + else: + if DEBUG: + print('[moesif] no unidentified user governance rules') + + if user_id_entity and identified_user_governance_rules: + identified_user_response_buffer = self.block_request_based_on_entity_governance_rule_identified( + request_mapping_for_regex_config, + ready_for_body_request, + identified_user_governance_rules, + entity_rules, + 'user_rules', + user_id_entity, + RuleType.USER.value, + DEBUG) - response_buffers['user'] = user_response_buffer + if not identified_user_response_buffer.blocked: + if DEBUG: + print('[moesif] No blocking from user: ', user_id_entity) + else: + response_buffers[RuleType.USER.value] = merge_block_response( + response_buffers.get(RuleType.USER.value, BlockResponseBufferList()), + identified_user_response_buffer) else: if DEBUG: - print('[moesif] user_id is not valid or no governance rules for the user') + print('[moesif] user_id is not valid or no identified user governance rules') blocking_response = self.generate_blocking_response(response_buffers) From fc4afab64b99ef4d900e1db0853932df7235f6ec Mon Sep 17 00:00:00 2001 From: xiluan Date: Wed, 19 Apr 2023 15:59:00 -0700 Subject: [PATCH 3/7] added unit tests and fix some response issue --- moesifdjango/moesif_gov.py | 86 +++++---- moesifdjango/tests/test_apply_gov_rules.py | 196 +++++++++++++++++++-- 2 files changed, 228 insertions(+), 54 deletions(-) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index c496784..16bc2e4 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -106,7 +106,7 @@ def transform_values(self, data, rule_values): if isinstance(data, str): max_index = max(rule_values.keys()) - rule_values_list = [rule_values[key] if key in rule_values.keys() else None for key in + rule_values_list = [rule_values[key] if key in rule_values.keys() else 'UNKNOWN' for key in range(max_index + 1)] try: @@ -285,8 +285,9 @@ def get_updated_response_with_matched_entity_rules(self, governance_rule, rule_a updated_gr_body = gr_body.copy() updated_gr_values = {} - if 'values' in rule_and_values and rule_and_values['values']: - rule_values = rule_and_values['values'] + rule_id = governance_rule.get('_id') + if rule_id in rule_and_values: + rule_values = rule_and_values.get(rule_id) for k, v in rule_values.items(): try: updated_gr_values[int(k)] = v @@ -302,10 +303,9 @@ def block_request_based_on_entity_governance_rule_identified(self, request_mapping_for_regex_config, ready_for_body_request, governance_rules, + rule_type, entity_rules, - rule_entity_type, entity_id, - rule_type, DEBUG): """ Check if need to block request based on the governance rule of the entity associated with the request @@ -321,31 +321,7 @@ def block_request_based_on_entity_governance_rule_identified(self, """ response_buffer = BlockResponseBufferList(rule_type) - entity_id_rules_mapping = None - - try: - entity_id_rules_mapping = entity_rules[rule_entity_type][entity_id] - except KeyError as ke: - print( - '[moesif] Skipped blocking request since no governance rules in type of {} with the entity Id - {}: {}'.format( - rule_entity_type, entity_id, ke)) - except Exception as e: - print('[moesif] Skipped blocking request, Error when fetching entity rule with entity {}, {}'.format( - entity_id, e)) - - if not entity_id_rules_mapping: - return response_buffer - - for rule_and_values in entity_id_rules_mapping: - - try: - rule_id = rule_and_values['rules'] # rule_id is represented as "rules" in the config schema - except KeyError as ke: - print( - '[moesif] Skipped a rule in type of {} since the [rule_id] is not found with entity - {}, {}'.format( - rule_entity_type, entity_id, ke)) - continue - + for rule_id, _ in entity_rules.items(): governance_rule = governance_rules.get(rule_id, None) if not governance_rule or 'response' not in governance_rule or 'status' not in governance_rule['response']: @@ -366,7 +342,7 @@ def block_request_based_on_entity_governance_rule_identified(self, # update response status, headers and body if one block rule matched updated_gr_status, updated_gr_headers, updated_gr_body = self.get_updated_response_with_matched_entity_rules( - governance_rule, rule_and_values, DEBUG) + governance_rule, entity_rules, DEBUG) block = governance_rule.get('block', False) response_buffer.update(block, updated_gr_status, updated_gr_headers, updated_gr_body) @@ -404,12 +380,9 @@ def get_rules_id_if_governance_rule_matched(self, governance_rules, request_mapp """ matched_rules_id = [] for id, rule in governance_rules.items(): - if 'regex_config' not in rule or not rule['regex_config']: - continue - regex_configs = rule['regex_config'] - matched = self.check_request_with_regex_match(regex_configs, request_mapping_for_regex_config, - ready_for_body_request) + matched = self.check_event_should_blocked_by_rule(rule, request_mapping_for_regex_config, + ready_for_body_request) if matched: matched_rules_id.append(id) @@ -420,10 +393,12 @@ def block_request_based_on_regex_or_unidentified_entity_governance_rule(self, ready_for_body_request, regex_governance_rules, rule_type, + entity_rules, DEBUG): """ Check if need to block request based on the governance rule regex config associated with the request - :param rule_type: + :param entity_rules: specific entity rules config {"rule_id": "values for merge tag"}, if merge tag is defined + :param rule_type: (user, company or regex) :param request_mapping_for_regex_config: :param ready_for_body_request: :param regex_governance_rules: @@ -449,7 +424,8 @@ def block_request_based_on_regex_or_unidentified_entity_governance_rule(self, continue block = governance_rule.get('block', False) - gr_status, gr_header, gr_body = self.fetch_governance_rule_response_details(governance_rule, DEBUG) + gr_status, gr_header, gr_body = \ + self.get_updated_response_with_matched_entity_rules(governance_rule, entity_rules, DEBUG) response_buffer.update(block, gr_status, gr_header, gr_body) if DEBUG: @@ -539,6 +515,24 @@ def check_if_request_blocked(cls, response_buffers): return True return False + def get_entity_rule_mapping_from_config(self, entity_rules, rule_type_in_config, entity_id): + rule_merge_tag_values_mapping = {} + try: + if entity_id: + rules_mapping_from_config = entity_rules[rule_type_in_config][entity_id] + for rule_values in rules_mapping_from_config: + if 'values' in rule_values: + rule_id = rule_values['rules'] + values = rule_values['values'] + + if rule_id not in rule_merge_tag_values_mapping: + rule_merge_tag_values_mapping[rule_id] = {} + rule_merge_tag_values_mapping[rule_id].update(values) + except Exception as e: + print('[moesif] Skipped blocking request, Error when fetching entity rule with entity {}, {}'.format( + entity_id, e)) + return rule_merge_tag_values_mapping + def apply_governance_rules(self, event, user_id_entity, company_id_entity, @@ -555,6 +549,9 @@ def apply_governance_rules(self, request_mapping_for_regex_config = self.prepare_request_config_based_on_regex_config(event, ready_for_body_request) + user_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'user_rules', user_id_entity) + company_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'company_rules', company_id_entity) + response_buffers = { RuleType.REGEX.value: BlockResponseBufferList(RuleType.REGEX.value), RuleType.COMPANY.value: BlockResponseBufferList(RuleType.COMPANY.value), @@ -566,6 +563,7 @@ def apply_governance_rules(self, ready_for_body_request, regex_governance_rules, RuleType.REGEX.value, + {}, DEBUG) if not regex_response_buffer.blocked: if DEBUG: @@ -583,6 +581,7 @@ def apply_governance_rules(self, ready_for_body_request, unidentified_company_governance_rules, RuleType.COMPANY.value, + company_rules_mapping_from_config, DEBUG ) if not unidentified_company_response_buffer.blocked: @@ -599,10 +598,9 @@ def apply_governance_rules(self, request_mapping_for_regex_config, ready_for_body_request, identified_company_governance_rules, - entity_rules, - 'company_rules', - company_id_entity, RuleType.COMPANY.value, + company_rules_mapping_from_config, + company_id_entity, DEBUG) if not company_response_buffer.blocked: if DEBUG: @@ -622,6 +620,7 @@ def apply_governance_rules(self, ready_for_body_request, unidentified_user_governance_rules, RuleType.USER.value, + user_rules_mapping_from_config, DEBUG ) if not unidentified_company_response_buffer.blocked: @@ -638,10 +637,9 @@ def apply_governance_rules(self, request_mapping_for_regex_config, ready_for_body_request, identified_user_governance_rules, - entity_rules, - 'user_rules', - user_id_entity, RuleType.USER.value, + user_rules_mapping_from_config, + user_id_entity, DEBUG) if not identified_user_response_buffer.blocked: diff --git a/moesifdjango/tests/test_apply_gov_rules.py b/moesifdjango/tests/test_apply_gov_rules.py index 0a302c0..dae7484 100644 --- a/moesifdjango/tests/test_apply_gov_rules.py +++ b/moesifdjango/tests/test_apply_gov_rules.py @@ -26,7 +26,7 @@ def setUp(self): verb='POST' ) self.gov_helper = MoesifGovRuleHelper() - self.gov_rule = { + self.user_gov_rule = { "_id": "642f4fcea6ca1c38705d660d", "created_at": "2023-04-06T23:03:42.130", "org_id": "125:14", @@ -35,7 +35,7 @@ def setUp(self): "block": True, "type": "user", "applied_to": "matching", - "applied_to_unidentified": True, + "applied_to_unidentified": False, "variables": [ { "name": "0", @@ -64,37 +64,213 @@ def setUp(self): "status": 305, "headers": {}, "body": { - "msg": "Blocked by Gov Rule on [DEV 125:14-768:64]", + "msg": "Blocked by Gov Rule", "user_id": "{{0}}" } } } + self.new_schema_regex_gov_rule = { + "_id": "6439b3f5d5762a04c0623d8b", + "regex_config": [ + { + "conditions": [ + { + "path": "request.route", + "value": ".*555.*" + } + ] + } + ], + "org_id": "125:14", + "response": { + "headers": {}, + "body": "eyJlcnJvciI6ImJsb2NrZWQgYnkgcmVnZXggcnVsZSBuZXcgc2NoZW1hIFtERVYgMTI1OjE0LTc2ODo2NF0ifQ==", + "status": 303 + }, + "name": "Allow list test regex rule", + "applied_to_unidentified": False, + "created_at": "2023-04-14T20:13:41.478", + "applied_to": "matching", + "block": True, + "state": 1, + "type": "regex", + "app_id": "768:64" + } + self.request_mapping_for_regex_config = \ { 'request.verb': 'GET', 'request.route': 'http://127.0.0.1:8000/users/', 'request.ip_address': '127.0.0.1', 'response.status': 200 - } + } + + self.request_mapping_for_regex_config2 = \ + { + 'request.verb': 'GET', + 'request.route': 'http://127.0.0.1:8000/555/', + 'request.ip_address': '127.0.0.1', + 'response.status': 200 + } self.ready_for_body_request = True - ob = MoesifGovRuleHelper() + self.request = EventRequestModel( + body="LS0tLS0tV2ViS2l0Rm9ybUJvdW5kYXJ5blNmSzhhVUUwRUYxSUdudw0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJjc3JmbWlkZGxld2FyZXRva2VuIg0KDQp1WEs3Qm5sMTFOUHJKNGFlVWthbGU2eDZ0TXdPQnk1SWlZRDQxRzR5cEhmZFhBRTZRZGNMNnFXV0ZsTTdXMjJIDQotLS0tLS1XZWJLaXRGb3JtQm91bmRhcnluU2ZLOGFVRTBFRjFJR253DQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9InVzZXJuYW1lIg0KDQp1c2VyIDU5DQotLS0tLS1XZWJLaXRGb3JtQm91bmRhcnluU2ZLOGFVRTBFRjFJR253DQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImVtYWlsIg0KDQoNCi0tLS0tLVdlYktpdEZvcm1Cb3VuZGFyeW5TZks4YVVFMEVGMUlHbnctLQ0K", + headers={'CONTENT-LENGTH': '408', + 'CONTENT-TYPE': 'multipart/form-data; boundary=----WebKitFormBoundarynSfK8aUE0EF1IGnw', + 'HOST': '127.0.0.1:8000', 'CONNECTION': 'keep-alive', 'CACHE-CONTROL': 'max-age=0', + 'SEC-CH-UA': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"', + 'SEC-CH-UA-MOBILE': '?0', 'SEC-CH-UA-PLATFORM': '"macOS"', 'UPGRADE-INSECURE-REQUESTS': '1', + 'ORIGIN': 'http://127.0.0.1:8000', + 'USER-AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36', + 'ACCEPT': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'SEC-FETCH-SITE': 'same-origin', 'SEC-FETCH-MODE': 'navigate', 'SEC-FETCH-USER': '?1', + 'SEC-FETCH-DEST': 'document', 'REFERER': 'http://127.0.0.1:8000/users/', + 'ACCEPT-ENCODING': 'gzip, deflate, br', 'ACCEPT-LANGUAGE': 'en-US,en;q=0.9', + 'COOKIE': 'csrftoken=Ys3hZTsC7OL7HacTyYvgAsaRbhXoPF7GMtWepcb9vIbTVGGLuRxGsMzHnQdHa94F', + 'X-MOESIF-TRANSACTION-ID': 'd1965385-708f-485d-8370-b02ab334f6ef'}, + ip_address='127.0.0.1', + time='2023-04-12T20:33:41.047', + transfer_encoding='base64', + uri='http://127.0.0.1:8000/users/', + verb='POST' + ) + self.response = EventResponseModel( + body='CgoKCjwhRE9DVFlQRSBodG1sPgo8aHRtbD4KICA8aGVhZD4KICAgIAoKICAgICAgCiAgICAgICAgPG1ldGEgaHR0cC1lcXVpdj0iQ29udGVudC1UeXBlIiBjb250ZW50PSJ0ZXh0L2h0bWw7IGNoYXJzZXQ9dXRmLTgiLz4KICAgICAgICA8bWV0YSBuYW1lPSJyb2JvdHMiIGNvbnRlbnQ9Ik5PTkUsTk9BUkNISVZFIiAvPgogICAgICAKCiAgICAgIDx0aXRsZT5Vc2VyIExpc3Qg4oCTIERqYW5nbyBSRVNUIGZyYW1ld29yazwvdGl0bGU+CgogICAgICAKICAgICAgICAKICAgICAgICAgIDxsaW5rIHJlbD0ic3R5bGVzaGVldCIgdHlwZT0idGV4dC9jc3MiIGhyZWY9Ii9zdGF0aWMvcmVzdF9mcmFtZXdvcmsvY3NzL2Jvb3RzdHJhcC5taW4uY3NzIi8+CiAgICAgICAgICA8bGluayByZWw9InN0eWxlc2hlZXQiIHR5cGU9InRleHQvY3NzIiBocmVmPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2Nzcy9ib290c3RyYXAtdHdlYWtzLmNzcyIvPgogICAgICAgIAoKICAgICAgICA8bGluayByZWw9InN0eWxlc2hlZXQiIHR5cGU9InRleHQvY3NzIiBocmVmPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2Nzcy9wcmV0dGlmeS5jc3MiLz4KICAgICAgICA8bGluayByZWw9InN0eWxlc2hlZXQiIHR5cGU9InRleHQvY3NzIiBocmVmPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2Nzcy9kZWZhdWx0LmNzcyIvPgogICAgICAgIAogICAgICAKCiAgICAKICA8L2hlYWQ+CgogIAogIDxib2R5IGNsYXNzPSIiPgoKICAgIDxkaXYgY2xhc3M9IndyYXBwZXIiPgogICAgICAKICAgICAgICA8ZGl2IGNsYXNzPSJuYXZiYXIgbmF2YmFyLXN0YXRpYy10b3AgbmF2YmFyLWludmVyc2UiCiAgICAgICAgICAgICByb2xlPSJuYXZpZ2F0aW9uIiBhcmlhLWxhYmVsPSJuYXZiYXIiPgogICAgICAgICAgPGRpdiBjbGFzcz0iY29udGFpbmVyIj4KICAgICAgICAgICAgPHNwYW4+CiAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICA8YSBjbGFzcz0nbmF2YmFyLWJyYW5kJyByZWw9Im5vZm9sbG93IiBocmVmPSdodHRwczovL3d3dy5kamFuZ28tcmVzdC1mcmFtZXdvcmsub3JnLyc+CiAgICAgICAgICAgICAgICAgICAgRGphbmdvIFJFU1QgZnJhbWV3b3JrCiAgICAgICAgICAgICAgICA8L2E+CiAgICAgICAgICAgICAgCiAgICAgICAgICAgIDwvc3Bhbj4KICAgICAgICAgICAgPHVsIGNsYXNzPSJuYXYgbmF2YmFyLW5hdiBwdWxsLXJpZ2h0Ij4KICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICA8bGk+PGEgaHJlZj0nL2FwaS1hdXRoL2xvZ2luLz9uZXh0PS91c2Vycy8nPkxvZyBpbjwvYT48L2xpPgogICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgCiAgICAgICAgICAgIDwvdWw+CiAgICAgICAgICA8L2Rpdj4KICAgICAgICA8L2Rpdj4KICAgICAgCgogICAgICA8ZGl2IGNsYXNzPSJjb250YWluZXIiPgogICAgICAgIAogICAgICAgICAgPHVsIGNsYXNzPSJicmVhZGNydW1iIj4KICAgICAgICAgICAgCiAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICA8bGk+PGEgaHJlZj0iLyI+QXBpIFJvb3Q8L2E+PC9saT4KICAgICAgICAgICAgICAKICAgICAgICAgICAgCiAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICA8bGkgY2xhc3M9ImFjdGl2ZSI+PGEgaHJlZj0iL3VzZXJzLyI+VXNlciBMaXN0PC9hPjwvbGk+CiAgICAgICAgICAgICAgCiAgICAgICAgICAgIAogICAgICAgICAgPC91bD4KICAgICAgICAKCiAgICAgICAgPCEtLSBDb250ZW50IC0tPgogICAgICAgIDxkaXYgaWQ9ImNvbnRlbnQiIHJvbGU9Im1haW4iIGFyaWEtbGFiZWw9ImNvbnRlbnQiPgogICAgICAgICAgCgogICAgICAgICAgPGRpdiBjbGFzcz0icmVnaW9uIiAgYXJpYS1sYWJlbD0icmVxdWVzdCBmb3JtIj4KICAgICAgICAgIAoKICAgICAgICAgIAogICAgICAgICAgICA8Zm9ybSBpZD0iZ2V0LWZvcm0iIGNsYXNzPSJwdWxsLXJpZ2h0Ij4KICAgICAgICAgICAgICA8ZmllbGRzZXQ+CiAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgPGRpdiBjbGFzcz0iYnRuLWdyb3VwIGZvcm1hdC1zZWxlY3Rpb24iPgogICAgICAgICAgICAgICAgICAgIDxhIGNsYXNzPSJidG4gYnRuLXByaW1hcnkganMtdG9vbHRpcCIgaHJlZj0iL3VzZXJzLyIgcmVsPSJub2ZvbGxvdyIgdGl0bGU9Ik1ha2UgYSBHRVQgcmVxdWVzdCBvbiB0aGUgVXNlciBMaXN0IHJlc291cmNlIj5HRVQ8L2E+CgogICAgICAgICAgICAgICAgICAgIDxidXR0b24gY2xhc3M9ImJ0biBidG4tcHJpbWFyeSBkcm9wZG93bi10b2dnbGUganMtdG9vbHRpcCIgZGF0YS10b2dnbGU9ImRyb3Bkb3duIiB0aXRsZT0iU3BlY2lmeSBhIGZvcm1hdCBmb3IgdGhlIEdFVCByZXF1ZXN0Ij4KICAgICAgICAgICAgICAgICAgICAgIDxzcGFuIGNsYXNzPSJjYXJldCI+PC9zcGFuPgogICAgICAgICAgICAgICAgICAgIDwvYnV0dG9uPgogICAgICAgICAgICAgICAgICAgIDx1bCBjbGFzcz0iZHJvcGRvd24tbWVudSI+CiAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgPGxpPgogICAgICAgICAgICAgICAgICAgICAgICAgIDxhIGNsYXNzPSJqcy10b29sdGlwIGZvcm1hdC1vcHRpb24iIGhyZWY9Ii91c2Vycy8/Zm9ybWF0PWpzb24iIHJlbD0ibm9mb2xsb3ciIHRpdGxlPSJNYWtlIGEgR0VUIHJlcXVlc3Qgb24gdGhlIFVzZXIgTGlzdCByZXNvdXJjZSB3aXRoIHRoZSBmb3JtYXQgc2V0IHRvIGBqc29uYCI+anNvbjwvYT4KICAgICAgICAgICAgICAgICAgICAgICAgPC9saT4KICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICA8bGk+CiAgICAgICAgICAgICAgICAgICAgICAgICAgPGEgY2xhc3M9ImpzLXRvb2x0aXAgZm9ybWF0LW9wdGlvbiIgaHJlZj0iL3VzZXJzLz9mb3JtYXQ9YXBpIiByZWw9Im5vZm9sbG93IiB0aXRsZT0iTWFrZSBhIEdFVCByZXF1ZXN0IG9uIHRoZSBVc2VyIExpc3QgcmVzb3VyY2Ugd2l0aCB0aGUgZm9ybWF0IHNldCB0byBgYXBpYCI+YXBpPC9hPgogICAgICAgICAgICAgICAgICAgICAgICA8L2xpPgogICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgPC91bD4KICAgICAgICAgICAgICAgICAgPC9kaXY+CiAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICA8L2ZpZWxkc2V0PgogICAgICAgICAgICA8L2Zvcm0+CiAgICAgICAgICAKCiAgICAgICAgICAKICAgICAgICAgICAgPGZvcm0gY2xhc3M9ImJ1dHRvbi1mb3JtIiBhY3Rpb249Ii91c2Vycy8iIGRhdGEtbWV0aG9kPSJPUFRJT05TIj4KICAgICAgICAgICAgICA8YnV0dG9uIGNsYXNzPSJidG4gYnRuLXByaW1hcnkganMtdG9vbHRpcCIgdGl0bGU9Ik1ha2UgYW4gT1BUSU9OUyByZXF1ZXN0IG9uIHRoZSBVc2VyIExpc3QgcmVzb3VyY2UiPk9QVElPTlM8L2J1dHRvbj4KICAgICAgICAgICAgPC9mb3JtPgogICAgICAgICAgCgogICAgICAgICAgCgogICAgICAgICAgCgogICAgICAgICAgCgogICAgICAgICAgCiAgICAgICAgICA8L2Rpdj4KCiAgICAgICAgICAgIDxkaXYgY2xhc3M9ImNvbnRlbnQtbWFpbiIgcm9sZT0ibWFpbiIgIGFyaWEtbGFiZWw9Im1haW4gY29udGVudCI+CiAgICAgICAgICAgICAgPGRpdiBjbGFzcz0icGFnZS1oZWFkZXIiPgogICAgICAgICAgICAgICAgPGgxPlVzZXIgTGlzdDwvaDE+CiAgICAgICAgICAgICAgPC9kaXY+CiAgICAgICAgICAgICAgPGRpdiBzdHlsZT0iZmxvYXQ6bGVmdCI+CiAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgPHA+QVBJIGVuZHBvaW50IHRoYXQgYWxsb3dzIHVzZXJzIHRvIGJlIHZpZXdlZCBvciBlZGl0ZWQuPC9wPgogICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgPC9kaXY+CgogICAgICAgICAgICAgIAoKICAgICAgICAgICAgICA8ZGl2IGNsYXNzPSJyZXF1ZXN0LWluZm8iIHN0eWxlPSJjbGVhcjogYm90aCIgYXJpYS1sYWJlbD0icmVxdWVzdCBpbmZvIj4KICAgICAgICAgICAgICAgIDxwcmUgY2xhc3M9InByZXR0eXByaW50Ij48Yj5HRVQ8L2I+IC91c2Vycy88L3ByZT4KICAgICAgICAgICAgICA8L2Rpdj4KCiAgICAgICAgICAgICAgPGRpdiBjbGFzcz0icmVzcG9uc2UtaW5mbyIgYXJpYS1sYWJlbD0icmVzcG9uc2UgaW5mbyI+CiAgICAgICAgICAgICAgICA8cHJlIGNsYXNzPSJwcmV0dHlwcmludCI+PHNwYW4gY2xhc3M9Im1ldGEgbm9jb2RlIj48Yj5IVFRQIDIwMCBPSzwvYj4KPGI+QWxsb3c6PC9iPiA8c3BhbiBjbGFzcz0ibGl0Ij5HRVQsIFBPU1QsIEhFQUQsIE9QVElPTlM8L3NwYW4+CjxiPkNvbnRlbnQtVHlwZTo8L2I+IDxzcGFuIGNsYXNzPSJsaXQiPmFwcGxpY2F0aW9uL2pzb248L3NwYW4+CjxiPlZhcnk6PC9iPiA8c3BhbiBjbGFzcz0ibGl0Ij5BY2NlcHQ8L3NwYW4+Cgo8L3NwYW4+WwogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzcvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvNy88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTcmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFtdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzYvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvNi88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTYmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFtdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzUvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvNS88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTUmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFtdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzQvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvNC88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTQmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFtdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzMvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvMy88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTMmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFtdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzIvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvMi88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTEmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFsKICAgICAgICAgICAgJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL2dyb3Vwcy8xLyIgcmVsPSJub2ZvbGxvdyI+aHR0cDovLzEyNy4wLjAuMTo4MDAwL2dyb3Vwcy8xLzwvYT4mcXVvdDsKICAgICAgICBdCiAgICB9LAogICAgewogICAgICAgICZxdW90O3VybCZxdW90OzogJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL3VzZXJzLzEvIiByZWw9Im5vZm9sbG93Ij5odHRwOi8vMTI3LjAuMC4xOjgwMDAvdXNlcnMvMS88L2E+JnF1b3Q7LAogICAgICAgICZxdW90O3VzZXJuYW1lJnF1b3Q7OiAmcXVvdDt1c2VyNTAmcXVvdDssCiAgICAgICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICAgICAmcXVvdDtncm91cHMmcXVvdDs6IFsKICAgICAgICAgICAgJnF1b3Q7PGEgaHJlZj0iaHR0cDovLzEyNy4wLjAuMTo4MDAwL2dyb3Vwcy8xLyIgcmVsPSJub2ZvbGxvdyI+aHR0cDovLzEyNy4wLjAuMTo4MDAwL2dyb3Vwcy8xLzwvYT4mcXVvdDsKICAgICAgICBdCiAgICB9Cl08L3ByZT4KICAgICAgICAgICAgICA8L2Rpdj4KICAgICAgICAgICAgPC9kaXY+CgogICAgICAgICAgICAKICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgIDxkaXYgY2xhc3M9InRhYmJhYmxlIj4KICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgPHVsIGNsYXNzPSJuYXYgbmF2LXRhYnMgZm9ybS1zd2l0Y2hlciI+CiAgICAgICAgICAgICAgICAgICAgICA8bGk+CiAgICAgICAgICAgICAgICAgICAgICAgIDxhIG5hbWU9J2h0bWwtdGFiJyBocmVmPSIjcG9zdC1vYmplY3QtZm9ybSIgZGF0YS10b2dnbGU9InRhYiI+SFRNTCBmb3JtPC9hPgogICAgICAgICAgICAgICAgICAgICAgPC9saT4KICAgICAgICAgICAgICAgICAgICAgIDxsaT4KICAgICAgICAgICAgICAgICAgICAgICAgPGEgbmFtZT0ncmF3LXRhYicgaHJlZj0iI3Bvc3QtZ2VuZXJpYy1jb250ZW50LWZvcm0iIGRhdGEtdG9nZ2xlPSJ0YWIiPlJhdyBkYXRhPC9hPgogICAgICAgICAgICAgICAgICAgICAgPC9saT4KICAgICAgICAgICAgICAgICAgICA8L3VsPgogICAgICAgICAgICAgICAgICAKCiAgICAgICAgICAgICAgICAgIDxkaXYgY2xhc3M9IndlbGwgdGFiLWNvbnRlbnQiPgogICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgPGRpdiBjbGFzcz0idGFiLXBhbmUiIGlkPSJwb3N0LW9iamVjdC1mb3JtIj4KICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgPGZvcm0gYWN0aW9uPSIvdXNlcnMvIiBtZXRob2Q9IlBPU1QiIGVuY3R5cGU9Im11bHRpcGFydC9mb3JtLWRhdGEiIGNsYXNzPSJmb3JtLWhvcml6b250YWwiIG5vdmFsaWRhdGU+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICA8ZmllbGRzZXQ+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxpbnB1dCB0eXBlPSJoaWRkZW4iIG5hbWU9ImNzcmZtaWRkbGV3YXJldG9rZW4iIHZhbHVlPSJHTzJLT2k4QmdwdmxqNXVLZEw3Tjc2eEJNQXhWZGZqM3VQVkhlQlI4RWpWN3hCWUM5RTlkWnFXclk5TmV5SmcyIj4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCgogIAoKICAKICAgIDxkaXYgY2xhc3M9ImZvcm0tZ3JvdXAgIj4KICAKICAgIDxsYWJlbCBjbGFzcz0iY29sLXNtLTIgY29udHJvbC1sYWJlbCAiPgogICAgICBVc2VybmFtZQogICAgPC9sYWJlbD4KICAKCiAgPGRpdiBjbGFzcz0iY29sLXNtLTEwIj4KICAgIDxpbnB1dCBuYW1lPSJ1c2VybmFtZSIgY2xhc3M9ImZvcm0tY29udHJvbCIgdHlwZT0idGV4dCIgIHZhbHVlPSIiID4KCiAgICAKCiAgICAKICAgICAgPHNwYW4gY2xhc3M9ImhlbHAtYmxvY2siPlJlcXVpcmVkLiAxNTAgY2hhcmFjdGVycyBvciBmZXdlci4gTGV0dGVycywgZGlnaXRzIGFuZCBALy4vKy8tL18gb25seS48L3NwYW4+CiAgICAKICA8L2Rpdj4KPC9kaXY+CgogIAoKICAKICAgIDxkaXYgY2xhc3M9ImZvcm0tZ3JvdXAgIj4KICAKICAgIDxsYWJlbCBjbGFzcz0iY29sLXNtLTIgY29udHJvbC1sYWJlbCAiPgogICAgICBFbWFpbCBhZGRyZXNzCiAgICA8L2xhYmVsPgogIAoKICA8ZGl2IGNsYXNzPSJjb2wtc20tMTAiPgogICAgPGlucHV0IG5hbWU9ImVtYWlsIiBjbGFzcz0iZm9ybS1jb250cm9sIiB0eXBlPSJlbWFpbCIgIHZhbHVlPSIiID4KCiAgICAKCiAgICAKICA8L2Rpdj4KPC9kaXY+CgogIAoKICAKICAgIAoKCgoKPGRpdiBjbGFzcz0iZm9ybS1ncm91cCI+CiAgCiAgICA8bGFiZWwgY2xhc3M9ImNvbC1zbS0yIGNvbnRyb2wtbGFiZWwgIj4KICAgICAgR3JvdXBzCiAgICA8L2xhYmVsPgogIAoKICA8ZGl2IGNsYXNzPSJjb2wtc20tMTAiPgogICAgPHNlbGVjdCBtdWx0aXBsZSAgY2xhc3M9ImZvcm0tY29udHJvbCIgbmFtZT0iZ3JvdXBzIj4KICAgICAgCiAgICAgICAgCiAgICAgICAgICA8b3B0aW9uIHZhbHVlPSJodHRwOi8vMTI3LjAuMC4xOjgwMDAvZ3JvdXBzLzEvIiAgPmE8L29wdGlvbj4KICAgICAgICAKICAgICAgCiAgICAgICAgCiAgICAgICAgICA8b3B0aW9uIHZhbHVlPSJodHRwOi8vMTI3LjAuMC4xOjgwMDAvZ3JvdXBzLzIvIiAgPmdyb3VwIDE8L29wdGlvbj4KICAgICAgICAKICAgICAgCiAgICA8L3NlbGVjdD4KCiAgICAKCiAgICAKICAgICAgPHNwYW4gY2xhc3M9ImhlbHAtYmxvY2siPlRoZSBncm91cHMgdGhpcyB1c2VyIGJlbG9uZ3MgdG8uIEEgdXNlciB3aWxsIGdldCBhbGwgcGVybWlzc2lvbnMgZ3JhbnRlZCB0byBlYWNoIG9mIHRoZWlyIGdyb3Vwcy48L3NwYW4+CiAgICAKICA8L2Rpdj4KPC9kaXY+CgogIAoKCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIDxkaXYgY2xhc3M9ImZvcm0tYWN0aW9ucyI+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPGJ1dHRvbiBjbGFzcz0iYnRuIGJ0bi1wcmltYXJ5IGpzLXRvb2x0aXAiIHRpdGxlPSJNYWtlIGEgUE9TVCByZXF1ZXN0IG9uIHRoZSBVc2VyIExpc3QgcmVzb3VyY2UiPlBPU1Q8L2J1dHRvbj4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPC9kaXY+CiAgICAgICAgICAgICAgICAgICAgICAgICAgICA8L2ZpZWxkc2V0PgogICAgICAgICAgICAgICAgICAgICAgICAgIDwvZm9ybT4KICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICA8L2Rpdj4KICAgICAgICAgICAgICAgICAgICAKCiAgICAgICAgICAgICAgICAgICAgPGRpdiBjbGFzcz0idGFiLXBhbmUiIGlkPSJwb3N0LWdlbmVyaWMtY29udGVudC1mb3JtIj4KICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICA8Zm9ybSBhY3Rpb249Ii91c2Vycy8iIG1ldGhvZD0iUE9TVCIgY2xhc3M9ImZvcm0taG9yaXpvbnRhbCI+CiAgICAgICAgICAgICAgICAgICAgICAgICAgPGZpZWxkc2V0PgogICAgICAgICAgICAgICAgICAgICAgICAgICAgCgoKICA8ZGl2IGNsYXNzPSJmb3JtLWdyb3VwIj4KICAgIDxsYWJlbCBmb3I9ImlkX19jb250ZW50X3R5cGUiIGNsYXNzPSJjb2wtc20tMiBjb250cm9sLWxhYmVsIj5NZWRpYSB0eXBlOjwvbGFiZWw+CiAgICA8ZGl2IGNsYXNzPSJjb2wtc20tMTAiPgogICAgICA8c2VsZWN0IG5hbWU9Il9jb250ZW50X3R5cGUiIGRhdGEtb3ZlcnJpZGU9ImNvbnRlbnQtdHlwZSIgaWQ9ImlkX19jb250ZW50X3R5cGUiIGNsYXNzPSJmb3JtLWNvbnRyb2wiPgogIDxvcHRpb24gdmFsdWU9ImFwcGxpY2F0aW9uL2pzb24iIHNlbGVjdGVkPmFwcGxpY2F0aW9uL2pzb248L29wdGlvbj4KCiAgPG9wdGlvbiB2YWx1ZT0iYXBwbGljYXRpb24veC13d3ctZm9ybS11cmxlbmNvZGVkIj5hcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQ8L29wdGlvbj4KCiAgPG9wdGlvbiB2YWx1ZT0ibXVsdGlwYXJ0L2Zvcm0tZGF0YSI+bXVsdGlwYXJ0L2Zvcm0tZGF0YTwvb3B0aW9uPgoKPC9zZWxlY3Q+CiAgICAgIDxzcGFuIGNsYXNzPSJoZWxwLWJsb2NrIj48L3NwYW4+CiAgICA8L2Rpdj4KICA8L2Rpdj4KCiAgPGRpdiBjbGFzcz0iZm9ybS1ncm91cCI+CiAgICA8bGFiZWwgZm9yPSJpZF9fY29udGVudCIgY2xhc3M9ImNvbC1zbS0yIGNvbnRyb2wtbGFiZWwiPkNvbnRlbnQ6PC9sYWJlbD4KICAgIDxkaXYgY2xhc3M9ImNvbC1zbS0xMCI+CiAgICAgIDx0ZXh0YXJlYSBuYW1lPSJfY29udGVudCIgY29scz0iNDAiIHJvd3M9IjEwIiBkYXRhLW92ZXJyaWRlPSJjb250ZW50IiBpZD0iaWRfX2NvbnRlbnQiIGNsYXNzPSJmb3JtLWNvbnRyb2wiPgp7CiAgICAmcXVvdDt1c2VybmFtZSZxdW90OzogJnF1b3Q7JnF1b3Q7LAogICAgJnF1b3Q7ZW1haWwmcXVvdDs6ICZxdW90OyZxdW90OywKICAgICZxdW90O2dyb3VwcyZxdW90OzogW10KfTwvdGV4dGFyZWE+CiAgICAgIDxzcGFuIGNsYXNzPSJoZWxwLWJsb2NrIj48L3NwYW4+CiAgICA8L2Rpdj4KICA8L2Rpdj4KCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgPGRpdiBjbGFzcz0iZm9ybS1hY3Rpb25zIj4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgPGJ1dHRvbiBjbGFzcz0iYnRuIGJ0bi1wcmltYXJ5IGpzLXRvb2x0aXAiIHRpdGxlPSJNYWtlIGEgUE9TVCByZXF1ZXN0IG9uIHRoZSBVc2VyIExpc3QgcmVzb3VyY2UiPlBPU1Q8L2J1dHRvbj4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIDwvZGl2PgogICAgICAgICAgICAgICAgICAgICAgICAgIDwvZmllbGRzZXQ+CiAgICAgICAgICAgICAgICAgICAgICAgIDwvZm9ybT4KICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgIDwvZGl2PgogICAgICAgICAgICAgICAgICA8L2Rpdj4KICAgICAgICAgICAgICAgIDwvZGl2PgogICAgICAgICAgICAgIAoKICAgICAgICAgICAgICAKICAgICAgICAgICAgCiAgICAgICAgICAKICAgICAgICA8L2Rpdj48IS0tIC8uY29udGVudCAtLT4KICAgICAgPC9kaXY+PCEtLSAvLmNvbnRhaW5lciAtLT4KICAgIDwvZGl2PjwhLS0gLi93cmFwcGVyIC0tPgoKICAgIAoKICAgIAogICAgICA8c2NyaXB0PgogICAgICAgIHdpbmRvdy5kcmYgPSB7CiAgICAgICAgICBjc3JmSGVhZGVyTmFtZTogIlgtQ1NSRlRPS0VOIiwKICAgICAgICAgIGNzcmZUb2tlbjogIkdPMktPaThCZ3B2bGo1dUtkTDdONzZ4Qk1BeFZkZmozdVBWSGVCUjhFalY3eEJZQzlFOWRacVdyWTlOZXlKZzIiCiAgICAgICAgfTsKICAgICAgPC9zY3JpcHQ+CiAgICAgIDxzY3JpcHQgc3JjPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2pzL2pxdWVyeS0zLjUuMS5taW4uanMiPjwvc2NyaXB0PgogICAgICA8c2NyaXB0IHNyYz0iL3N0YXRpYy9yZXN0X2ZyYW1ld29yay9qcy9hamF4LWZvcm0uanMiPjwvc2NyaXB0PgogICAgICA8c2NyaXB0IHNyYz0iL3N0YXRpYy9yZXN0X2ZyYW1ld29yay9qcy9jc3JmLmpzIj48L3NjcmlwdD4KICAgICAgPHNjcmlwdCBzcmM9Ii9zdGF0aWMvcmVzdF9mcmFtZXdvcmsvanMvYm9vdHN0cmFwLm1pbi5qcyI+PC9zY3JpcHQ+CiAgICAgIDxzY3JpcHQgc3JjPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2pzL3ByZXR0aWZ5LW1pbi5qcyI+PC9zY3JpcHQ+CiAgICAgIDxzY3JpcHQgc3JjPSIvc3RhdGljL3Jlc3RfZnJhbWV3b3JrL2pzL2RlZmF1bHQuanMiPjwvc2NyaXB0PgogICAgICA8c2NyaXB0PgogICAgICAgICQoZG9jdW1lbnQpLnJlYWR5KGZ1bmN0aW9uKCkgewogICAgICAgICAgJCgnZm9ybScpLmFqYXhGb3JtKCk7CiAgICAgICAgfSk7CiAgICAgIDwvc2NyaXB0PgogICAgCgogIDwvYm9keT4KICAKPC9odG1sPgo=', + headers={'Content-Type': 'text/html; charset=utf-8', 'Vary': 'Accept', 'Allow': 'GET, POST, HEAD, OPTIONS', + 'X-Frame-Options': 'DENY', 'x-moesif-transaction-id': '7fa3a0a9-a6a2-42a3-893f-037c339fa290'}, + status=200, + time='2023-04-12T21:49:41.206', + transfer_encoding='base64' + ) + + self.event = EventModel( + user_id='u1', + company_id='c1', + direction='Incoming', + metadata={'datacenter': 'westus', 'deployment_version': 'v1.2.3'}, + request=self.request, + response=self.response, + session_token='XXXXXXXXXX') + + self.user_entity_rules_from_config = {'user_rules': + { + 'u1': [ + {'rules': '642f4fcea6ca1c38705d660d', 'values': {'0': 'u1'}}, + {'rules': '6435bc43682c9b013785e60f'}, + {'rules': '643e13106bd6416306aa187c', 'values': {'0': 'u1', '1': 'c1'}} + ] + }, + 'company_rules': {} + } + + def test_get_entity_rule_mapping_from_config(self): + rule_values = self.gov_helper.get_entity_rule_mapping_from_config(self.user_entity_rules_from_config, 'user_rules', 'u1') + print(rule_values) - def test_something(self): - print(self.request) - self.assertEqual(True, True) # add assertion here def test_check_event_should_blocked_by_rule_should_block(self): - block = self.gov_helper.check_event_should_blocked_by_rule(self.gov_rule, self.request_mapping_for_regex_config, self.ready_for_body_request) + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.request_mapping_for_regex_config, + self.ready_for_body_request) self.assertEqual(block, True) def test_check_event_should_blocked_by_rule_should_not_block(self): self.request_mapping_for_regex_config['request.route'] = 'http://127.0.0.1:8000/groups/' - block = self.gov_helper.check_event_should_blocked_by_rule(self.gov_rule, self.request_mapping_for_regex_config, self.ready_for_body_request) + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.request_mapping_for_regex_config, + self.ready_for_body_request) self.assertEqual(block, False) + def test_apply_governance_rules_regex_matching(self): + self.new_schema_regex_gov_rule['applied_to'] = AppliedTo.MATCHING.value + self.request.uri = 'http://127.0.0.1:8000/555/' + self.event.request = self.request + + regex_gov_rules = {self.new_schema_regex_gov_rule.get('_id'): self.new_schema_regex_gov_rule} + blocking_response = self.gov_helper.apply_governance_rules( + self.event, + None, None, None, None, + None, None, None, None, + regex_gov_rules, + True + ) + self.assertEqual(blocking_response.blocked, True) + self.assertEqual(blocking_response.block_response_status, self.new_schema_regex_gov_rule['response']['status']) + self.assertEqual(blocking_response.block_response_body, self.new_schema_regex_gov_rule['response']['body']) + self.assertEqual(blocking_response.block_response_headers, + self.new_schema_regex_gov_rule['response']['headers']) + + def test_apply_governance_rules_regex_not_matching(self): + self.new_schema_regex_gov_rule['applied_to'] = AppliedTo.NOT_MATCHING.value + self.request.uri = 'http://127.0.0.1:8000/556/' # the url not matching with regex rule config + self.event.request = self.request + + regex_gov_rules = {self.new_schema_regex_gov_rule.get('_id'): self.new_schema_regex_gov_rule} + blocking_response = self.gov_helper.apply_governance_rules( + self.event, + None, None, None, None, + None, None, None, None, + regex_gov_rules, + True + ) + self.assertEqual(blocking_response.blocked, True) + self.assertEqual(blocking_response.block_response_status, self.new_schema_regex_gov_rule['response']['status']) + self.assertEqual(blocking_response.block_response_body, self.new_schema_regex_gov_rule['response']['body']) + self.assertEqual(blocking_response.block_response_headers, + self.new_schema_regex_gov_rule['response']['headers']) + # self.assertIsNone(blocking_response) + + def test_apply_governance_rules_user_matching_identified(self): + self.user_gov_rule['type'] = RuleType.USER.value + self.user_gov_rule['applied_to'] = AppliedTo.MATCHING.value + self.user_gov_rule['applied_to_unidentified'] = False + identified_user_rules = {self.user_gov_rule.get('_id'): self.user_gov_rule} + blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + identified_user_rules, + None, None, None, None, True + ) + + self.assertEqual(blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], blocking_response.block_response_status) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], blocking_response.block_response_headers) + + def test_apply_governance_rules_user_matching_unidentified(self): + self.user_gov_rule['type'] = RuleType.USER.value + self.user_gov_rule['applied_to'] = AppliedTo.MATCHING.value + self.user_gov_rule['applied_to_unidentified'] = True + unidentified_user_rules = {self.user_gov_rule.get('_id'): self.user_gov_rule} + not_null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, unidentified_user_rules, None, None, None, True + ) + self.assertEqual(not_null_user_blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], not_null_user_blocking_response.block_response_status) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, not_null_user_blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], not_null_user_blocking_response.block_response_headers) + + self.event.user_id = None + null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, + unidentified_user_rules, + None, None, None, True + ) + self.assertEqual(null_user_blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], null_user_blocking_response.block_response_status) + self.assertEqual(self.user_gov_rule['response']['body'], null_user_blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], null_user_blocking_response.block_response_headers) + if __name__ == '__main__': unittest.main() From abb089975846e62567201b4de04d843f83c09c11 Mon Sep 17 00:00:00 2001 From: xiluan Date: Wed, 19 Apr 2023 16:28:40 -0700 Subject: [PATCH 4/7] add unidentified, not matching rules tests --- moesifdjango/tests/test_apply_gov_rules.py | 99 ++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/moesifdjango/tests/test_apply_gov_rules.py b/moesifdjango/tests/test_apply_gov_rules.py index dae7484..9382ca3 100644 --- a/moesifdjango/tests/test_apply_gov_rules.py +++ b/moesifdjango/tests/test_apply_gov_rules.py @@ -271,6 +271,105 @@ def test_apply_governance_rules_user_matching_unidentified(self): self.assertEqual(self.user_gov_rule['response']['body'], null_user_blocking_response.block_response_body) self.assertEqual(self.user_gov_rule['response']['headers'], null_user_blocking_response.block_response_headers) + def test_apply_governance_rules_user_not_matching_identified(self): + self.user_gov_rule['type'] = RuleType.USER.value + self.user_gov_rule['applied_to'] = AppliedTo.NOT_MATCHING.value + self.user_gov_rule['applied_to_unidentified'] = False + + identified_user_rules = {self.user_gov_rule.get('_id'): self.user_gov_rule} + # not matched event: should block + self.request.uri = 'http://127.0.0.1:8000/not_matching/' # the url not matching with regex rule config + self.event.request = self.request + + blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + identified_user_rules, + None, None, None, None, True + ) + + self.assertEqual(blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], blocking_response.block_response_status) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, + blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], blocking_response.block_response_headers) + + # matched event: should not block + self.request.uri = 'http://127.0.0.1:8000/users/' # the url not matching with regex rule config + self.event.request = self.request + + blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + identified_user_rules, + None, None, None, None, True + ) + + self.assertIsNone(blocking_response) + + def test_apply_governance_rules_user_not_matching_unidentified(self): + self.user_gov_rule['type'] = RuleType.USER.value + self.user_gov_rule['applied_to'] = AppliedTo.NOT_MATCHING.value + self.user_gov_rule['applied_to_unidentified'] = True + unidentified_user_rules = {self.user_gov_rule.get('_id'): self.user_gov_rule} + + # matched event: should not block + self.request.uri = 'http://127.0.0.1:8000/users/' # the url not matching with regex rule config + self.event.request = self.request + self.event.user_id = 'u1' + not_null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, unidentified_user_rules, None, None, None, True + ) + self.assertIsNone(not_null_user_blocking_response) + + self.event.user_id = None + null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, + unidentified_user_rules, + None, None, None, True + ) + self.assertIsNone(null_user_blocking_response) + + # not matched event: should block + self.request.uri = 'http://127.0.0.1:8000/not_matching/' # the url not matching with regex rule config + self.event.request = self.request + self.event.user_id = 'u1' + not_null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, unidentified_user_rules, None, None, None, True + ) + self.assertEqual(not_null_user_blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], + not_null_user_blocking_response.block_response_status) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, + not_null_user_blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], + not_null_user_blocking_response.block_response_headers) + + self.event.user_id = None + null_user_blocking_response = self.gov_helper.apply_governance_rules( + self.event, + self.event.user_id, None, None, + self.user_entity_rules_from_config, + None, + unidentified_user_rules, + None, None, None, True + ) + self.assertEqual(null_user_blocking_response.blocked, True) + self.assertEqual(self.user_gov_rule['response']['status'], null_user_blocking_response.block_response_status) + self.assertEqual(self.user_gov_rule['response']['body'], null_user_blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], null_user_blocking_response.block_response_headers) + if __name__ == '__main__': unittest.main() From 50e1c0c0ddfc9b417ceaa2d687a97704501e3af7 Mon Sep 17 00:00:00 2001 From: xiluan Date: Thu, 20 Apr 2023 10:50:00 -0700 Subject: [PATCH 5/7] added test case for data transform, fix UNKNOWN populate issue --- moesifdjango/moesif_gov.py | 60 +++++++++++++++++----- moesifdjango/tests/test_apply_gov_rules.py | 53 ++++++++++++++++--- 2 files changed, 92 insertions(+), 21 deletions(-) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index 16bc2e4..e512287 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -103,6 +103,10 @@ def transform_values(self, data, rule_values): """ if data is None: return None + + if not rule_values or len(rule_values) == 0: + return data + if isinstance(data, str): max_index = max(rule_values.keys()) @@ -202,7 +206,6 @@ def check_request_with_regex_match(self, gr_regex_configs_list, request_mapping_ # If regex conditions are not matched, return default sample rate (nil) and do not block request (false) return False - @classmethod def get_req_content_type(cls, request): """ @@ -294,8 +297,15 @@ def get_updated_response_with_matched_entity_rules(self, governance_rule, rule_a except Exception as e: print('[moesif] Error when converting entity rules values key: ', e) - updated_gr_headers = self.transform_values(updated_gr_headers, updated_gr_values) - updated_gr_body = self.transform_values(updated_gr_body, updated_gr_values) + merge_tag_variables = self.merge_tag_variables_rule_mapping(governance_rule) + + # set default value if user/company entity value is None + for k, v in merge_tag_variables.items(): + if k not in updated_gr_values or not updated_gr_values[k]: + updated_gr_values[k] = v['default'] + + updated_gr_headers = self.transform_values(updated_gr_headers, updated_gr_values) + updated_gr_body = self.transform_values(updated_gr_body, updated_gr_values) return gr_status, updated_gr_headers, updated_gr_body @@ -515,19 +525,41 @@ def check_if_request_blocked(cls, response_buffers): return True return False - def get_entity_rule_mapping_from_config(self, entity_rules, rule_type_in_config, entity_id): + @classmethod + def merge_tag_variables_rule_mapping(cls, governance_rule): + merge_tag_variables = {} + variables = governance_rule.get('variables', []) + for variable in variables: + try: + name = int(variable['name']) + path = variable['path'] + default_value = variable.get('default', 'UNKNOWN') + merge_tag_variables.update( + { + name: { + 'path': path, + 'default': default_value + } + } + ) + except Exception as e: + print('[moesif] Error when parsing rule {} variable'.format(governance_rule.get('id')), e) + return merge_tag_variables + + @classmethod + def get_entity_rule_mapping_from_config(cls, entity_rules, rule_type_in_config, entity_id): rule_merge_tag_values_mapping = {} try: if entity_id: rules_mapping_from_config = entity_rules[rule_type_in_config][entity_id] for rule_values in rules_mapping_from_config: + rule_id = rule_values['rules'] + if rule_id not in rule_merge_tag_values_mapping: + rule_merge_tag_values_mapping[rule_id] = {} if 'values' in rule_values: - rule_id = rule_values['rules'] values = rule_values['values'] - - if rule_id not in rule_merge_tag_values_mapping: - rule_merge_tag_values_mapping[rule_id] = {} rule_merge_tag_values_mapping[rule_id].update(values) + except Exception as e: print('[moesif] Skipped blocking request, Error when fetching entity rule with entity {}, {}'.format( entity_id, e)) @@ -549,8 +581,10 @@ def apply_governance_rules(self, request_mapping_for_regex_config = self.prepare_request_config_based_on_regex_config(event, ready_for_body_request) - user_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'user_rules', user_id_entity) - company_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'company_rules', company_id_entity) + user_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'user_rules', + user_id_entity) + company_rules_mapping_from_config = self.get_entity_rule_mapping_from_config(entity_rules, 'company_rules', + company_id_entity) response_buffers = { RuleType.REGEX.value: BlockResponseBufferList(RuleType.REGEX.value), @@ -614,7 +648,7 @@ def apply_governance_rules(self, print('[moesif] company_id is not valid or no identified company governance rules') if unidentified_user_governance_rules: - unidentified_company_response_buffer = \ + unidentified_user_response_buffer = \ self.block_request_based_on_regex_or_unidentified_entity_governance_rule( request_mapping_for_regex_config, ready_for_body_request, @@ -623,11 +657,11 @@ def apply_governance_rules(self, user_rules_mapping_from_config, DEBUG ) - if not unidentified_company_response_buffer.blocked: + if not unidentified_user_response_buffer.blocked: if DEBUG: print('[moesif] No matching with the request from unidentified user rules') else: - response_buffers[RuleType.USER.value] = unidentified_company_response_buffer + response_buffers[RuleType.USER.value] = unidentified_user_response_buffer else: if DEBUG: print('[moesif] no unidentified user governance rules') diff --git a/moesifdjango/tests/test_apply_gov_rules.py b/moesifdjango/tests/test_apply_gov_rules.py index 9382ca3..f8b331b 100644 --- a/moesifdjango/tests/test_apply_gov_rules.py +++ b/moesifdjango/tests/test_apply_gov_rules.py @@ -166,11 +166,44 @@ def setUp(self): 'company_rules': {} } + def test_transform_values(self): + response_body = self.gov_helper.transform_values( + {'msg': 'Blocked by Gov Rule', + 'user_id': '{{0}}'}, + {0: 'u1'}) + self.assertEqual(response_body['user_id'], 'u1') + + def test_get_updated_response_with_matched_entity_rules(self): + rule_to_value_none = { + '642f4fcea6ca1c38705d660d': {'0': None} + } + updated_gr_status, updated_gr_headers, updated_gr_body = \ + self.gov_helper.get_updated_response_with_matched_entity_rules( + self.user_gov_rule, + rule_to_value_none, + True + ) + print(updated_gr_body) + self.assertEqual('UNKNOWN', updated_gr_body['user_id']) + + rule_to_value_no_index = { + '642f4fcea6ca1c38705d660d': {} + } + updated_gr_status2, updated_gr_headers2, updated_gr_body2 = \ + self.gov_helper.get_updated_response_with_matched_entity_rules( + self.user_gov_rule, + rule_to_value_no_index, + True + ) + print(updated_gr_body2) + self.assertEqual('UNKNOWN', updated_gr_body2['user_id']) + + def test_get_entity_rule_mapping_from_config(self): - rule_values = self.gov_helper.get_entity_rule_mapping_from_config(self.user_entity_rules_from_config, 'user_rules', 'u1') + rule_values = self.gov_helper.get_entity_rule_mapping_from_config(self.user_entity_rules_from_config, + 'user_rules', 'u1') print(rule_values) - def test_check_event_should_blocked_by_rule_should_block(self): block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, self.request_mapping_for_regex_config, @@ -238,7 +271,8 @@ def test_apply_governance_rules_user_matching_identified(self): self.assertEqual(blocking_response.blocked, True) self.assertEqual(self.user_gov_rule['response']['status'], blocking_response.block_response_status) - self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, blocking_response.block_response_body) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, + blocking_response.block_response_body) self.assertEqual(self.user_gov_rule['response']['headers'], blocking_response.block_response_headers) def test_apply_governance_rules_user_matching_unidentified(self): @@ -253,9 +287,12 @@ def test_apply_governance_rules_user_matching_unidentified(self): None, unidentified_user_rules, None, None, None, True ) self.assertEqual(not_null_user_blocking_response.blocked, True) - self.assertEqual(self.user_gov_rule['response']['status'], not_null_user_blocking_response.block_response_status) - self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, not_null_user_blocking_response.block_response_body) - self.assertEqual(self.user_gov_rule['response']['headers'], not_null_user_blocking_response.block_response_headers) + self.assertEqual(self.user_gov_rule['response']['status'], + not_null_user_blocking_response.block_response_status) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': '{}'.format(self.event.user_id)}, + not_null_user_blocking_response.block_response_body) + self.assertEqual(self.user_gov_rule['response']['headers'], + not_null_user_blocking_response.block_response_headers) self.event.user_id = None null_user_blocking_response = self.gov_helper.apply_governance_rules( @@ -268,7 +305,7 @@ def test_apply_governance_rules_user_matching_unidentified(self): ) self.assertEqual(null_user_blocking_response.blocked, True) self.assertEqual(self.user_gov_rule['response']['status'], null_user_blocking_response.block_response_status) - self.assertEqual(self.user_gov_rule['response']['body'], null_user_blocking_response.block_response_body) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': 'UNKNOWN'}, null_user_blocking_response.block_response_body) self.assertEqual(self.user_gov_rule['response']['headers'], null_user_blocking_response.block_response_headers) def test_apply_governance_rules_user_not_matching_identified(self): @@ -367,7 +404,7 @@ def test_apply_governance_rules_user_not_matching_unidentified(self): ) self.assertEqual(null_user_blocking_response.blocked, True) self.assertEqual(self.user_gov_rule['response']['status'], null_user_blocking_response.block_response_status) - self.assertEqual(self.user_gov_rule['response']['body'], null_user_blocking_response.block_response_body) + self.assertEqual({'msg': 'Blocked by Gov Rule', 'user_id': 'UNKNOWN'}, null_user_blocking_response.block_response_body) self.assertEqual(self.user_gov_rule['response']['headers'], null_user_blocking_response.block_response_headers) From a9bcadc882b00b3a33cf6d44eb1b3a9a4844df44 Mon Sep 17 00:00:00 2001 From: xiluan Date: Wed, 5 Jul 2023 15:46:22 -0700 Subject: [PATCH 6/7] add the case when user is not in cohort with unidentified gov rules --- moesifdjango/moesif_gov.py | 79 +++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index 77ca430..bf9289d 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -324,7 +324,6 @@ def block_request_based_on_entity_governance_rule_identified(self, :param ready_for_body_request: :param governance_rules: :param entity_rules: - :param rule_entity_type: the type of rules from config (user_rules, company_rules or regex_rules) :param entity_id: user_id or company_id from event :param DEBUG: :return: object of updated response status, headers and body, if criteria is matched and block is true, otherwise return None @@ -346,7 +345,8 @@ def block_request_based_on_entity_governance_rule_identified(self, if not should_block: if DEBUG: print( - "[moesif] Skipped blocking request because it's not satisfied with governance rule {}".format( + "[moesif] Skipped blocking request because it's not satisfied with {} governance rule {}".format( + rule_type, rule_id)) continue @@ -437,9 +437,9 @@ def block_request_based_on_regex_or_unidentified_entity_governance_rule(self, gr_status, gr_header, gr_body = \ self.get_updated_response_with_matched_entity_rules(governance_rule, entity_rules, DEBUG) - response_buffer.update(block, gr_status, gr_header, gr_body, rule_id) - if DEBUG: - print('[moesif] request matched with regex rule with rule_id {}'.format(rule_id)) + response_buffer.update(block, gr_status, gr_header, gr_body, rule_id) + if DEBUG: + print('[moesif] request matched with regex rule with rule_id {}'.format(rule_id)) return response_buffer @@ -614,18 +614,30 @@ def apply_governance_rules(self, print('[moesif] No regex rules') if unidentified_company_governance_rules: - unidentified_company_response_buffer = \ - self.block_request_based_on_regex_or_unidentified_entity_governance_rule( - request_mapping_for_regex_config, - ready_for_body_request, - unidentified_company_governance_rules, - RuleType.COMPANY.value, - company_rules_mapping_from_config, - DEBUG - ) - if not unidentified_company_response_buffer.blocked: + if not company_id_entity: + unidentified_company_response_buffer = \ + self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + request_mapping_for_regex_config, + ready_for_body_request, + unidentified_company_governance_rules, + RuleType.COMPANY.value, + company_rules_mapping_from_config, + DEBUG + ) + else: + unidentified_company_response_buffer = \ + self.block_request_based_on_entity_governance_rule_identified( + request_mapping_for_regex_config, + ready_for_body_request, + unidentified_company_governance_rules, + RuleType.COMPANY.value, + company_rules_mapping_from_config, + company_id_entity, + DEBUG) + if not unidentified_company_response_buffer or not unidentified_company_response_buffer.blocked: if DEBUG: - print('[moesif] No matching with the request from unidentified company rules') + print('[moesif] No matching with the request from unidentified company rules for {}' + .format(company_id_entity if company_id_entity else 'unidentified company')) else: response_buffers[RuleType.COMPANY.value] = unidentified_company_response_buffer else: @@ -643,33 +655,45 @@ def apply_governance_rules(self, DEBUG) if not company_response_buffer.blocked: if DEBUG: - print('[moesif] No blocking from company: ', company_id_entity) + print('[moesif] No matching with the request from identified company rules for {}'.format(company_id_entity)) else: response_buffers[RuleType.COMPANY.value] = merge_block_response( response_buffers.get(RuleType.COMPANY.value, BlockResponseBufferList()), company_response_buffer) else: if DEBUG: - print('[moesif] company_id is not valid or no identified company governance rules') + print('[moesif] company_id is unidentified or no identified company governance rules') if unidentified_user_governance_rules: - unidentified_user_response_buffer = \ - self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + if not user_id_entity: + unidentified_user_response_buffer = \ + self.block_request_based_on_regex_or_unidentified_entity_governance_rule( + request_mapping_for_regex_config, + ready_for_body_request, + unidentified_user_governance_rules, + RuleType.USER.value, + user_rules_mapping_from_config, + DEBUG + ) + else: + unidentified_user_response_buffer = self.block_request_based_on_entity_governance_rule_identified( request_mapping_for_regex_config, ready_for_body_request, unidentified_user_governance_rules, RuleType.USER.value, user_rules_mapping_from_config, - DEBUG - ) - if not unidentified_user_response_buffer.blocked: + user_id_entity, + DEBUG) + if not unidentified_user_response_buffer or not unidentified_user_response_buffer.blocked: if DEBUG: - print('[moesif] No matching with the request from unidentified user rules') + print('[moesif] No matching with the request from unidentified user rules for {}'.format( + user_id_entity if user_id_entity else "unidentified user" + )) else: response_buffers[RuleType.USER.value] = unidentified_user_response_buffer else: if DEBUG: - print('[moesif] no unidentified user governance rules') + print('[moesif] No unidentified user governance rules') if user_id_entity and identified_user_governance_rules: identified_user_response_buffer = self.block_request_based_on_entity_governance_rule_identified( @@ -683,14 +707,15 @@ def apply_governance_rules(self, if not identified_user_response_buffer.blocked: if DEBUG: - print('[moesif] No blocking from user: ', user_id_entity) + print('[moesif] No matching with the request from identified user rules for {}'.format( + user_id_entity)) else: response_buffers[RuleType.USER.value] = merge_block_response( response_buffers.get(RuleType.USER.value, BlockResponseBufferList()), identified_user_response_buffer) else: if DEBUG: - print('[moesif] user_id is not valid or no identified user governance rules') + print('[moesif] user_id is unidentified or no identified user governance rules') blocking_response = self.generate_blocking_response(response_buffers) From 3641193f323c14918c8c1c4a0418a5d96ed7a594 Mon Sep 17 00:00:00 2001 From: xiluan Date: Fri, 10 Nov 2023 17:58:57 -0800 Subject: [PATCH 7/7] add apply_to in cohort --- moesifdjango/moesif_gov.py | 41 ++++++++++++++-------- moesifdjango/tests/test_apply_gov_rules.py | 36 ++++++++++++++++++- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/moesifdjango/moesif_gov.py b/moesifdjango/moesif_gov.py index bf9289d..7f59bbe 100644 --- a/moesifdjango/moesif_gov.py +++ b/moesifdjango/moesif_gov.py @@ -330,8 +330,8 @@ def block_request_based_on_entity_governance_rule_identified(self, """ response_buffer = BlockResponseBufferList(rule_type) - for rule_id, _ in entity_rules.items(): - governance_rule = governance_rules.get(rule_id, None) + # for rule_id, _ in entity_rules.items(): + for rule_id, governance_rule in governance_rules.items(): if not governance_rule or 'response' not in governance_rule or 'status' not in governance_rule['response']: if DEBUG: @@ -340,7 +340,7 @@ def block_request_based_on_entity_governance_rule_identified(self, entity_id) continue - should_block = self.check_event_should_blocked_by_rule(governance_rule, request_mapping_for_regex_config, ready_for_body_request) + should_block = self.check_event_should_blocked_by_rule(governance_rule, entity_rules, request_mapping_for_regex_config, ready_for_body_request) if not should_block: if DEBUG: @@ -363,6 +363,7 @@ def block_request_based_on_entity_governance_rule_identified(self, return response_buffer def check_event_should_blocked_by_rule(self, governance_rule, + entity_rules, request_mapping_for_regex_config, ready_for_body_request): applied_to = governance_rule.get('applied_to', AppliedTo.MATCHING.value) @@ -376,8 +377,14 @@ def check_event_should_blocked_by_rule(self, governance_rule, request_mapping_for_regex_config, ready_for_body_request) - return (matched and applied_to == AppliedTo.MATCHING.value) \ - or (not matched and applied_to == AppliedTo.NOT_MATCHING.value) + in_cohort = self.is_in_cohort(entity_rules, governance_rule.get('_id')) + + return (matched and in_cohort and applied_to == AppliedTo.MATCHING.value) \ + or (not matched and not in_cohort and applied_to == AppliedTo.NOT_MATCHING.value) + + def is_in_cohort(self, entity_rules, rule_id): + return entity_rules is not None and len(entity_rules) != 0 and rule_id in entity_rules + def get_rules_id_if_governance_rule_matched(self, governance_rules, request_mapping_for_regex_config, ready_for_body_request): @@ -391,7 +398,7 @@ def get_rules_id_if_governance_rule_matched(self, governance_rules, request_mapp matched_rules_id = [] for id, rule in governance_rules.items(): - matched = self.check_event_should_blocked_by_rule(rule, request_mapping_for_regex_config, + matched = self.check_event_should_blocked_by_rule(rule, {}, request_mapping_for_regex_config, ready_for_body_request) if matched: @@ -552,18 +559,22 @@ def merge_tag_variables_rule_mapping(cls, governance_rule): return merge_tag_variables @classmethod - def get_entity_rule_mapping_from_config(cls, entity_rules, rule_type_in_config, entity_id): + def get_entity_rule_mapping_from_config(cls, entity_rules, rule_type, entity_id): rule_merge_tag_values_mapping = {} try: if entity_id: - rules_mapping_from_config = entity_rules[rule_type_in_config][entity_id] - for rule_values in rules_mapping_from_config: - rule_id = rule_values['rules'] - if rule_id not in rule_merge_tag_values_mapping: - rule_merge_tag_values_mapping[rule_id] = {} - if 'values' in rule_values: - values = rule_values['values'] - rule_merge_tag_values_mapping[rule_id].update(values) + entities_rules = entity_rules[rule_type] + if entity_id in entities_rules: + rules_mapping_from_config = entities_rules[entity_id] + for rule_values in rules_mapping_from_config: + rule_id = rule_values['rules'] + if rule_id not in rule_merge_tag_values_mapping: + rule_merge_tag_values_mapping[rule_id] = {} + if 'values' in rule_values: + values = rule_values['values'] + rule_merge_tag_values_mapping[rule_id].update(values) + else: + rule_merge_tag_values_mapping[rule_id] = {} except Exception as e: print('[moesif] Skipped blocking request, Error when fetching entity rule with entity {}, {}'.format( diff --git a/moesifdjango/tests/test_apply_gov_rules.py b/moesifdjango/tests/test_apply_gov_rules.py index f8b331b..06e2e6e 100644 --- a/moesifdjango/tests/test_apply_gov_rules.py +++ b/moesifdjango/tests/test_apply_gov_rules.py @@ -70,6 +70,10 @@ def setUp(self): } } + self.non_empty_entity_rules = {"642f4fcea6ca1c38705d660d":{'0': 'u1'}} + self.non_empty_entity_rules2 = {"xxx": {'0': 'u1'}} + self.empty_entity_rules = {} + self.new_schema_regex_gov_rule = { "_id": "6439b3f5d5762a04c0623d8b", "regex_config": [ @@ -204,12 +208,42 @@ def test_get_entity_rule_mapping_from_config(self): 'user_rules', 'u1') print(rule_values) - def test_check_event_should_blocked_by_rule_should_block(self): + def test_should_block(self): block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.non_empty_entity_rules, self.request_mapping_for_regex_config, self.ready_for_body_request) self.assertEqual(block, True) + def test_should_not_block(self): + # event match & 0 entity rules + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.empty_entity_rules, + self.request_mapping_for_regex_config, + self.ready_for_body_request) + self.assertEqual(block, False) + + # event match & other entity rules(not in cohort) + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.non_empty_entity_rules2, + self.request_mapping_for_regex_config, + self.ready_for_body_request) + self.assertEqual(block, False) + + # event not match & matched entity rules(in cohort) + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.non_empty_entity_rules, + self.request_mapping_for_regex_config2, + self.ready_for_body_request) + self.assertEqual(block, False) + + # event not match & matched entity rules(in cohort) + block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule, + self.non_empty_entity_rules2, + self.request_mapping_for_regex_config2, + self.ready_for_body_request) + self.assertEqual(block, False) + def test_check_event_should_blocked_by_rule_should_not_block(self): self.request_mapping_for_regex_config['request.route'] = 'http://127.0.0.1:8000/groups/' block = self.gov_helper.check_event_should_blocked_by_rule(self.user_gov_rule,