update code with permissions, export

Author: sressier

lambdas/functions/iroco2-client-side-scanner/cur.tf

@@ -2,7 +2,7 @@ resource "aws_bcmdataexports_export" "CUR" {
   export {
     name = var.bcm_data_export_name
     data_query {
-      query_statement = "SELECT identity_line_item_id, identity_time_interval, line_item_product_code,line_item_unblended_cost FROM COST_AND_USAGE_REPORT"
+      query_statement = "SELECT line_item_usage_amount, product_region_code, product_servicecode, line_item_usage_type FROM COST_AND_USAGE_REPORT"
       table_configurations = {
         COST_AND_USAGE_REPORT = {
           BILLING_VIEW_ARN                      = "arn:aws:billing::${data.aws_caller_identity.current.account_id}:billingview/primary"

lambdas/functions/iroco2-client-side-scanner/iam.tf

@@ -39,7 +39,10 @@ resource "aws_iam_role_policy" "lambda_s3_kms_access" {
           "s3:GetObjectVersion",
           "s3:ListBucket"
         ]
-        Resource = "${aws_s3_bucket.cur_output.arn}/*"
+        Resource = [
+          "${aws_s3_bucket.cur_output.arn}",
+          "${aws_s3_bucket.cur_output.arn}/*"
+        ]
       },
       {
         Effect = "Allow"

lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/ec2_service.py

@@ -22,22 +22,26 @@ class CurProcessorEC2Service:
 
     cur_useful_column_keyname_matrix = {
         'service_code': {
+            'gz': 'product/servicecode',
             'zip': 'product/servicecode',
             'csv': 'product/servicecode',
             'parquet': 'product_servicecode'
         },
         'usage_type': {
+            'gz': 'lineItem/UsageType',
             'zip': 'lineItem/UsageType',
             'csv': 'lineItem/UsageType',
             'parquet': 'line_item_usage_type'
         },
         'region_code': {
             'zip': 'product/regionCode',
+            'gz': 'product/regionCode',
             'csv': 'product/regionCode',
             'parquet': 'product_region_code'
         },
         'instance_type': {
             'zip': 'product/instanceType',
+            'gz': 'product/instanceType',
             'csv': 'product/instanceType',
             'parquet': 'product_instance_type'
         }
@@ -47,6 +51,8 @@ def creating_message_from_cur(self, cur, cur_file_type):
         try:
             useful_column = [values[cur_file_type] for values in self.cur_useful_column_keyname_matrix.values()]
             match cur_file_type:
+                case 'gz':
+                    cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_column, compression='gzip', low_memory=False)
                 case 'zip':
                     cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_column, compression='zip', low_memory=False)
                 case 'csv':

lambdas/functions/iroco2-client-side-scanner/layers/python/lib/python3.11/site-packages/s3_service.py

@@ -25,21 +25,25 @@ class CurProcessorS3Service:
     cur_useful_column_keyname_matrix = {
         'service_code': {
             'zip': 'product/servicecode',
+            'gz': 'product/servicecode',
             'csv': 'product/servicecode',
             'parquet': 'product_servicecode'
         },
         'usage_type': {
             'zip': 'lineItem/UsageType',
+            'gz': 'lineItem/UsageType',
             'csv': 'lineItem/UsageType',
             'parquet': 'line_item_usage_type'
         },
         'region_code': {
             'zip': 'product/regionCode',
+            'gz': 'product/regionCode',
             'csv': 'product/regionCode',
             'parquet': 'product_region_code'
         },
         'usage_amount': {
             'zip': 'lineItem/UsageAmount',
+            'gz': 'lineItem/UsageAmount',
             'csv': 'lineItem/UsageAmount',
             'parquet': 'line_item_usage_amount'
         }
@@ -86,6 +90,8 @@ def __creating_message_from_cur_for_storage(self, cur):
         match self.cur_file_type:
             case 'zip':
                 cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_columns, compression='zip', low_memory=False)
+            case 'gz':
+                cur_with_useful_colum = pd.read_csv(BytesIO(cur), usecols=useful_columns, compression='gzip', low_memory=False)
             case 'csv':
                 cur_with_useful_colum = pd.read_csv(StringIO(cur.decode('utf-8')), usecols=useful_columns, low_memory=False)
             case 'parquet':

lambdas/functions/iroco2-client-side-scanner/package/cur_scrapper.py

@@ -31,7 +31,7 @@ def __init__(self):
     def parsing_cur_and_send_to_api(self, bucket_name, s3_file_name):
         cur_file_type = os.path.splitext(s3_file_name)[1].replace(".", "")
         print(cur_file_type)
-        cur = self.s3_repository.read_file(bucket_name, s3_file_name)
+        cur = self.s3_repository.read_file(bucket_name, urllib.parse.unquote_plus(s3_file_name))
         message_parsed = []
         parsed_ec2_message = self.cur_processor_ec2_service.creating_message_from_cur(cur, cur_file_type)
         print(f'EC2 messages: {parsed_ec2_message}')

lambdas/functions/iroco2-client-side-scanner/s3.tf

@@ -71,18 +71,15 @@ resource "aws_s3_bucket_policy" "cur_output" {
         }
         Action = [
           "s3:GetBucketPolicy",
-          "s3:PutObject",
-          "s3:ListBucket"
+          "s3:PutObject"
         ]
         Resource = [
           "arn:aws:s3:::${var.cur_output_bucket_name}",
           "arn:aws:s3:::${var.cur_output_bucket_name}/*"
         ]
         Condition = {
-          StringEquals = {
-            "aws:SourceAccount" = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root",
-          },
           StringLike = {
+            "aws:SourceAccount" = data.aws_caller_identity.current.account_id,
             "aws:SourceArn" = [
               "arn:aws:bcm-data-exports:us-east-1:${data.aws_caller_identity.current.account_id}:export/*",
               "arn:aws:cur:us-east-1:${data.aws_caller_identity.current.account_id}:definition/*"
@@ -95,8 +92,8 @@ resource "aws_s3_bucket_policy" "cur_output" {
         Principal = {
           AWS = aws_iam_role.lambda_execution.arn
         }
-        NotAction = "s3:GetObject"
-        Resource  = "${aws_s3_bucket.cur_output.arn}/*"
+        NotAction = ["s3:GetObject", "s3:ListBucket"]
+        Resource  = ["${aws_s3_bucket.cur_output.arn}/*", aws_s3_bucket.cur_output.arn]
         Condition = {
           Bool = {
             "aws:SecureTransport" = "true"