[SPIKE] s3 lifecycle rules for backups

app/models/conditions_response/backup.rb

@@ -86,6 +86,8 @@ def self.condition_response(condition, log, use_slack_notification: true)
 
     iterate_and_log_notify_errors(backup_files, 'in backup_files loop, uploading_file_to_s3', log) do |backup_file|
       upload_file_to_s3(aws_s3, aws_s3_backup_bucket, aws_backup_bucket_full_prefix, backup_file)
+      # When we first upload our file to s3, the default storage class is STANDARD_IA
+      set_s3_lifecycle_rules(bucket_name: aws_s3_backup_bucket, bucket_full_prefix: aws_backup_bucket_full_prefix, status: 'enabled', storage_rules: [{days: 90, storage_class: 'GLACIER'}, {days: 450, storage_class: 'DEEP_ARCHIVE'}])
     end
 
     log.record('info', 'Pruning older backups on local storage')
@@ -142,7 +144,7 @@ def self.s3_backup_bucket_full_prefix(today = Date.current)
   # @see https://aws.amazon.com/blogs/developer/uploading-files-to-amazon-s3/
   def self.upload_file_to_s3(s3, bucket, bucket_folder, file)
     obj = s3.bucket(bucket).object(bucket_folder + File.basename(file))
-    obj.upload_file(file, { tagging: aws_date_tags })
+    obj.upload_file(file, { tagging: aws_date_tags, storage_class: 'STANDARD_IA' })
   end
 
 
@@ -287,6 +289,48 @@ def self.iterate_and_log_notify_errors(list, additional_error_info, log, use_sla
     end
   end
 
+
+  STORAGE_CLASSES = %w(GLACIER DEEP_ARCHIVE).freeze
+  class << self
+    define_method(:storage_class_is_valid?) do |storage_class_list| 
+      unless storage_class_list.empty?
+        (storage_class_list + STORAGE_CLASSES).uniq.count <= STORAGE_CLASSES.count ? true : "Invalid storage class"
+      else
+        "Empty storage class"
+      end
+    end
+  end
+
+  # s3_lifecycle_rules(bucket_name: 'bucket_name', bucket_full_prefix: 'bucket_full_prefix', status: 'enabled', storage_rules: [{days: 90, storage_class: 'GLACIER'}, {days: 450, storage_class: 'DEEP_ARCHIVE'}])
+  def self.set_s3_lifecycle_rules(bucket_name:, bucket_full_prefix:, status:, storage_rules:)
+    client = Aws::S3::Client.new(region: ENV['SHF_AWS_S3_BACKUP_REGION'], 
+      credentials: Aws::Credentials.new(ENV['SHF_AWS_S3_BACKUP_KEY_ID'], ENV['SHF_AWS_S3_BACKUP_SECRET_ACCESS_KEY']))
+
+    storage_class_list = storage_rules.flatten.map{|h| h.values.last}
+    unless storage_class_is_valid? storage_class_list
+      client.put_bucket_lifecycle_configuration({
+        bucket: bucket_name, 
+        lifecycle_configuration: {
+          rules: [
+            {
+              expiration: {
+                # Expire objects after 10 years
+                date: Time.now,
+                days: 3650, 
+                expired_object_delete_marker: false
+              },
+              filter: {
+                prefix: bucket_full_prefix
+              }, 
+              id: ENV['SHF_AWS_S3_BACKUP_KEY_ID'], 
+              status: status.capitalize,
+              transitions: storage_rules
+            }
+          ]
+        }
+      })  
+    end
+  end
 
   # Record the error and additional_info to the given log
   # and send a Slack notification if we are using Slack notifications

spec/models/conditions_response/backup_spec.rb

@@ -692,18 +692,17 @@ def create_faux_backup_file(backups_dir, file_prefix)
       let!(:temp_backups_dir) { Dir.mktmpdir('faux-backups-dir') }
       let!(:faux_backup_fn) { create_faux_backup_file(temp_backups_dir, 'faux_backup.bak') }
 
-
       it '.upload_file_to_s3 calls .upload_file for the bucket, full object name, and file to upload' do
         expect(mock_bucket_object).to receive(:upload_file).with(faux_backup_fn, anything)
         Backup.upload_file_to_s3(mock_s3, bucket_name, bucket_full_prefix, faux_backup_fn)
 
         FileUtils.remove_entry(temp_backups_dir, true)
       end
 
-      it 'adds date tags to the object' do
+      it 'adds date tags and STANDARD_IA storage class to the object' do
         expect(mock_bucket_object).to receive(:upload_file)
                                        .with(faux_backup_fn,
-                                             {tagging: 'this is the tagging string'})
+                                        {storage_class: 'STANDARD_IA', tagging: 'this is the tagging string'})
 
         expect(described_class).to receive(:aws_date_tags).and_return('this is the tagging string')
         Backup.upload_file_to_s3(mock_s3, bucket_name, bucket_full_prefix, faux_backup_fn)
@@ -1251,7 +1250,9 @@ def create_faux_backup_file(backups_dir, file_prefix)
 
 
     describe 'iterate_and_log_notify_errors(list, slack_error_details, log)' do
-
+      let(:status) { 'Enabled' }
+      let(:storage_rules) { [{days: 30, storage_class: 'STANDARD_IA'}, {days: 90, storage_class: 'GLACIER'}] }
+
       before(:each) do
         allow(SHFNotifySlack).to receive(:failure_notification)
                                      .with(anything, anything)
@@ -1303,6 +1304,78 @@ def create_faux_backup_file(backups_dir, file_prefix)
         expect(@result_str).to eq 'ac'
       end
 
+      it 'adds a bucket lifecycle policy to the object' do
+        expect(described_class).to receive(:set_s3_lifecycle_rules).with(bucket_name: bucket_name, bucket_full_prefix: bucket_full_prefix, status: status, storage_rules: storage_rules)
+        described_class.set_s3_lifecycle_rules(bucket_name: bucket_name, bucket_full_prefix: bucket_full_prefix, status: status, storage_rules: storage_rules)
+      end
+
+    end
+
+    describe 'set_s3_lifecycle_rules(bucket, bucket_full_prefix, status, *storage_rules_kwargs)' do
+      let(:invalid_storage_class_list) { ['INVALID_STORAGE_CLASS', 'OTHER_INVALID_STORAGE_CLASS'] }
+      let(:another_invalid_storage_class_list) { ['INVALID_STORAGE_CLASS', 'STANDARD_IA', 'GLACIER'] }
+      let(:status) { 'Enabled' }
+      let(:storage_rules) { [{days: 30, storage_class: 'STANDARD_IA'}, {days: 90, storage_class: 'GLACIER'}] }
+      let(:stub_s3_client) { double('Aws::S3::Client', bucket: mock_bucket) }
+
+      let(:mock_s3_client) do
+        client = Aws::S3::Client.new(stub_responses: true)
+        client.stub_responses(
+          :put_bucket_lifecycle_configuration, ->(context) {
+            bucket = context.params[:bucket]
+            lifecycle_configuration = context.params[:lifecycle_configuration][:rules]
+          }
+        )
+        client
+      end
+
+      it 'calls #set_s3_lifecycle_rules once' do
+        expect(described_class).to receive(:set_s3_lifecycle_rules).with(bucket_name: bucket_name, bucket_full_prefix: bucket_full_prefix, status: status, storage_rules: storage_rules)
+        described_class.set_s3_lifecycle_rules(bucket_name: bucket_name, bucket_full_prefix: bucket_full_prefix, status: status, storage_rules: storage_rules)
+      end
+
+      it "returns 'Invalid storage class' for a list containing only invalid storage classes" do
+        expect(described_class.storage_class_is_valid? invalid_storage_class_list).to eq("Invalid storage class")
+      end
+
+      it "returns 'Invalid storage class' for a list containing both valid and invalid storage classes" do
+        expect(described_class.storage_class_is_valid? another_invalid_storage_class_list).to eq("Invalid storage class")
+      end
+
+      it "returns 'Empty storage class' for empty storage classes list" do
+        expect(described_class.storage_class_is_valid? []).to eq("Empty storage class")
+      end
+
+      it 'returns the correct lifecycle rules transitions' do
+        put_mock_data = mock_s3_client.put_bucket_lifecycle_configuration(
+          bucket: bucket_name,
+          lifecycle_configuration: {
+            rules: [
+              {
+                expiration: {
+                  days: 365
+                }, 
+                filter: {
+                  prefix: bucket_full_prefix
+                }, 
+                id: 'TestOnly', 
+                status: status, 
+                transitions: storage_rules
+              }
+            ]
+          }
+        )
+
+        allow(stub_s3_client).to receive(:get_bucket_lifecycle_configuration).with({bucket: bucket_name}).and_return(put_mock_data)
+        get_mock_data = stub_s3_client.get_bucket_lifecycle_configuration(bucket: bucket_name)
+
+        expect(get_mock_data[0][:id]).to eq 'TestOnly'
+        expect(get_mock_data[0][:status]).to eq 'Enabled'
+        expect(get_mock_data[0][:filter][:prefix]).to eq 'bucket/top/prefix'
+        expect(get_mock_data[0][:expiration][:days]).to eq 365
+        expect(get_mock_data[0][:transitions].count).to eq 2
+        expect(get_mock_data[0][:transitions]).to eq [{days: 30, storage_class: 'STANDARD_IA'}, {days: 90, storage_class: 'GLACIER'}]
+      end
     end
   end