Add rake task for bucket lifecycle config

Gemfile.lock

@@ -51,21 +51,21 @@ GEM
     ancestry (4.1.0)
       activerecord (>= 5.2.6)
     arel (9.0.0)
-    ast (2.4.2)
-    autoprefixer-rails (10.3.3.0)
-      execjs (~> 2)
+    ast (2.4.1)
+    autoprefixer-rails (10.0.1.0)
+      execjs
     aws-eventstream (1.2.0)
-    aws-partitions (1.510.0)
-    aws-sdk-core (3.121.1)
+    aws-partitions (1.519.0)
+    aws-sdk-core (3.121.3)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-kms (1.48.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-kms (1.50.0)
+      aws-sdk-core (~> 3, >= 3.121.2)
       aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.103.0)
-      aws-sdk-core (~> 3, >= 3.120.0)
+    aws-sdk-s3 (1.104.0)
+      aws-sdk-core (~> 3, >= 3.121.2)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.4)
     aws-sigv4 (1.4.0)

app/models/conditions_response/backup.rb

@@ -142,7 +142,7 @@ def self.s3_backup_bucket_full_prefix(today = Date.current)
   # @see https://aws.amazon.com/blogs/developer/uploading-files-to-amazon-s3/
   def self.upload_file_to_s3(s3, bucket, bucket_folder, file)
     obj = s3.bucket(bucket).object(bucket_folder + File.basename(file))
-    obj.upload_file(file, { tagging: aws_date_tags })
+    obj.upload_file(file, { tagging: aws_date_tags, storage_class: 'STANDARD_IA' })
   end
 
 

lib/tasks/shf_aws.rake

@@ -168,7 +168,91 @@ namespace :shf do
       tags_array = tags_string.split('&').map { |t| t.split('=') }
       tags_array.to_h.map { |k, v| { key: k, value: v } }
     end
-
+
+    desc "add lifecycle rules to s3 bucket"
+    task :bucket_lifecycle_config => :environment do 
+      # Not using arguments since we are aware before hand what transitions objects in a bucket make. See https://github.com/AgileVentures/shf-project/wiki/Backups-on-AWS
+      # We can make this task dynamic if the need arises.
+      # This simplifies our rake task syntax in the terminal.
+      aws_s3 = Backup.s3_backup_resource
+      aws_client = aws_s3.client
+      aws_s3_backup_bucket_name = Backup.s3_backup_bucket # Remember to revert to production bucket
+      aws_s3_backup_bucket_full_prefix = Backup.s3_backup_bucket_full_prefix
+
+      ActivityLogger.open(LogfileNamer.name_for(LOGFILENAME), LOG_FACILITY, 'Add bucket lifecycle configuration') do |log|
+        aws_client.put_bucket_lifecycle_configuration({
+          bucket: aws_s3_backup_bucket_name, 
+          lifecycle_configuration: {
+            rules: [
+              {
+                expiration: {
+                  days: 60
+                },
+                filter: {
+                  prefix: aws_s3_backup_bucket_full_prefix
+                }, 
+                id: 'Daily backup schedule',
+                status: 'Enabled', 
+                transitions: [
+                  {
+                    days: 30, 
+                    storage_class: 'STANDARD_IA'
+                  }
+                ]
+              },
+              {
+                expiration: {
+                  days: 366
+                }, 
+                id: 'Monthly backup schedule', 
+                filter: {
+                  tag: {
+                    key: 'date-month-day',
+                    value: '1'
+                  }
+                }, 
+                status: 'Enabled', 
+                transitions: [
+                  {
+                    days: 90, 
+                    storage_class: 'GLACIER'
+                  }
+                ]
+              }, 
+              {
+                expiration: {
+                  days: 3650
+                },
+                id: 'Yearly backup schedule',
+                filter: {
+                  and: {
+                    tags: [
+                      {
+                        key: 'date-month-day',
+                        value: '1'
+                      },
+                      {
+                        key: 'date-month-num',
+                        value: '1'
+                      }
+                    ]
+                  }
+                }, 
+                status: 'Enabled',
+                transitions: [
+                  {
+                    days: 366,
+                    storage_class: 'DEEP_ARCHIVE'
+                  }
+                ]
+              }
+            ]
+          }
+        })
+
+        log.info("Lifecycle configuration added for #{aws_s3_backup_bucket_name}")
+      end
+    end
 
     # Note that this is in a different region than the current production backups bucket.
     def aws_test_bucket_resource
@@ -199,7 +283,6 @@ namespace :shf do
       obj = s3_resource.bucket(bucketname).object(prefix + File.basename(file))
       obj.upload_file(file.path, { tagging: date_tags, storage_class: storage_class })
     end
-
   end
 end
 

spec/lib/tasks/shf_aws_spec.rb

@@ -0,0 +1,32 @@
+require 'rails_helper'
+
+Rails.application.load_tasks
+
+RSpec.describe 'shf::aws rake tasks', type: :task do
+  describe 'copy old backups to new prefix names' do
+    let(:first_date) {'2021-07-01'}
+    let(:last_date) {'2021-08-02'}
+
+    it 'it copies old backups to new prefix names' do
+      expect(Rake::Task['shf:aws:copy_to_new_prefixes']).to receive(:invoke).with(first_date, last_date)
+      Rake::Task['shf:aws:copy_to_new_prefixes'].invoke(first_date, last_date)
+    end
+  end
+
+  describe 'apply date tags to prefixed backups' do
+    let(:first_date) {'2021-07-01'}
+    let(:last_date) {'2021-08-02'}
+
+    it 'it applies date tags to prefixed backups' do
+      expect(Rake::Task['shf:aws:apply_date_tags']).to receive(:invoke).with(first_date, last_date)
+      Rake::Task['shf:aws:apply_date_tags'].invoke(first_date, last_date)
+    end
+  end
+
+  describe 'add lifecycle rules to s3 bucket' do
+    it 'it adds lifecycle config to s3 bucket' do
+      expect(Rake::Task['shf:aws:bucket_lifecycle_config']).to receive(:invoke)
+      Rake::Task['shf:aws:bucket_lifecycle_config'].invoke
+    end
+  end
+end

spec/models/conditions_response/backup_spec.rb

@@ -700,10 +700,10 @@ def create_faux_backup_file(backups_dir, file_prefix)
         FileUtils.remove_entry(temp_backups_dir, true)
       end
 
-      it 'adds date tags to the object' do
+      it 'adds date tags and STANDARD_IA storage class to the object' do
         expect(mock_bucket_object).to receive(:upload_file)
                                        .with(faux_backup_fn,
-                                             {tagging: 'this is the tagging string'})
+                                        {storage_class: 'STANDARD_IA', tagging: 'this is the tagging string'})
 
         expect(described_class).to receive(:aws_date_tags).and_return('this is the tagging string')
         Backup.upload_file_to_s3(mock_s3, bucket_name, bucket_full_prefix, faux_backup_fn)