Release 166

CHANGELOG.md

@@ -4,6 +4,14 @@
 
 [Full changelog][unreleased]
 
+## Release 166 - 2025-02-05
+
+[Full changelog][166]
+
+- Anonymise users background job
+- Devise two-factor-auth 4.x -> 5.x cleanup
+- Export Level B activities fixes (part one)
+
 ## Release 165 – 2025-02-03
 
 [Full changelog][165]
@@ -1799,7 +1807,8 @@
 - Planned start and end dates are mandatory
 - Actual start and end dates must not be in the future
 
-[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-165...HEAD
+[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-166...HEAD
+[166]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-165...release-166
 [165]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-164...release-165
 [164]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-163...release-164
 [163]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-162...release-163

Gemfile

@@ -42,6 +42,7 @@ gem "wicked"
 gem "strip_attributes"
 gem "breadcrumbs_on_rails"
 gem "sprockets-rails"
+gem "sidekiq-scheduler"
 
 # Authentication
 gem "devise"

Gemfile.lock

@@ -165,6 +165,8 @@ GEM
       railties (>= 6.1)
     erubi (1.13.1)
     erubis (2.7.0)
+    et-orbi (1.2.11)
+      tzinfo
     factory_bot (6.5.0)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.4.4)
@@ -178,6 +180,9 @@ GEM
     ffi (1.17.0-x86_64-darwin)
     ffi (1.17.0-x86_64-linux)
     foreman (0.88.1)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     govuk_design_system_formbuilder (5.8.0)
@@ -328,6 +333,7 @@ GEM
       rspec-expectations (~> 3.12)
       rspec-mocks (~> 3.12)
       rspec-support (~> 3.12)
+    raabro (1.4.0)
     racc (1.8.1)
     rack (2.2.10)
     rack-attack (6.7.0)
@@ -451,6 +457,8 @@ GEM
     ruby_parser (3.19.1)
       sexp_processor (~> 4.16)
     rubyzip (2.4.1)
+    rufus-scheduler (3.9.2)
+      fugit (~> 1.1, >= 1.11.1)
     selenium-webdriver (4.28.0)
       base64 (~> 0.2)
       logger (~> 1.4)
@@ -465,6 +473,10 @@ GEM
       logger
       rack (>= 2.2.4)
       redis-client (>= 0.22.2)
+    sidekiq-scheduler (5.0.6)
+      rufus-scheduler (~> 3.2)
+      sidekiq (>= 6, < 8)
+      tilt (>= 1.4.0, < 3)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -612,6 +624,7 @@ DEPENDENCIES
   selenium-webdriver
   shoulda-matchers
   sidekiq (~> 7)
+  sidekiq-scheduler
   simplecov (~> 0.22.0)
   simplecov-lcov (~> 0.8.0)
   sprockets-rails

app/jobs/anonymise_deactivated_users_job.rb

@@ -0,0 +1,9 @@
+class AnonymiseDeactivatedUsersJob
+  include Sidekiq::Job
+
+  def perform
+    User.deactivated.where("deactivated_at < ?", 5.years.ago).each do |user|
+      AnonymiseUser.new(user:).call
+    end
+  end
+end

app/models/user.rb

@@ -1,6 +1,5 @@
 class User < ApplicationRecord
-  devise :two_factor_authenticatable, :rememberable, :secure_validatable, :recoverable,
-    otp_secret_encryption_key: ENV["SECRET_KEY_BASE"]
+  devise :two_factor_authenticatable, :rememberable, :secure_validatable, :recoverable
 
   belongs_to :organisation
   has_and_belongs_to_many :additional_organisations, class_name: "Organisation", join_table: "organisations_users"
@@ -76,78 +75,4 @@ def email_cannot_be_changed_after_create
       errors.add(:email, :cannot_be_changed)
     end
   end
-
-  # :nocov:
-  ##
-  # Decrypt and return the `encrypted_otp_secret` attribute which was used in
-  # versions of devise-two-factor < 5.x. In practice this will be in use for the
-  # gap between deployment of 5.x and the running of
-  # db/data/20250117151047_regenerate_otp_secrets.rb, and will be removed in
-  # the very next release. Lifted from
-  # https://github.com/devise-two-factor/devise-two-factor/blob/main/UPGRADING.md
-  # @return [String] The decrypted OTP secret
-  def legacy_otp_secret
-    return nil unless self[:encrypted_otp_secret]
-    return nil unless self.class.otp_secret_encryption_key
-
-    hmac_iterations = 2000 # a default set by the Encryptor gem
-    key = self.class.otp_secret_encryption_key
-    salt = Base64.decode64(encrypted_otp_secret_salt)
-    iv = Base64.decode64(encrypted_otp_secret_iv)
-
-    raw_cipher_text = Base64.decode64(encrypted_otp_secret)
-    # The last 16 bytes of the ciphertext are the authentication tag - we use
-    # Galois Counter Mode which is an authenticated encryption mode
-    cipher_text = raw_cipher_text[0..-17]
-    auth_tag = raw_cipher_text[-16..-1] # standard:disable Style/SlicingWithRange
-
-    # this algorithm lifted from
-    # https://github.com/attr-encrypted/encryptor/blob/master/lib/encryptor.rb#L54
-
-    # create an OpenSSL object which will decrypt the AES cipher with 256 bit
-    # keys in Galois Counter Mode (GCM). See
-    # https://ruby.github.io/openssl/OpenSSL/Cipher.html
-    cipher = OpenSSL::Cipher.new("aes-256-gcm")
-
-    # tell the cipher we want to decrypt. Symmetric algorithms use a very
-    # similar process for encryption and decryption, hence the same object can
-    # do both.
-    cipher.decrypt
-
-    # Use a Password-Based Key Derivation Function to generate the key actually
-    # used for encryption from the key we got as input.
-    cipher.key = OpenSSL::PKCS5.pbkdf2_hmac_sha1(key, salt, hmac_iterations, cipher.key_len)
-
-    # set the Initialization Vector (IV)
-    cipher.iv = iv
-
-    # The tag must be set after calling Cipher#decrypt, Cipher#key= and
-    # Cipher#iv=, but before calling Cipher#final. After all decryption is
-    # performed, the tag is verified automatically in the call to Cipher#final.
-    #
-    # If the auth_tag does not verify, then #final will raise OpenSSL::Cipher::CipherError
-    cipher.auth_tag = auth_tag
-
-    # auth_data must be set after auth_tag has been set when decrypting See
-    # http://ruby-doc.org/stdlib-2.0.0/libdoc/openssl/rdoc/OpenSSL/Cipher.html#method-i-auth_data-3D
-    # we are not adding any authenticated data but OpenSSL docs say this should
-    # still be called.
-    cipher.auth_data = ""
-
-    # #update is (somewhat confusingly named) the method which actually
-    # performs the decryption on the given chunk of data. Our OTP secret is
-    # short so we only need to call it once.
-    #
-    # It is very important that we call #final because:
-    #
-    # 1. The authentication tag is checked during the call to #final
-    # 2. Block based cipher modes (e.g. CBC) work on fixed size chunks. We need
-    #    to call #final to get it to process the last chunk properly. The output
-    #    of #final should be appended to the decrypted value. This isn't
-    #    required for streaming cipher modes but including it is a best practice
-    #    so that your code will continue to function correctly even if you later
-    #    change to a block cipher mode.
-    cipher.update(cipher_text) + cipher.final
-  end
-  # :nocov:
 end

app/services/export/activities_level_b.rb

@@ -1,48 +1,91 @@
 class Export::ActivitiesLevelB
-  HEADERS = [
-    "Partner Organisation",
-    "Activity level",
-    "Parent activity",
-    "ODA or Non-ODA",
-    "Partner organisation identifier",
-    "RODA identifier",
-    "IATI identifier",
-    "Linked activity",
-    "Activity title",
-    "Activity description",
-    "Aims or objectives",
-    "Sector",
-    "Original commitment figure",
-    "Activity status",
-    "Planned start date",
-    "Planned end date",
-    "Actual start date",
-    "Actual end date",
-    "ISPF ODA partner countries",
-    "Benefitting countries",
-    "Benefitting region",
-    "Global Development Impact",
-    "Sustainable Development Goals",
-    "ISPF themes",
-    "Aid type",
-    "ODA eligibility",
-    "Publish to IATI?",
-    "Tags",
-    "Budget 2023-2024",
-    "Budget 2024-2025",
-    "Budget 2025-2026",
-    "Budget 2026-2027",
-    "Budget 2027-2028",
-    "Budget 2028-2029",
-    "Comments"
+  Field = Data.define(:name, :fund, :value)
+
+  # A place to:
+  #   - Name all the fields on the left
+  #   - filter applicable fields on a per-fund basis in the middle
+  #   - evaluate a row's values in the context of a fund's activity via a `value` Proc on the right
+  #   - show all this on a line-by-line basis to avoid one tall export per fund, given
+  #     there are many common fields
+  # standard:disable Layout/ExtraSpacing
+  FIELDS = [
+    Field.new("Partner Organisation",                      "ALL",  -> { activity.organisation.name }),
+    Field.new("Activity level",                            "ALL",  -> { activity.level }),
+    Field.new("Parent activity",                           "ALL",  -> { activity.source_fund.name }),
+    Field.new("ODA or Non-ODA",                            "ISPF", -> { activity.is_oda }),
+    Field.new("Partner organisation identifier",           "ALL",  -> { activity.partner_organisation_identifier }),
+    Field.new("RODA identifier",                           "ALL",  -> { activity.roda_identifier }),
+    Field.new("IATI identifier",                           "ALL",  -> { activity.transparency_identifier }),
+    Field.new("Linked activity",                           "ALL",  -> { activity.linked_activity_identifier }),
+    Field.new("Activity title",                            "ALL",  -> { activity.title }),
+    Field.new("Activity description",                      "ALL",  -> { activity.description }),
+    Field.new("Aims or objectives",                        "ALL",  -> { activity.objectives }),
+    Field.new("Sector",                                    "ALL",  -> { activity.sector }),
+    Field.new("Original commitment figure",                "ALL",  -> { activity.commitment&.value }),
+    Field.new("Activity status",                           "ALL",  -> { activity.programme_status }),
+    Field.new("Planned start date",                        "ALL",  -> { activity.planned_start_date }),
+    Field.new("Planned end date",                          "ALL",  -> { activity.planned_end_date }),
+    Field.new("Actual start date",                         "ALL",  -> { activity.actual_start_date }),
+    Field.new("Actual end date",                           "ALL",  -> { activity.actual_end_date }),
+    Field.new("ISPF ODA partner countries",                "ISPF", -> { activity.ispf_oda_partner_countries }),
+    Field.new("ISPF non-ODA partner countries",            "ISPF", -> { activity.ispf_non_oda_partner_countries }),
+    Field.new("GCRF Strategic Area",                       "GCRF", -> { activity.gcrf_strategic_area }),
+    Field.new("GCRF Challenge Area",                       "GCRF", -> { activity.gcrf_challenge_area }),
+    Field.new("Newton Fund Country Partner Organisations", "NF",   -> { activity.country_partner_organisations }),
+    Field.new("Newton Fund Pillar",                        "NF",   -> { activity.fund_pillar }),
+    Field.new("Benefitting countries",                     "ALL",  -> { activity.benefitting_countries }),
+    Field.new("Benefitting region",                        "ALL",  -> { activity.benefitting_region }),
+    Field.new("Global Development Impact",                 "ALL",  -> { activity.gdi }),
+    Field.new("Sustainable Development Goals",             "ALL",  -> { activity.sustainable_development_goals }),
+    Field.new("ISPF themes",                               "ISPF", -> { activity.ispf_themes }),
+    Field.new("Aid type",                                  "ALL",  -> { activity.aid_type }),
+    Field.new("ODA eligibility",                           "ALL",  -> { activity.oda_eligibility }),
+    Field.new("Publish to IATI?",                          "ALL",  -> { activity.publish_to_iati }),
+    Field.new("Tags",                                      "ISPF", -> { activity.tags }),
+    Field.new("Budget 2023-2024",                          "ALL",  -> { budgets_by_year[2023]&.value }),
+    Field.new("Budget 2024-2025",                          "ALL",  -> { budgets_by_year[2024]&.value }),
+    Field.new("Budget 2025-2026",                          "ALL",  -> { budgets_by_year[2025]&.value }),
+    Field.new("Budget 2026-2027",                          "ALL",  -> { budgets_by_year[2026]&.value }),
+    Field.new("Budget 2027-2028",                          "ALL",  -> { budgets_by_year[2027]&.value }),
+    Field.new("Budget 2028-2029",                          "ALL",  -> { budgets_by_year[2028]&.value }),
+    Field.new("Comments",                                  "ALL",  -> { activity.comments.map(&:body).join("|") })
   ].freeze
+  # standard:enable Layout/ExtraSpacing
+
+  # Given a fund and an activity (or nil for a header row), return all the cell values in a row
+  # via #to_a
+  Row = Struct.new(:fund, :activity) do
+    def budgets_by_year
+      @budgets_by_year ||= activity.budgets.each_with_object({}) do |budget, years|
+        years[budget.financial_year.start_year] = BudgetPresenter.new(budget)
+      end
+    end
+
+    def to_a
+      return applicable_fields.map(&:name) if header?
+
+      applicable_fields.map do |field|
+        instance_exec(&field.value) # get the field's value from its Proc in the context of this Row
+      end
+    end
+
+    private
+
+    def header?
+      activity.nil?
+    end
+
+    def applicable_fields
+      FIELDS.select { |field| field.fund.in? ["ALL", fund.short_name] }
+    end
+  end
 
   def initialize(fund:)
     @fund = fund
   end
 
   def headers
-    HEADERS
+    Row.new(fund: @fund, activity: nil).to_a
   end
 
   def filename
@@ -51,58 +94,14 @@ def filename
 
   def rows
     activities.map do |activity|
-      row_for(activity)
+      Row.new(fund: @fund, activity: ActivityCsvPresenter.new(activity)).to_a
     end
   end
 
   private
 
-  def row_for(activity)
-    activity = ActivityCsvPresenter.new(activity)
-    budgets_by_year = activity.budgets.each_with_object({}) do |budget, hash|
-      hash[budget.financial_year.start_year] = BudgetPresenter.new(budget)
-    end
-    [
-      activity.organisation.name,               # "Partner Organisation",
-      activity.level,                           # "Activity level",
-      @fund.name,                               # "Parent activity",
-      activity.is_oda,                          # "ODA or Non-ODA",
-      activity.partner_organisation_identifier, # "Partner organisation identifier",
-      activity.roda_identifier,                 # "RODA identifier", e.g. GCRF-LCXHF
-      activity.organisation.iati_reference,     # "IATI identifier",
-      activity.linked_activity_identifier,      # "Linked activity",
-      activity.title,                           # "Activity title",
-      activity.description,                     # "Activity description",
-      activity.objectives,                      # "Aims or objectives",
-      activity.sector,                          # "Sector",
-      activity.commitment&.value,               # "Original commitment figure",
-      activity.programme_status,                # "Activity status",
-      activity.planned_start_date,              # "Planned start date",
-      activity.planned_end_date,                # "Planned end date",
-      activity.actual_start_date,               # "Actual start date",
-      activity.actual_end_date,                 # "Actual end date",
-      activity.ispf_oda_partner_countries,      # "ISPF ODA partner countries",
-      activity.benefitting_countries,           # "Benefitting countries",
-      activity.benefitting_region,              # "Benefitting region",
-      activity.gdi,                             # "Global Development Impact",
-      activity.sustainable_development_goals,   # "Sustainable Development Goals",
-      activity.ispf_themes,                     # "ISPF themes",
-      activity.aid_type,                        # "Aid type",
-      activity.oda_eligibility,                 # "ODA eligibility",
-      activity.publish_to_iati,                 # "Publish to IATI?",
-      activity.tags,                            # "Tags",
-      budgets_by_year[2023]&.value,             # "Budget 2023-2024",
-      budgets_by_year[2024]&.value,             # "Budget 2024-2025",
-      budgets_by_year[2025]&.value,             # "Budget 2025-2026",
-      budgets_by_year[2026]&.value,             # "Budget 2026-2027",
-      budgets_by_year[2027]&.value,             # "Budget 2027-2028",
-      budgets_by_year[2028]&.value,             # "Budget 2028-2029",
-      activity.comments.map(&:body).join("\n")  # "Comments"
-    ]
-  end
-
   def activities
     @activities ||= @fund.activity.child_activities
-      .includes(:organisation, :linked_activity, :commitment, :budgets, :comments)
+      .includes(:organisation, :commitment, :budgets, :linked_activity, :comments)
   end
 end

config/environments/test.rb

@@ -74,6 +74,7 @@
     Bullet.add_safelist type: :unused_eager_loading, class_name: "User", association: :organisation
     Bullet.add_safelist type: :unused_eager_loading, class_name: "Activity", association: :organisation
     Bullet.add_safelist type: :unused_eager_loading, class_name: "Activity", association: :child_activities
+    Bullet.add_safelist type: :unused_eager_loading, class_name: "Activity", association: :linked_activity
     Bullet.add_safelist type: :unused_eager_loading, class_name: "Transaction", association: :provider
     Bullet.add_safelist type: :unused_eager_loading, class_name: "Transaction", association: :receiver
     Bullet.add_safelist type: :unused_eager_loading, class_name: "Activity", association: :parent

config/routes.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq/web"
+require "sidekiq-scheduler/web"
 
 Rails.application.routes.draw do
   devise_scope :user do

config/sidekiq.yml

@@ -2,3 +2,8 @@
 :queues:
   - default
   - mailers
+:scheduler:
+  :schedule:
+    anonymise_deactivated_users:
+      cron: '0 0 * * 0' # Every Sunday at midnight
+      class: AnonymiseDeactivatedUsersJob