Improve CurricleImporter to update existing Course records during import

Resolves: #16271

app/models/course.rb

@@ -9,6 +9,12 @@ class Course < ApplicationRecord
   has_many :users, through: :user_courses, inverse_of: :courses
   has_many :annotations, dependent: :destroy
 
+  validates :external_course_id, presence: true
+  validates :offer_number, presence: true
+  validates :term_code, presence: true
+  validates :session_code, presence: true
+  validates :class_section, presence: true
+
   searchable do
     integer :id
     integer :external_course_id

config/application.rb

@@ -25,5 +25,7 @@ class Application < Rails::Application
     config.autoload_paths += Dir.glob("#{config.root}/app/graphql/types/inputs")
 
     config.middleware.use BatchLoader::Middleware
+
+    config.active_record.schema_format = :sql
   end
 end

db/migrate/20190315151234_add_term_code_to_courses.rb

@@ -0,0 +1,5 @@
+class AddTermCodeToCourses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :courses, :term_code, :integer
+  end
+end

db/migrate/20190318142236_add_offer_number_to_courses.rb

@@ -0,0 +1,5 @@
+class AddOfferNumberToCourses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :courses, :offer_number, :integer
+  end
+end

db/migrate/20190318143427_add_session_code_to_courses.rb

@@ -0,0 +1,5 @@
+class AddSessionCodeToCourses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :courses, :session_code, :string
+  end
+end

db/migrate/20190318143539_add_composite_key_to_courses.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Create a unique composite key for courses
+class AddCompositeKeyToCourses < ActiveRecord::Migration[5.2]
+  def change
+    add_index(
+      :courses,
+      %i[external_course_id offer_number term_code session_code class_section],
+      unique: true,
+      name: 'by_unique_composite_key'
+    )
+  end
+end

db/migrate/20190318144611_add_constraints_to_courses.rb

@@ -0,0 +1,9 @@
+class AddConstraintsToCourses < ActiveRecord::Migration[5.2]
+  def change
+    change_column_null :courses, :external_course_id, false
+    change_column_null :courses, :offer_number, false
+    change_column_null :courses, :term_code, false
+    change_column_null :courses, :session_code, false
+    change_column_null :courses, :class_section, false
+  end
+end

db/seeds/course_instructors.seeds.rb

@@ -31,6 +31,8 @@ class CurricleCourseInstructorImporter < CurricleImporter
     SI
   ].freeze
 
+  UNIQUE_KEY_COLUMNS = ['id'].freeze
+
   def format_row(row) # rubocop:disable Metrics/MethodLength
     return unless row[:instructor_role].in?(VALID_INSTRUCTOR_ROLES)
 

db/seeds/course_meeting_patterns.seeds.rb

@@ -29,6 +29,8 @@ class CurricleCourseMeetingPatternImporter < CurricleImporter
     updated_at
   ].freeze
 
+  UNIQUE_KEY_COLUMNS = ['id'].freeze
+
   def format_row(row) # rubocop:disable Metrics/MethodLength
     external_course_id = row[:course_id].to_i
     term_year, term_name = row[:term_description].to_s.split(' ')

db/seeds/courses.seeds.rb

@@ -7,10 +7,13 @@ class CurricleCourseImporter < CurricleImporter
 
   SQL_COLUMNS = %w[
     external_course_id
+    offer_number
     course_description
     title
     term_name
     term_year
+    term_code
+    session_code
     academic_year
     class_section
     component
@@ -31,22 +34,35 @@ class CurricleCourseImporter < CurricleImporter
     updated_at
   ].freeze
 
+  UNIQUE_KEY_COLUMNS = %w[
+    external_course_id
+    offer_number
+    term_code
+    session_code
+    class_section
+  ].freeze
+
   def format_row(row) # rubocop:disable Metrics/MethodLength
     external_course_id = row[:course_id].to_i
     term_year, term_name = row[:term_description].to_s.split(' ')
     term_year = term_year.to_i
-    key = "#{term_year}#{term_name}#{external_course_id}#{row[:class_section]}"
 
-    return if COURSES_CACHE.key?(key)
+    # These fields are used in the composite key in the database and are required,
+    # skip the rows that have incomplete data
+    return if (row[:course_offer_number] && row[:session_code] && row[:class_section]).blank?
+
     return if row[:class_status] == 'X' # do not import courses that have been cancelled
     return if row[:class_type] == 'N' # do not import discussion sections
 
     [
       external_course_id,
+      row[:course_offer_number],
       row[:course_descr],
       row[:course_title_long],
       term_name,
-      term_year.to_i,
+      term_year,
+      row[:term_code],
+      row[:session_code],
       row[:academic_year],
       row[:class_section],
       row[:component_description],

db/seeds/curricle_importer.rb

@@ -17,16 +17,19 @@ class CurricleImporter
   def initialize
     @table_name = self.class::TABLE_NAME
     @sql_columns = self.class::SQL_COLUMNS
+    @unique_key_columns = self.class::UNIQUE_KEY_COLUMNS
     @csv_file = Rails.root.join('lib', 'seeds', "#{@table_name}.csv")
     @progressbar = progressbar(@csv_file)
     @pgconn = Course.connection.raw_connection
     @enco = PG::TextEncoder::CopyRow.new
   end
 
   def run
-    puts "Seeding #{@table_name}"
+    puts "Loading #{@table_name} into temporary #{@table_name}_temp table"
 
-    @pgconn.copy_data("COPY #{@table_name} (#{@sql_columns.join(',')}) FROM STDIN", @enco) do
+    @pgconn.exec("CREATE TEMP TABLE #{@table_name}_temp AS SELECT #{@sql_columns.join(',')} FROM #{@table_name} WITH NO DATA;")
+
+    @pgconn.copy_data("COPY #{@table_name}_temp (#{@sql_columns.join(',')}) FROM STDIN", @enco) do
       CSV.foreach(@csv_file, headers: true, header_converters: :symbol) do |row|
         formatted_row = format_row(row)
 
@@ -35,6 +38,20 @@ def run
         @progressbar.increment!
       end
     end
+
+    puts "Copying courses from temporary #{@table_name}_temp table into #{@table_name} table"
+
+    sql = "INSERT INTO #{@table_name} (#{@sql_columns.join(',')})
+      SELECT #{@sql_columns.join(',')}
+      FROM #{@table_name}_temp
+      ON CONFLICT (#{@unique_key_columns.join(',')})
+      DO UPDATE SET
+        (#{@sql_columns.join(',')}) =
+        (#{@sql_columns.map { |c| "EXCLUDED.#{c}" }.join(',')});
+
+      DISCARD TEMP;"
+
+    @pgconn.exec(sql)
   end
 
   private