feat(contact): optimize contact import and pagination performance

Gemfile.lock

@@ -1095,7 +1095,7 @@ DEPENDENCIES
   working_hours
 
 RUBY VERSION
-   ruby 3.4.4p34
+  ruby 3.4.4p34
 
 BUNDLED WITH
-   2.5.16
+  4.0.11

app/controllers/api/v1/contacts_controller.rb

@@ -35,6 +35,11 @@ class Api::V1::ContactsController < Api::V1::BaseController
   def index
     @contacts = fetch_contacts(listable_contacts)
 
+    # Use cached count to avoid expensive COUNT(*) queries on large datasets
+    @contacts_count = Rails.cache.fetch(cache_key_for_contacts_count, expires_in: 1.minute) do
+      listable_contacts.count
+    end
+
     apply_pagination
 
     paginated_response(
@@ -331,6 +336,18 @@ def pipelines
 
   private
 
+  # Cache key for contacts count, varies by query parameters that affect listable contacts
+  def cache_key_for_contacts_count
+    # Build a deterministic string based on filters that influence the count
+    key_parts = [
+      params[:type],
+      params[:company_id],
+      params[:labels]&.sort&.join(','),
+      params[:q] # search query, if any
+    ].compact.join('/')
+    "contacts_count/#{key_parts.presence || 'all'}"
+  end
+
   # TODO: Move this to a finder class
   def listable_contacts
     return @listable_contacts if @listable_contacts

app/jobs/data_import_job.rb

@@ -8,67 +8,175 @@ class DataImportJob < ApplicationJob
   def perform(data_import)
     @data_import = data_import
     @contact_manager = DataImport::ContactManager.new
+    Rails.logger.info "📊 DataImportJob: Starting import for data_import_id=#{@data_import.id}"
     begin
       process_import_file
       send_import_notification_to_admin
+      Rails.logger.info "📊 DataImportJob: Import completed for data_import_id=#{@data_import.id}"
     rescue CSV::MalformedCSVError => e
+      Rails.logger.error "📊 DataImportJob: CSV error for data_import_id=#{@data_import.id}: #{e.message}"
       handle_csv_error(e)
+    rescue => e
+      Rails.logger.error "📊 DataImportJob: Unexpected error for data_import_id=#{@data_import.id}: #{e.class} - #{e.message}"
+      Rails.logger.error e.backtrace.join("\n")
+      raise
     end
   end
 
   private
 
   def process_import_file
     @data_import.update!(status: :processing)
-    contacts, rejected_contacts = parse_csv_and_build_contacts
+    Rails.logger.info "📊 DataImportJob: Processing file for data_import_id=#{@data_import.id}"
 
-    import_contacts(contacts)
-    update_data_import_status(contacts.length, rejected_contacts.length)
+    # Separar contatos por tipo
+    person_rows, company_rows, rejected_contacts = parse_csv_and_classify_contacts
+    Rails.logger.info "📊 DataImportJob: Parsed CSV - persons: #{person_rows.length}, companies: #{company_rows.length}, rejected: #{rejected_contacts.length}"
+
+    # Primeiro importar empresas (necessário para vínculos)
+    import_companies(company_rows)
+
+    # Depois importar pessoas
+    import_persons(person_rows)
+
+    # Processar vínculos entre pessoas e empresas
+    process_company_linkages(person_rows)
+
+    total_imported = company_rows.length + person_rows.length
+    Rails.logger.info "📊 DataImportJob: Total to import: #{total_imported} (persons: #{person_rows.length}, companies: #{company_rows.length})"
+    update_data_import_status(total_imported, rejected_contacts.length)
     save_failed_records_csv(rejected_contacts)
   end
 
-  def parse_csv_and_build_contacts
-    contacts = []
+  def parse_csv_and_classify_contacts
+    person_rows = []
+    company_rows = []
     rejected_contacts = []
+
     # Ensuring that importing non utf-8 characters will not throw error
     data = @data_import.import_file.download
+    Rails.logger.info "📊 DataImportJob: Downloaded CSV file, size: #{data.size} bytes"
     utf8_data = data.force_encoding('UTF-8')
 
     # Ensure that the data is valid UTF-8, preserving valid characters
     clean_data = utf8_data.valid_encoding? ? utf8_data : utf8_data.encode('UTF-16le', invalid: :replace, replace: '').encode('UTF-8')
 
     csv = CSV.parse(clean_data, headers: true)
-
-    csv.each do |row|
-      current_contact = @contact_manager.build_contact(row.to_h.with_indifferent_access)
-      if current_contact.valid?
-        contacts << current_contact
+    Rails.logger.info "📊 DataImportJob: Parsed CSV, total rows: #{csv.count}"
+
+    csv.each_with_index do |row, index|
+      params = row.to_h.with_indifferent_access
+      tipo = params[:tipo] || params[:type] || 'person'
+
+      Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - tipo=#{tipo}, name=#{params[:nome] || params[:name]}, phone=#{params[:telefone] || params[:phone_number]}"
+
+      if tipo == 'company'
+        current_contact = @contact_manager.build_contact(params)
+        if current_contact.valid?
+          company_rows << { row: params, contact: current_contact }
+          Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - COMPANY valid"
+        else
+          append_rejected_contact(row, current_contact, rejected_contacts)
+          Rails.logger.warn "📊 DataImportJob: Row #{index + 1} - COMPANY invalid - errors: #{current_contact.errors.full_messages.join(', ')}"
+        end
       else
-        append_rejected_contact(row, current_contact, rejected_contacts)
+        current_contact = @contact_manager.build_contact(params)
+        if current_contact.valid?
+          person_rows << { row: params, contact: current_contact }
+          Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - PERSON valid"
+        else
+          append_rejected_contact(row, current_contact, rejected_contacts)
+          Rails.logger.warn "📊 DataImportJob: Row #{index + 1} - PERSON invalid - errors: #{current_contact.errors.full_messages.join(', ')}"
+        end
       end
     end
 
-    [contacts, rejected_contacts]
+    [person_rows, company_rows, rejected_contacts]
   end
 
   def append_rejected_contact(row, contact, rejected_contacts)
     row['errors'] = contact.errors.full_messages.join(', ')
     rejected_contacts << row
+    Rails.logger.debug "📊 DataImportJob: Rejected contact - name: #{row[:nome] || row[:name]}, errors: #{row['errors']}"
   end
 
   def import_contacts(contacts)
     # <struct ActiveRecord::Import::Result failed_instances=[], num_inserts=1, ids=[444, 445], results=[]>
-    Contact.import(contacts, synchronize: contacts, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000)
+    result = Contact.import(contacts, synchronize: contacts, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000)
+    Rails.logger.info "📊 DataImportJob: Import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}"
+    result
+  end
+
+  def import_companies(company_rows)
+    return if company_rows.empty?
+    Rails.logger.info "📊 DataImportJob: Importing #{company_rows.length} companies"
+
+    companies = company_rows.map { |row_data| row_data[:contact] }
+    result = Contact.import(companies, synchronize: companies, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000)
+    Rails.logger.info "📊 DataImportJob: Companies import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}"
+
+    result.failed_instances.each do |failed|
+      Rails.logger.warn "📊 DataImportJob: Failed company - #{failed.name} - errors: #{failed.errors.full_messages.join(', ')}"
+    end
+  end
+
+  def import_persons(person_rows)
+    return if person_rows.empty?
+    Rails.logger.info "📊 DataImportJob: Importing #{person_rows.length} persons"
+
+    persons = person_rows.map { |row_data| row_data[:contact] }
+    result = Contact.import(persons, synchronize: persons, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000)
+    Rails.logger.info "📊 DataImportJob: Persons import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}"
+
+    result.failed_instances.each do |failed|
+      Rails.logger.warn "📊 DataImportJob: Failed person - #{failed.name} - errors: #{failed.errors.full_messages.join(', ')}"
+    end
+  end
+
+  def process_company_linkages(person_rows)
+    Rails.logger.info "📊 DataImportJob: Processing company linkages for #{person_rows.length} persons"
+
+    person_rows.each do |person_data|
+      empresas_vinculadas = person_data[:row][:empresas_vinculadas]
+      next unless empresas_vinculadas.present?
+
+      contact = person_data[:contact]
+      company_names = empresas_vinculadas.split('|').map(&:strip)
+      Rails.logger.debug "📊 DataImportJob: Linking #{contact.name} to companies: #{company_names.join(', ')}"
+
+      company_names.each do |company_name|
+        company = Contact.companies.find_by("LOWER(name) = ?", company_name.downcase)
+        if company
+          contact.add_company(company)
+          Rails.logger.debug "📊 DataImportJob: Linked #{contact.name} to existing company: #{company.name}"
+        else
+          # Criar nova empresa se não existir
+          new_company = Contact.create!(
+            type: 'company',
+            name: company_name,
+            email: nil,
+            phone_number: nil
+          )
+          contact.add_company(new_company)
+          Rails.logger.info "📊 DataImportJob: Created new company and linked #{contact.name} to: #{company_name}"
+        end
+      end
+    end
   end
 
   def update_data_import_status(processed_records, rejected_records)
+    Rails.logger.info "📊 DataImportJob: Updating status - processed: #{processed_records}, rejected: #{rejected_records}"
     @data_import.update!(status: :completed, processed_records: processed_records, total_records: processed_records + rejected_records)
   end
 
   def save_failed_records_csv(rejected_contacts)
     csv_data = generate_csv_data(rejected_contacts)
-    return if csv_data.blank?
+    if csv_data.blank?
+      Rails.logger.info "📊 DataImportJob: No rejected contacts to save"
+      return
+    end
 
+    Rails.logger.info "📊 DataImportJob: Saving #{rejected_contacts.length} rejected contacts to CSV"
     @data_import.failed_records.attach(io: StringIO.new(csv_data), filename: "#{Time.zone.today.strftime('%Y%m%d')}_contacts.csv",
                                        content_type: 'text/csv')
     send_import_notification_to_admin
@@ -88,15 +196,18 @@ def generate_csv_data(rejected_contacts)
   end
 
   def handle_csv_error(error) # rubocop:disable Lint/UnusedMethodArgument
+    Rails.logger.error "📊 DataImportJob: Handling CSV error - marking as failed"
     @data_import.update!(status: :failed)
     send_import_failed_notification_to_admin
   end
 
   def send_import_notification_to_admin
+    Rails.logger.info "📊 DataImportJob: Sending import completion notification"
     AdministratorNotifications::AccountNotificationMailer.with(account: nil).contact_import_complete(@data_import).deliver_later
   end
 
   def send_import_failed_notification_to_admin
+    Rails.logger.info "📊 DataImportJob: Sending import failed notification"
     AdministratorNotifications::AccountNotificationMailer.with(account: nil).contact_import_failed.deliver_later
   end
 end

app/mailers/administrator_notifications/base_mailer.rb

@@ -22,7 +22,7 @@ def settings_url(section)
   private
 
   def admin_emails
-    User.where(role: 'administrator').pluck(:email)
+    User.joins(:roles).where(roles: { key: %w[account_owner administrator admin] }).pluck(:email)
   end
 
   def liquid_locals

app/models/contact.rb

@@ -195,9 +195,11 @@ def webhook_data
   def self.resolved_contacts
     # Include contacts that have email, phone_number, or identifier
     # Also include contacts that have contact_inboxes (have at least one conversation)
-    # This ensures Telegram and other social media contacts appear in the list
-    where(
-      "contacts.email <> '' OR contacts.phone_number <> '' OR contacts.identifier <> '' OR contacts.id IN (SELECT DISTINCT contact_id FROM contact_inboxes WHERE contact_id IS NOT NULL)"
+    # This uses LEFT JOIN for better performance
+    joins(
+      "LEFT JOIN contact_inboxes ON contact_inboxes.contact_id = contacts.id"
+    ).where(
+      "contacts.email <> '' OR contacts.phone_number <> '' OR contacts.identifier <> '' OR contact_inboxes.id IS NOT NULL"
     )
   end
 

app/models/role.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# Role model - synced from evo-auth-service
+# This model provides read-only access to roles managed by evo-auth-service
+class Role < ApplicationRecord
+  # Evolution Reference Model - managed by evo-auth-service
+  # This model serves only as a reference to sync data from evo-auth-service
+
+  self.table_name = 'roles'
+
+  # Read-only model - data is synced from evo-auth-service
+  has_many :user_roles, dependent: :destroy_async
+  has_many :users, through: :user_roles
+
+  validates :key, presence: true, uniqueness: true
+  validates :name, presence: true
+
+  # Check if this is an administrator role
+  def administrator?
+    key.in?(%w[account_owner administrator admin])
+  end
+
+  # Find administrator role
+  def self.administrator_role
+    find_by(key: %w[account_owner administrator admin])
+  end
+
+  # Find users with administrator roles
+  def self.administrator_users
+    Role.where(key: %w[account_owner administrator admin]).flat_map(&:users).uniq
+  end
+end

app/models/user.rb

@@ -63,6 +63,10 @@ class User < ApplicationRecord
   include Avatarable
   include UserAttributeHelpers
 
+  # Role relationships (synced from evo-auth-service)
+  has_many :user_roles, dependent: :destroy_async
+  has_many :roles, through: :user_roles
+
   # Evolution-specific relationships only
   has_many :assigned_conversations, foreign_key: 'assignee_id', class_name: 'Conversation', dependent: :nullify, inverse_of: :assignee
   has_many :csat_survey_responses, foreign_key: 'assigned_agent_id', dependent: :nullify, inverse_of: :assigned_agent

app/models/user_role.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# UserRole model - joins table for users and roles
+# This model provides read-only access to user_roles managed by evo-auth-service
+class UserRole < ApplicationRecord
+  # Evolution Reference Model - managed by evo-auth-service
+  # This model serves only as a reference to sync data from evo-auth-service
+
+  self.table_name = 'user_roles'
+
+  belongs_to :user
+  belongs_to :role
+  belongs_to :granted_by, class_name: 'User', optional: true
+
+  validates :user, presence: true
+  validates :role, presence: true
+end