From d0c2fbc0ef4090e82c45bceefa5b0a89b9ea5fad Mon Sep 17 00:00:00 2001 From: Luiz Paulo Coutinho Date: Mon, 4 May 2026 08:31:38 -0300 Subject: [PATCH 1/2] feat: expand contact import with full field support - Add support for person/company type differentiation - Add processing for social profiles (linkedin, facebook, instagram, twitter, github) - Add location attributes (city, country, country_code) - Add custom attributes auto-mapping for unknown fields - Add empresas_vinculadas field for linking persons to companies - Import companies first, then persons, then process linkages - Updated CSV sample with all 19 supported fields --- app/jobs/data_import_job.rb | 85 +++++++++++++++--- app/services/data_import/contact_manager.rb | 95 +++++++++++++++++++-- public/downloads/import-contacts-sample.csv | 32 ++----- 3 files changed, 171 insertions(+), 41 deletions(-) diff --git a/app/jobs/data_import_job.rb b/app/jobs/data_import_job.rb index d281779..30db714 100644 --- a/app/jobs/data_import_job.rb +++ b/app/jobs/data_import_job.rb @@ -20,16 +20,29 @@ def perform(data_import) def process_import_file @data_import.update!(status: :processing) - contacts, rejected_contacts = parse_csv_and_build_contacts - import_contacts(contacts) - update_data_import_status(contacts.length, rejected_contacts.length) + # Separar contatos por tipo + person_rows, company_rows, rejected_contacts = parse_csv_and_classify_contacts + + # Primeiro importar empresas (necessário para vínculos) + import_companies(company_rows) + + # Depois importar pessoas + import_persons(person_rows) + + # Processar vínculos entre pessoas e empresas + process_company_linkages(person_rows) + + total_imported = company_rows.length + person_rows.length + update_data_import_status(total_imported, rejected_contacts.length) save_failed_records_csv(rejected_contacts) end - def parse_csv_and_build_contacts - contacts = [] + def parse_csv_and_classify_contacts + person_rows = [] + company_rows = [] rejected_contacts = [] + # Ensuring that importing non utf-8 characters will not throw error data = @data_import.import_file.download utf8_data = data.force_encoding('UTF-8') @@ -40,15 +53,27 @@ def parse_csv_and_build_contacts csv = CSV.parse(clean_data, headers: true) csv.each do |row| - current_contact = @contact_manager.build_contact(row.to_h.with_indifferent_access) - if current_contact.valid? - contacts << current_contact + params = row.to_h.with_indifferent_access + tipo = params[:tipo] || params[:type] || 'person' + + if tipo == 'company' + current_contact = @contact_manager.build_contact(params) + if current_contact.valid? + company_rows << { row: params, contact: current_contact } + else + append_rejected_contact(row, current_contact, rejected_contacts) + end else - append_rejected_contact(row, current_contact, rejected_contacts) + current_contact = @contact_manager.build_contact(params) + if current_contact.valid? + person_rows << { row: params, contact: current_contact } + else + append_rejected_contact(row, current_contact, rejected_contacts) + end end end - [contacts, rejected_contacts] + [person_rows, company_rows, rejected_contacts] end def append_rejected_contact(row, contact, rejected_contacts) @@ -61,6 +86,46 @@ def import_contacts(contacts) Contact.import(contacts, synchronize: contacts, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) end + def import_companies(company_rows) + return if company_rows.empty? + + companies = company_rows.map { |row_data| row_data[:contact] } + Contact.import(companies, synchronize: companies, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + end + + def import_persons(person_rows) + return if person_rows.empty? + + persons = person_rows.map { |row_data| row_data[:contact] } + Contact.import(persons, synchronize: persons, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + end + + def process_company_linkages(person_rows) + person_rows.each do |person_data| + empresas_vinculadas = person_data[:row][:empresas_vinculadas] + next unless empresas_vinculadas.present? + + contact = person_data[:contact] + company_names = empresas_vinculadas.split('|').map(&:strip) + + company_names.each do |company_name| + company = Contact.companies.find_by("LOWER(name) = ?", company_name.downcase) + if company + contact.add_company(company) + else + # Criar nova empresa se não existir + new_company = Contact.create!( + type: 'company', + name: company_name, + email: nil, + phone_number: nil + ) + contact.add_company(new_company) + end + end + end + end + def update_data_import_status(processed_records, rejected_records) @data_import.update!(status: :completed, processed_records: processed_records, total_records: processed_records + rejected_records) end diff --git a/app/services/data_import/contact_manager.rb b/app/services/data_import/contact_manager.rb index a0bed78..f5c9daf 100644 --- a/app/services/data_import/contact_manager.rb +++ b/app/services/data_import/contact_manager.rb @@ -12,6 +12,7 @@ def find_or_initialize_contact(params) contact = find_existing_contact(params) contact_params = params.slice(:email, :identifier, :phone_number) contact_params[:phone_number] = format_phone_number(contact_params[:phone_number]) if contact_params[:phone_number].present? + contact_params[:type] = params[:tipo] || params[:type] || 'person' contact ||= Contact.new(contact_params) contact end @@ -20,6 +21,7 @@ def find_existing_contact(params) contact = find_contact_by_identifier(params) contact ||= find_contact_by_email(params) contact ||= find_contact_by_phone_number(params) + contact ||= find_contact_by_tax_id(params) update_contact_with_merged_attributes(params, contact) if contact.present? && contact.valid? contact @@ -43,8 +45,18 @@ def find_contact_by_phone_number(params) Contact.find_by(phone_number: format_phone_number(params[:phone_number])) end + def find_contact_by_tax_id(params) + tax_id = params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id] + return unless tax_id.present? + + Contact.find_by(tax_id: tax_id) + end + def format_phone_number(phone_number) - phone_number.start_with?('+') ? phone_number : "+#{phone_number}" + return unless phone_number.present? + + cleaned = phone_number.to_s.gsub(/\D/, '') + cleaned.start_with?('+') ? cleaned : "+#{cleaned}" end def update_contact_with_merged_attributes(params, contact) @@ -58,10 +70,83 @@ def update_contact_with_merged_attributes(params, contact) private def update_contact_attributes(params, contact) - contact.name = params[:name] if params[:name].present? + # Tipo de contato + contact.type = params[:tipo] || params[:type] || 'person' + + # Campos base + contact.name = build_name(params) + contact.last_name = params[:sobrenome] if params[:sobrenome].present? + contact.email = params[:email] if params[:email].present? + contact.phone_number = format_phone_number(params[:telefone] || params[:phone_number]) if (params[:telefone] || params[:phone_number]).present? + contact.tax_id = params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id] + + # Campos específicos + contact.website = params[:website] if params[:website].present? + contact.industry = params[:segmento_industria] || params[:industry] if (params[:segmento_industria] || params[:industry]).present? + + # Localização + process_location_attributes(params, contact) + + # Redes sociais + process_social_attributes(params, contact) + + # Descrição + process_description(params, contact) + + # Atributos customizados (campos não reconhecidos) + process_custom_attributes(params, contact) + + contact + end + + def build_name(params) + # Para pessoa física: primeiro_nome + if params[:tipo] == 'person' || params[:type] == 'person' + first_name = params[:primeiro_nome] || params[:first_name] || params[:nome] || params[:name] + last_name = params[:sobrenome] || params[:last_name] + return "#{first_name} #{last_name}".strip if first_name.present? + end + + # Para empresa: razao_social ou name + params[:razao_social] || params[:nome] || params[:name] || '' + end + + def process_location_attributes(params, contact) + contact.additional_attributes ||= {} + contact.additional_attributes[:city] = params[:cidade] if params[:cidade].present? + contact.additional_attributes[:country] = params[:pais] if params[:pais].present? + contact.country_code = params[:codigo_pais] if params[:codigo_pais].present? + end + + def process_social_attributes(params, contact) + contact.additional_attributes ||= {} + contact.additional_attributes[:social_profiles] ||= {} + + social_fields = %w[linkedin facebook instagram twitter github] + social_fields.each do |field| + contact.additional_attributes[:social_profiles][field] = params[field.to_sym] if params[field.to_sym].present? + end + end + + def process_description(params, contact) contact.additional_attributes ||= {} - contact.additional_attributes[:company] = params[:company] if params[:company].present? - contact.additional_attributes[:city] = params[:city] if params[:city].present? - contact.assign_attributes(custom_attributes: contact.custom_attributes.merge(params.except(:identifier, :email, :name, :phone_number))) + contact.additional_attributes[:description] = params[:descricao] if params[:descricao].present? + end + + def process_custom_attributes(params, contact) + contact.custom_attributes ||= {} + + # Campos conhecidos que NÃO são custom_attributes + known_fields = %i[ + id tipo type name nome primeiro_nome first_name sobrenome last_name email telefone phone_number + cpf_cnpj cpf cnpj tax_id website segmento_industria industry cidade pais codigo_pais + linkedin facebook instagram twitter github descricao empresas_vinculadas company + identifier ip_address custom_attribute_1 custom_attribute_2 + ] + + # Qualquer outro campo vira custom_attribute + params.except(*known_fields).each do |key, value| + contact.custom_attributes[key] = value if value.present? + end end end diff --git a/public/downloads/import-contacts-sample.csv b/public/downloads/import-contacts-sample.csv index a11edbc..a16559d 100644 --- a/public/downloads/import-contacts-sample.csv +++ b/public/downloads/import-contacts-sample.csv @@ -1,26 +1,6 @@ -id,name,email,identifier,phone_number,ip_address,custom_attribute_1,custom_attribute_2 -1,Clarice Uzzell,cuzzell0@mozilla.org,bb4e11cd-0f23-49da-a123-dcc1fec6852c,+498963648018,70.61.11.201,Random-value-1,Random-value-1 -2,Marieann Creegan,mcreegan1@cornell.edu,e60bab4c-9fbb-47eb-8f75-42025b789c47,+15417543010,168.186.4.241,Random-value0,Random-value0 -3,Nancey Windibank,nwindibank2@bluehost.com,f793e813-4210-4bf3-a812-711418de25d2,+15417543011,73.44.41.59,Random-value1,Random-value1 -4,Sibel Stennine,sstennine3@yellowbook.com,d6e35a2d-d093-4437-a577-7df76316b937,+15417543011,115.249.27.155,Random-value2,Random-value2 -5,Tina O'Lunney,tolunney4@si.edu,3540d40a-5567-4f28-af98-5583a7ddbc56,+15417543011,219.181.212.8,Random-value3,Random-value3 -6,Quinn Neve,qneve5@army.mil,ba0e1bf0-c74b-41ce-8a2d-0b08fa0e5aa5,+15417543011,231.210.115.166,Random-value4,Random-value4 -7,Karylin Gaunson,kgaunson6@tripod.com,d24cac79-c81b-4b84-a33e-0441b7c6a981,+15417543011,160.189.41.11,Random-value5,Random-value5 -8,Jamison Shenton,jshenton7@upenn.edu,29a7a8c0-c7f7-4af9-852f-761b1a784a7a,+15417543011,53.94.18.201,Random-value6,Random-value6 -9,Gavan Threlfall,gthrelfall8@spotify.com,847d4943-ddb5-47cc-8008-ed5092c675c5,+15417543011,18.87.247.249,Random-value7,Random-value7 -10,Katina Hemmingway,khemmingway9@ameblo.jp,8f0b5efd-b6a8-4f1e-a1e3-b0ea8c9e3048,+15417543011,25.191.96.124,Random-value8,Random-value8 -11,Jillian Deinhard,jdeinharda@canalblog.com,bd952787-1b05-411f-9975-b916ec0950cc,+15417543011,11.211.174.93,Random-value9,Random-value9 -12,Blake Finden,bfindenb@wsj.com,12c95613-e49d-4fa2-86fb-deabb6ebe600,+15417543011,47.26.205.153,Random-value10,Random-value10 -13,Liane Maxworthy,lmaxworthyc@un.org,36b68e4c-40d6-4e09-bf59-7db3b27b18f0,+15417543011,157.196.34.166,Random-value11,Random-value11 -14,Martynne Ledley,mledleyd@sourceforge.net,1856bceb-cb36-415c-8ffc-0527f3f750d8,+15417543011,109.231.152.148,Random-value12,Random-value12 -15,Katharina Ruffli,krufflie@huffingtonpost.com,604de5c9-b154-4279-8978-41fb71f0f773,+15417543011,20.43.146.179,Random-value13,Random-value13 -16,Tucker Simmance,tsimmancef@bbc.co.uk,0a8fc3a7-4986-4a51-a503-6c7f974c90ad,+15417543011,179.76.226.171,Random-value14,Random-value14 -17,Wenona Martinson,wmartinsong@census.gov,0e5ea6e3-6824-4e78-a6f5-672847eafa17,+15417543011,92.243.194.160,Random-value15,Random-value15 -18,Gretna Vedyasov,gvedyasovh@lycos.com,6becf55b-a7b5-48f6-8788-b89cae85b066,+15417543011,25.22.86.101,Random-value16,Random-value16 -19,Lurline Abdon,labdoni@archive.org,afa9429f-9034-4b06-9efa-980e01906ebf,+15417543011,150.249.116.118,Random-value17,Random-value17 -20,Fiann Norcliff,fnorcliffj@istockphoto.com,59f72dec-14ba-4d6e-b17c-0d962e69ffac,+15417543011,237.167.197.197,Random-value18,Random-value18 -21,Zed Linn,zlinnk@phoca.cz,95f7bc56-be92-4c9c-ad58-eff3e63c7bea,+15417543011,88.102.64.113,Random-value19,Random-value19 -22,Averyl Simyson,asimysonl@livejournal.com,bde1fe59-c9bd-440c-bb39-79fe61dac1d1,+15417543011,141.248.89.29,Random-value20,Random-value20 -23,Camella Blackadder,cblackadderm@nifty.com,0c981752-5857-487c-b9b5-5d0253df740a,+15417543011,118.123.138.115,Random-value21,Random-value21 -24,Aurie Spatig,aspatign@printfriendly.com,4cf22bfb-2c3f-41d1-9993-6e3758e457ba,+15417543011,157.45.102.235,Random-value22,Random-value22 -25,Adrienne Bellard,abellardo@cnn.com,f10f9b8d-38ac-4e17-8a7d-d2e6a055f944,+15417543011,170.73.198.47,Random-value23,Random-value23 \ No newline at end of file +tipo,nome,primeiro_nome,sobrenome,email,telefone,cpf_cnpj,website,segmento_industria,cidade,pais,codigo_pais,linkedin,facebook,instagram,twitter,github,descricao,empresas_vinculadas +person,João Silva,João,Silva,joao@example.com,+5511999999999,12345678901,,,São Paulo,Brasil,BR,https://linkedin.com/in/joaosilva,,,https://github.com/joaosilva,Engenheiro de Software,Empresa X|Outra Empresa +company,Empresa X,,,,+5511888888888,12345678000190,https://empresa-x.com,Tecnologia,São Paulo,Brasil,BR,https://linkedin.com/company/empresax,,,,,Empresa de tecnologia,, +company,Tech Solutions Ltda,Empresa,,,,,+5511877777777,12345678000191,https://tech.com,Tecnologia,Rio de Janeiro,Brasil,BR,,,,,,Consultoria,, +person,Maria Santos,Maria,Santos,maria@example.com,+5511977777777,12345678902,,,São Paulo,Brasil,BR,,,,,Gerente de Projetos,Empresa X +person,Pedro Costa,Pedro,Costa,pedro@example.com,+5511966666666,12345678903,,,Campinas,Brasil,BR,https://linkedin.com/in/pedrocosta,,,,https://github.com/pedrocosta,Desenvolvedor Fullstack,Empresa X|Tech Solutions Ltda From 85bf60143cc192c0efad167551b48a4613dddaa3 Mon Sep 17 00:00:00 2001 From: Luiz Paulo Coutinho Date: Mon, 4 May 2026 17:04:29 -0300 Subject: [PATCH 2/2] feat(contact): optimize contacts import and pagination --- Gemfile.lock | 4 +- app/controllers/api/v1/contacts_controller.rb | 17 +++++ app/jobs/data_import_job.rb | 56 +++++++++++++++-- .../base_mailer.rb | 2 +- app/models/contact.rb | 8 ++- app/models/role.rb | 32 ++++++++++ app/models/user.rb | 4 ++ app/models/user_role.rb | 17 +++++ app/services/data_import/contact_manager.rb | 8 ++- ...020000100_optimize_contacts_performance.rb | 30 +++++++++ db/schema.rb | 63 +++---------------- docker/entrypoints/rails.sh | 12 ++-- public/downloads/import-contacts-sample.csv | 12 ++-- 13 files changed, 182 insertions(+), 83 deletions(-) create mode 100644 app/models/role.rb create mode 100644 app/models/user_role.rb create mode 100644 db/migrate/20241020000100_optimize_contacts_performance.rb diff --git a/Gemfile.lock b/Gemfile.lock index dd7f9c1..fb427e4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1095,7 +1095,7 @@ DEPENDENCIES working_hours RUBY VERSION - ruby 3.4.4p34 + ruby 3.4.4p34 BUNDLED WITH - 2.5.16 + 4.0.11 diff --git a/app/controllers/api/v1/contacts_controller.rb b/app/controllers/api/v1/contacts_controller.rb index 1aa55ab..2270c04 100644 --- a/app/controllers/api/v1/contacts_controller.rb +++ b/app/controllers/api/v1/contacts_controller.rb @@ -35,6 +35,11 @@ class Api::V1::ContactsController < Api::V1::BaseController def index @contacts = fetch_contacts(listable_contacts) + # Use cached count to avoid expensive COUNT(*) queries on large datasets + @contacts_count = Rails.cache.fetch(cache_key_for_contacts_count, expires_in: 1.minute) do + listable_contacts.count + end + apply_pagination paginated_response( @@ -331,6 +336,18 @@ def pipelines private + # Cache key for contacts count, varies by query parameters that affect listable contacts + def cache_key_for_contacts_count + # Build a deterministic string based on filters that influence the count + key_parts = [ + params[:type], + params[:company_id], + params[:labels]&.sort&.join(','), + params[:q] # search query, if any + ].compact.join('/') + "contacts_count/#{key_parts.presence || 'all'}" + end + # TODO: Move this to a finder class def listable_contacts return @listable_contacts if @listable_contacts diff --git a/app/jobs/data_import_job.rb b/app/jobs/data_import_job.rb index 30db714..79919c9 100644 --- a/app/jobs/data_import_job.rb +++ b/app/jobs/data_import_job.rb @@ -8,11 +8,18 @@ class DataImportJob < ApplicationJob def perform(data_import) @data_import = data_import @contact_manager = DataImport::ContactManager.new + Rails.logger.info "📊 DataImportJob: Starting import for data_import_id=#{@data_import.id}" begin process_import_file send_import_notification_to_admin + Rails.logger.info "📊 DataImportJob: Import completed for data_import_id=#{@data_import.id}" rescue CSV::MalformedCSVError => e + Rails.logger.error "📊 DataImportJob: CSV error for data_import_id=#{@data_import.id}: #{e.message}" handle_csv_error(e) + rescue => e + Rails.logger.error "📊 DataImportJob: Unexpected error for data_import_id=#{@data_import.id}: #{e.class} - #{e.message}" + Rails.logger.error e.backtrace.join("\n") + raise end end @@ -20,9 +27,11 @@ def perform(data_import) def process_import_file @data_import.update!(status: :processing) + Rails.logger.info "📊 DataImportJob: Processing file for data_import_id=#{@data_import.id}" # Separar contatos por tipo person_rows, company_rows, rejected_contacts = parse_csv_and_classify_contacts + Rails.logger.info "📊 DataImportJob: Parsed CSV - persons: #{person_rows.length}, companies: #{company_rows.length}, rejected: #{rejected_contacts.length}" # Primeiro importar empresas (necessário para vínculos) import_companies(company_rows) @@ -34,6 +43,7 @@ def process_import_file process_company_linkages(person_rows) total_imported = company_rows.length + person_rows.length + Rails.logger.info "📊 DataImportJob: Total to import: #{total_imported} (persons: #{person_rows.length}, companies: #{company_rows.length})" update_data_import_status(total_imported, rejected_contacts.length) save_failed_records_csv(rejected_contacts) end @@ -45,30 +55,38 @@ def parse_csv_and_classify_contacts # Ensuring that importing non utf-8 characters will not throw error data = @data_import.import_file.download + Rails.logger.info "📊 DataImportJob: Downloaded CSV file, size: #{data.size} bytes" utf8_data = data.force_encoding('UTF-8') # Ensure that the data is valid UTF-8, preserving valid characters clean_data = utf8_data.valid_encoding? ? utf8_data : utf8_data.encode('UTF-16le', invalid: :replace, replace: '').encode('UTF-8') csv = CSV.parse(clean_data, headers: true) + Rails.logger.info "📊 DataImportJob: Parsed CSV, total rows: #{csv.count}" - csv.each do |row| + csv.each_with_index do |row, index| params = row.to_h.with_indifferent_access tipo = params[:tipo] || params[:type] || 'person' + Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - tipo=#{tipo}, name=#{params[:nome] || params[:name]}, phone=#{params[:telefone] || params[:phone_number]}" + if tipo == 'company' current_contact = @contact_manager.build_contact(params) if current_contact.valid? company_rows << { row: params, contact: current_contact } + Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - COMPANY valid" else append_rejected_contact(row, current_contact, rejected_contacts) + Rails.logger.warn "📊 DataImportJob: Row #{index + 1} - COMPANY invalid - errors: #{current_contact.errors.full_messages.join(', ')}" end else current_contact = @contact_manager.build_contact(params) if current_contact.valid? person_rows << { row: params, contact: current_contact } + Rails.logger.debug "📊 DataImportJob: Row #{index + 1} - PERSON valid" else append_rejected_contact(row, current_contact, rejected_contacts) + Rails.logger.warn "📊 DataImportJob: Row #{index + 1} - PERSON invalid - errors: #{current_contact.errors.full_messages.join(', ')}" end end end @@ -79,39 +97,58 @@ def parse_csv_and_classify_contacts def append_rejected_contact(row, contact, rejected_contacts) row['errors'] = contact.errors.full_messages.join(', ') rejected_contacts << row + Rails.logger.debug "📊 DataImportJob: Rejected contact - name: #{row[:nome] || row[:name]}, errors: #{row['errors']}" end def import_contacts(contacts) # - Contact.import(contacts, synchronize: contacts, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + result = Contact.import(contacts, synchronize: contacts, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + Rails.logger.info "📊 DataImportJob: Import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}" + result end def import_companies(company_rows) return if company_rows.empty? + Rails.logger.info "📊 DataImportJob: Importing #{company_rows.length} companies" companies = company_rows.map { |row_data| row_data[:contact] } - Contact.import(companies, synchronize: companies, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + result = Contact.import(companies, synchronize: companies, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + Rails.logger.info "📊 DataImportJob: Companies import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}" + + result.failed_instances.each do |failed| + Rails.logger.warn "📊 DataImportJob: Failed company - #{failed.name} - errors: #{failed.errors.full_messages.join(', ')}" + end end def import_persons(person_rows) return if person_rows.empty? + Rails.logger.info "📊 DataImportJob: Importing #{person_rows.length} persons" persons = person_rows.map { |row_data| row_data[:contact] } - Contact.import(persons, synchronize: persons, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + result = Contact.import(persons, synchronize: persons, on_duplicate_key_ignore: true, track_validation_failures: true, validate: true, batch_size: 1000) + Rails.logger.info "📊 DataImportJob: Persons import result - num_inserts: #{result.num_inserts}, failed_instances: #{result.failed_instances.size}" + + result.failed_instances.each do |failed| + Rails.logger.warn "📊 DataImportJob: Failed person - #{failed.name} - errors: #{failed.errors.full_messages.join(', ')}" + end end def process_company_linkages(person_rows) + Rails.logger.info "📊 DataImportJob: Processing company linkages for #{person_rows.length} persons" + person_rows.each do |person_data| empresas_vinculadas = person_data[:row][:empresas_vinculadas] next unless empresas_vinculadas.present? contact = person_data[:contact] company_names = empresas_vinculadas.split('|').map(&:strip) + Rails.logger.debug "📊 DataImportJob: Linking #{contact.name} to companies: #{company_names.join(', ')}" company_names.each do |company_name| company = Contact.companies.find_by("LOWER(name) = ?", company_name.downcase) if company contact.add_company(company) + Rails.logger.debug "📊 DataImportJob: Linked #{contact.name} to existing company: #{company.name}" else # Criar nova empresa se não existir new_company = Contact.create!( @@ -121,19 +158,25 @@ def process_company_linkages(person_rows) phone_number: nil ) contact.add_company(new_company) + Rails.logger.info "📊 DataImportJob: Created new company and linked #{contact.name} to: #{company_name}" end end end end def update_data_import_status(processed_records, rejected_records) + Rails.logger.info "📊 DataImportJob: Updating status - processed: #{processed_records}, rejected: #{rejected_records}" @data_import.update!(status: :completed, processed_records: processed_records, total_records: processed_records + rejected_records) end def save_failed_records_csv(rejected_contacts) csv_data = generate_csv_data(rejected_contacts) - return if csv_data.blank? + if csv_data.blank? + Rails.logger.info "📊 DataImportJob: No rejected contacts to save" + return + end + Rails.logger.info "📊 DataImportJob: Saving #{rejected_contacts.length} rejected contacts to CSV" @data_import.failed_records.attach(io: StringIO.new(csv_data), filename: "#{Time.zone.today.strftime('%Y%m%d')}_contacts.csv", content_type: 'text/csv') send_import_notification_to_admin @@ -153,15 +196,18 @@ def generate_csv_data(rejected_contacts) end def handle_csv_error(error) # rubocop:disable Lint/UnusedMethodArgument + Rails.logger.error "📊 DataImportJob: Handling CSV error - marking as failed" @data_import.update!(status: :failed) send_import_failed_notification_to_admin end def send_import_notification_to_admin + Rails.logger.info "📊 DataImportJob: Sending import completion notification" AdministratorNotifications::AccountNotificationMailer.with(account: nil).contact_import_complete(@data_import).deliver_later end def send_import_failed_notification_to_admin + Rails.logger.info "📊 DataImportJob: Sending import failed notification" AdministratorNotifications::AccountNotificationMailer.with(account: nil).contact_import_failed.deliver_later end end diff --git a/app/mailers/administrator_notifications/base_mailer.rb b/app/mailers/administrator_notifications/base_mailer.rb index 08800a3..705cc0a 100644 --- a/app/mailers/administrator_notifications/base_mailer.rb +++ b/app/mailers/administrator_notifications/base_mailer.rb @@ -22,7 +22,7 @@ def settings_url(section) private def admin_emails - User.where(role: 'administrator').pluck(:email) + User.joins(:roles).where(roles: { key: %w[account_owner administrator admin] }).pluck(:email) end def liquid_locals diff --git a/app/models/contact.rb b/app/models/contact.rb index 997daf9..ba108f4 100644 --- a/app/models/contact.rb +++ b/app/models/contact.rb @@ -195,9 +195,11 @@ def webhook_data def self.resolved_contacts # Include contacts that have email, phone_number, or identifier # Also include contacts that have contact_inboxes (have at least one conversation) - # This ensures Telegram and other social media contacts appear in the list - where( - "contacts.email <> '' OR contacts.phone_number <> '' OR contacts.identifier <> '' OR contacts.id IN (SELECT DISTINCT contact_id FROM contact_inboxes WHERE contact_id IS NOT NULL)" + # This uses LEFT JOIN for better performance + joins( + "LEFT JOIN contact_inboxes ON contact_inboxes.contact_id = contacts.id" + ).where( + "contacts.email <> '' OR contacts.phone_number <> '' OR contacts.identifier <> '' OR contact_inboxes.id IS NOT NULL" ) end diff --git a/app/models/role.rb b/app/models/role.rb new file mode 100644 index 0000000..473a99b --- /dev/null +++ b/app/models/role.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Role model - synced from evo-auth-service +# This model provides read-only access to roles managed by evo-auth-service +class Role < ApplicationRecord + # Evolution Reference Model - managed by evo-auth-service + # This model serves only as a reference to sync data from evo-auth-service + + self.table_name = 'roles' + + # Read-only model - data is synced from evo-auth-service + has_many :user_roles, dependent: :destroy_async + has_many :users, through: :user_roles + + validates :key, presence: true, uniqueness: true + validates :name, presence: true + + # Check if this is an administrator role + def administrator? + key.in?(%w[account_owner administrator admin]) + end + + # Find administrator role + def self.administrator_role + find_by(key: %w[account_owner administrator admin]) + end + + # Find users with administrator roles + def self.administrator_users + Role.where(key: %w[account_owner administrator admin]).flat_map(&:users).uniq + end +end diff --git a/app/models/user.rb b/app/models/user.rb index bc4d2a0..4d735f8 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -63,6 +63,10 @@ class User < ApplicationRecord include Avatarable include UserAttributeHelpers + # Role relationships (synced from evo-auth-service) + has_many :user_roles, dependent: :destroy_async + has_many :roles, through: :user_roles + # Evolution-specific relationships only has_many :assigned_conversations, foreign_key: 'assignee_id', class_name: 'Conversation', dependent: :nullify, inverse_of: :assignee has_many :csat_survey_responses, foreign_key: 'assigned_agent_id', dependent: :nullify, inverse_of: :assigned_agent diff --git a/app/models/user_role.rb b/app/models/user_role.rb new file mode 100644 index 0000000..823312f --- /dev/null +++ b/app/models/user_role.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# UserRole model - joins table for users and roles +# This model provides read-only access to user_roles managed by evo-auth-service +class UserRole < ApplicationRecord + # Evolution Reference Model - managed by evo-auth-service + # This model serves only as a reference to sync data from evo-auth-service + + self.table_name = 'user_roles' + + belongs_to :user + belongs_to :role + belongs_to :granted_by, class_name: 'User', optional: true + + validates :user, presence: true + validates :role, presence: true +end diff --git a/app/services/data_import/contact_manager.rb b/app/services/data_import/contact_manager.rb index f5c9daf..9501d48 100644 --- a/app/services/data_import/contact_manager.rb +++ b/app/services/data_import/contact_manager.rb @@ -46,7 +46,7 @@ def find_contact_by_phone_number(params) end def find_contact_by_tax_id(params) - tax_id = params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id] + tax_id = sanitize_tax_id(params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id]) return unless tax_id.present? Contact.find_by(tax_id: tax_id) @@ -59,6 +59,10 @@ def format_phone_number(phone_number) cleaned.start_with?('+') ? cleaned : "+#{cleaned}" end + def sanitize_tax_id(tax_id) + tax_id.to_s.gsub(/\D/, '') if tax_id.present? + end + def update_contact_with_merged_attributes(params, contact) contact.identifier = params[:identifier] if params[:identifier].present? contact.email = params[:email] if params[:email].present? @@ -78,7 +82,7 @@ def update_contact_attributes(params, contact) contact.last_name = params[:sobrenome] if params[:sobrenome].present? contact.email = params[:email] if params[:email].present? contact.phone_number = format_phone_number(params[:telefone] || params[:phone_number]) if (params[:telefone] || params[:phone_number]).present? - contact.tax_id = params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id] + contact.tax_id = sanitize_tax_id(params[:cpf_cnpj] || params[:cpf] || params[:cnpj] || params[:tax_id]) # Campos específicos contact.website = params[:website] if params[:website].present? diff --git a/db/migrate/20241020000100_optimize_contacts_performance.rb b/db/migrate/20241020000100_optimize_contacts_performance.rb new file mode 100644 index 0000000..7238157 --- /dev/null +++ b/db/migrate/20241020000100_optimize_contacts_performance.rb @@ -0,0 +1,30 @@ +class OptimizeContactsPerformance < ActiveRecord::Migration[7.0] + def up + # Index to speed up the subquery used in Contact.resolved_contacts + execute <<-SQL + CREATE INDEX idx_contact_inboxes_contact_id + ON contact_inboxes (contact_id) + WHERE contact_id IS NOT NULL; + SQL + + # Partial index for contacts that have at least one identifier (email, phone, or identifier) + execute <<-SQL + CREATE INDEX idx_contacts_with_identity + ON contacts (id) + WHERE (email <> '' OR phone_number <> '' OR identifier <> ''); + SQL + + # Composite index for common ordering and filtering by name/type on resolved contacts + execute <<-SQL + CREATE INDEX idx_contacts_name_type_resolved + ON contacts (name, type, id) + WHERE (email <> '' OR phone_number <> '' OR identifier <> ''); + SQL + end + + def down + remove_index :contact_inboxes, name: :idx_contact_inboxes_contact_id rescue nil + remove_index :contacts, name: :idx_contacts_with_identity rescue nil + remove_index :contacts, name: :idx_contacts_name_type_resolved rescue nil + end +end diff --git a/db/schema.rb b/db/schema.rb index 1e37de6..30fd1a3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -327,6 +327,7 @@ t.string "pubsub_token" t.text "bsuid" t.text "whatsapp_username" + t.index ["contact_id"], name: "idx_contact_inboxes_contact_id", where: "(contact_id IS NOT NULL)" t.index ["contact_id"], name: "index_contact_inboxes_on_contact_id" t.index ["inbox_id", "bsuid"], name: "index_contact_inboxes_on_inbox_id_and_bsuid", unique: true, where: "(bsuid IS NOT NULL)" t.index ["inbox_id", "source_id"], name: "index_contact_inboxes_on_inbox_id_and_source_id", unique: true @@ -357,9 +358,11 @@ t.string "industry" t.index ["blocked"], name: "index_contacts_on_blocked" t.index ["email"], name: "uniq_email_per_account_contact", unique: true + t.index ["id"], name: "idx_contacts_with_identity", where: "(((email)::text <> ''::text) OR ((phone_number)::text <> ''::text) OR ((identifier)::text <> ''::text))" t.index ["identifier"], name: "uniq_identifier_per_account_contact", unique: true t.index ["last_activity_at"], name: "index_contacts_on_last_activity_at", order: "DESC NULLS LAST" t.index ["name", "email", "phone_number", "identifier"], name: "index_contacts_on_name_email_phone_number_identifier", opclass: :gin_trgm_ops, using: :gin + t.index ["name", "type", "id"], name: "idx_contacts_name_type_resolved", where: "(((email)::text <> ''::text) OR ((phone_number)::text <> ''::text) OR ((identifier)::text <> ''::text))" t.index ["phone_number"], name: "index_contacts_on_phone_number" t.index ["tax_id"], name: "index_contacts_on_tax_id", unique: true, where: "(tax_id IS NOT NULL)" t.index ["type"], name: "index_contacts_on_type" @@ -500,31 +503,6 @@ t.index ["user_id"], name: "index_data_privacy_consents_on_user_id" end - create_table "events", primary_key: ["id", "app_name", "user_id", "session_id"], force: :cascade do |t| - t.string "id", limit: 128, null: false - t.string "app_name", limit: 128, null: false - t.string "user_id", limit: 128, null: false - t.string "session_id", limit: 128, null: false - t.string "invocation_id", limit: 256, null: false - t.string "author", limit: 256, null: false - t.binary "actions", null: false - t.text "long_running_tool_ids_json" - t.string "branch", limit: 256 - t.datetime "timestamp", precision: nil, null: false - t.jsonb "content" - t.jsonb "grounding_metadata" - t.jsonb "custom_metadata" - t.jsonb "usage_metadata" - t.jsonb "citation_metadata" - t.boolean "partial" - t.boolean "turn_complete" - t.string "error_code", limit: 256 - t.string "error_message", limit: 1024 - t.boolean "interrupted" - t.jsonb "input_transcription" - t.jsonb "output_transcription" - end - create_table "evo_agent_processor_execution_metrics", id: :uuid, default: nil, force: :cascade do |t| t.uuid "agent_id" t.string "session_id", null: false @@ -619,6 +597,10 @@ t.index ["name"], name: "idx_evo_core_api_keys_name_unique", unique: true end + create_table "evo_core_community_schema_migrations", primary_key: "version", id: :bigint, default: nil, force: :cascade do |t| + t.boolean "dirty", null: false + end + create_table "evo_core_custom_mcp_servers", id: :uuid, default: -> { "uuid_generate_v4()" }, force: :cascade do |t| t.string "name", limit: 255, null: false t.text "description" @@ -694,10 +676,6 @@ t.check_constraint "type::text = ANY (ARRAY['official'::text, 'community'::text])", name: "check_mcp_server_type" end - create_table "evo_core_schema_community_migrations", primary_key: "version", id: :bigint, default: nil, force: :cascade do |t| - t.boolean "dirty", null: false - end - create_table "facebook_comment_moderations", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "conversation_id", null: false t.uuid "message_id", null: false @@ -1051,7 +1029,6 @@ t.integer "position", default: 0, null: false t.integer "depth", default: 0, null: false t.index ["assigned_to_id", "status", "due_date"], name: "index_pipeline_tasks_on_assigned_to_id_and_status_and_due_date" - t.index ["assigned_to_id"], name: "index_pipeline_tasks_on_assigned_to_id" t.index ["created_by_id"], name: "index_pipeline_tasks_on_created_by_id" t.index ["due_date"], name: "index_pipeline_tasks_on_due_date" t.index ["parent_task_id", "position"], name: "index_pipeline_tasks_on_parent_task_id_and_position" @@ -1229,15 +1206,6 @@ t.index ["status"], name: "index_scheduled_actions_on_status" end - create_table "sessions", primary_key: ["app_name", "user_id", "id"], force: :cascade do |t| - t.string "app_name", limit: 128, null: false - t.string "user_id", limit: 128, null: false - t.string "id", limit: 128, null: false - t.jsonb "state", null: false - t.datetime "create_time", precision: nil, null: false - t.datetime "update_time", precision: nil, null: false - end - create_table "stage_movements", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.uuid "pipeline_item_id", null: false t.uuid "from_stage_id" @@ -1315,13 +1283,6 @@ t.index ["user_id"], name: "index_user_roles_on_user_id" end - create_table "user_states", primary_key: ["app_name", "user_id"], force: :cascade do |t| - t.string "app_name", limit: 128, null: false - t.string "user_id", limit: 128, null: false - t.jsonb "state", null: false - t.datetime "update_time", precision: nil, null: false - end - create_table "user_tours", force: :cascade do |t| t.uuid "user_id", null: false t.string "tour_key", null: false @@ -1330,7 +1291,6 @@ t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["user_id", "tour_key"], name: "index_user_tours_on_user_id_and_tour_key", unique: true - t.index ["user_id"], name: "index_user_tours_on_user_id" end create_table "users", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -1413,7 +1373,6 @@ add_foreign_key "contact_companies", "contacts" add_foreign_key "contact_companies", "contacts", column: "company_id" add_foreign_key "data_privacy_consents", "users" - add_foreign_key "events", "sessions", column: ["app_name", "user_id", "session_id"], primary_key: ["app_name", "user_id", "id"], name: "events_app_name_user_id_session_id_fkey", on_delete: :cascade add_foreign_key "evo_agent_processor_execution_metrics", "evo_core_agents", column: "agent_id", name: "evo_agent_processor_execution_metrics_agent_id_fkey", on_delete: :cascade add_foreign_key "evo_ai_agent_processor_execution_metrics", "evo_core_agents", column: "agent_id", name: "evo_ai_agent_processor_execution_metrics_agent_id_fkey", on_delete: :cascade add_foreign_key "evo_core_agent_integrations", "evo_core_agents", column: "agent_id", name: "evo_core_agent_integrations_agent_id_fkey", on_delete: :cascade @@ -1422,7 +1381,6 @@ add_foreign_key "evo_core_folder_shares", "evo_core_folders", column: "folder_id", name: "evo_core_folder_shares_folder_id_fkey", on_delete: :cascade add_foreign_key "facebook_comment_moderations", "conversations" add_foreign_key "facebook_comment_moderations", "messages" - add_foreign_key "facebook_comment_moderations", "users", column: "moderated_by_id" add_foreign_key "oauth_access_grants", "oauth_applications", column: "application_id" add_foreign_key "oauth_access_tokens", "oauth_applications", column: "application_id" add_foreign_key "pipeline_items", "contacts" @@ -1432,24 +1390,17 @@ add_foreign_key "pipeline_service_definitions", "pipelines" add_foreign_key "pipeline_tasks", "pipeline_items" add_foreign_key "pipeline_tasks", "pipeline_tasks", column: "parent_task_id" - add_foreign_key "pipeline_tasks", "users", column: "assigned_to_id" - add_foreign_key "pipeline_tasks", "users", column: "created_by_id" add_foreign_key "plan_features", "features", name: "plan_features_feature_id_fkey" add_foreign_key "plan_features", "plans", name: "plan_features_plan_id_fkey" add_foreign_key "role_permissions_actions", "roles" add_foreign_key "scheduled_action_execution_logs", "scheduled_actions" add_foreign_key "scheduled_action_notifications", "scheduled_actions", on_delete: :cascade - add_foreign_key "scheduled_action_notifications", "users", on_delete: :cascade - add_foreign_key "scheduled_action_templates", "users", column: "created_by", on_delete: :cascade add_foreign_key "scheduled_actions", "contacts", on_delete: :cascade add_foreign_key "scheduled_actions", "conversations", on_delete: :cascade - add_foreign_key "scheduled_actions", "users", column: "created_by", on_delete: :cascade - add_foreign_key "scheduled_actions", "users", column: "notify_user_id", on_delete: :nullify add_foreign_key "stage_movements", "pipeline_items" add_foreign_key "stage_movements", "pipeline_stages", column: "from_stage_id" add_foreign_key "stage_movements", "pipeline_stages", column: "to_stage_id" add_foreign_key "user_roles", "roles" add_foreign_key "user_roles", "users" add_foreign_key "user_roles", "users", column: "granted_by_id" - add_foreign_key "user_tours", "users" end diff --git a/docker/entrypoints/rails.sh b/docker/entrypoints/rails.sh index 77657f6..56f0501 100755 --- a/docker/entrypoints/rails.sh +++ b/docker/entrypoints/rails.sh @@ -20,15 +20,11 @@ done echo "Database ready to accept connections." -#install missing gems for local dev as we are using base image compiled for production -bundle install +# Ensure gems are installed and up-to-date +bundle check || bundle install -BUNDLE="bundle check" - -until $BUNDLE -do - sleep 2; -done +# Prepare the database (create if missing, run migrations) +bundle exec rails db:prepare # Execute the main process of the container exec "$@" diff --git a/public/downloads/import-contacts-sample.csv b/public/downloads/import-contacts-sample.csv index a16559d..f9f1a8e 100644 --- a/public/downloads/import-contacts-sample.csv +++ b/public/downloads/import-contacts-sample.csv @@ -1,6 +1,6 @@ -tipo,nome,primeiro_nome,sobrenome,email,telefone,cpf_cnpj,website,segmento_industria,cidade,pais,codigo_pais,linkedin,facebook,instagram,twitter,github,descricao,empresas_vinculadas -person,João Silva,João,Silva,joao@example.com,+5511999999999,12345678901,,,São Paulo,Brasil,BR,https://linkedin.com/in/joaosilva,,,https://github.com/joaosilva,Engenheiro de Software,Empresa X|Outra Empresa -company,Empresa X,,,,+5511888888888,12345678000190,https://empresa-x.com,Tecnologia,São Paulo,Brasil,BR,https://linkedin.com/company/empresax,,,,,Empresa de tecnologia,, -company,Tech Solutions Ltda,Empresa,,,,,+5511877777777,12345678000191,https://tech.com,Tecnologia,Rio de Janeiro,Brasil,BR,,,,,,Consultoria,, -person,Maria Santos,Maria,Santos,maria@example.com,+5511977777777,12345678902,,,São Paulo,Brasil,BR,,,,,Gerente de Projetos,Empresa X -person,Pedro Costa,Pedro,Costa,pedro@example.com,+5511966666666,12345678903,,,Campinas,Brasil,BR,https://linkedin.com/in/pedrocosta,,,,https://github.com/pedrocosta,Desenvolvedor Fullstack,Empresa X|Tech Solutions Ltda +tipo,nome,primeiro_nome,sobrenome,email,telefone,cpf_cnpj,website,segmento_industria,cidade,pais,codigo_pais,linkedin,facebook,instagram,twitter,github,descricao,empresas_vinculadas,custom_attribute_1,custom_attribute_2 +person,João Silva,João,Silva,joao@example.com,5511999999999,12345678901,,,São Paulo,Brasil,BR,https://linkedin.com/in/joaosilva,,,,https://github.com/joaosilva,Engenheiro de Software,Empresa X|Outra Empresa,,Outro atributo +company,Empresa X,,,,5511888888888,12345678000190,https://empresa-x.com,Tecnologia,São Paulo,Brasil,BR,https://linkedin.com/company/empresax,,,,,Empresa de tecnologia,,, +company,Tech Solutions Ltda,Empresa,,,5511877777777,12345678000191,https://tech.com,Tecnologia,Rio de Janeiro,Brasil,BR,,,,,,,,Consultoria, +person,Maria Santos,Maria,Santos,maria@example.com,5511977777777,12345678902,,,São Paulo,Brasil,BR,,,,,,Gerente de Projetos,Empresa X,, +person,Pedro Costa,Pedro,Costa,pedro@example.com,5511966666666,12345678903,,,Campinas,Brasil,BR,https://linkedin.com/in/pedrocosta,,,,https://github.com/pedrocosta,Desenvolvedor Fullstack,Empresa X|Tech Solutions Ltda,, \ No newline at end of file