diff --git a/app/mailers/devise_notify_mailer.rb b/app/mailers/devise_notify_mailer.rb index 07d8f69eb..5560e7925 100644 --- a/app/mailers/devise_notify_mailer.rb +++ b/app/mailers/devise_notify_mailer.rb @@ -6,11 +6,7 @@ class DeviseNotifyMailer < Devise::Mailer end def send_email(email, template_id, personalisation) - if intercept_send?(email) - Rails.logger.info("Intercepted send to #{email}") - return true - end - #return true if intercept_send?(email) + return true if intercept_send?(email) notify_client.send_email( email_address: email, diff --git a/app/mailers/notify_mailer.rb b/app/mailers/notify_mailer.rb index e76fa9ab2..21a6e0270 100644 --- a/app/mailers/notify_mailer.rb +++ b/app/mailers/notify_mailer.rb @@ -6,10 +6,7 @@ class NotifyMailer < ApplicationMailer end def send_email(email, template_id, personalisation) - if intercept_send?(email) - Rails.logger.info("Intercepted email send to #{email}") - return true - end + return true if intercept_send?(email) notify_client.send_email( email_address: email, diff --git a/app/services/imports/user_import_service.rb b/app/services/imports/user_import_service.rb index bbeaa1b58..b0fe32bef 100644 --- a/app/services/imports/user_import_service.rb +++ b/app/services/imports/user_import_service.rb @@ -18,15 +18,14 @@ module Imports deleted = user_field_value(xml_document, "deleted") if LegacyUser.find_by(old_user_id:) - #@logger.warn("User #{name} with old user id #{old_user_id} is already present, skipping.") + @logger.warn("User #{name} with old user id #{old_user_id} is already present, skipping.") elsif deleted == "true" - #@logger.warn("User #{name} with old user id #{old_user_id} is deleted, skipping.") + @logger.warn("User #{name} with old user id #{old_user_id} is deleted, skipping.") elsif (user = User.find_by(email:, organisation:)) is_dpo = user.is_data_protection_officer? || is_dpo?(user_field_value(xml_document, "user-type")) role = highest_role(user.role, role(user_field_value(xml_document, "user-type"))) user.update!(role:, is_dpo:) user.legacy_users.create!(old_user_id:) - #@logger.info("Found duplicated email, updating user #{user.id} with role #{role} and is_dpo #{is_dpo}") else user = User.new user.email = email diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index 427821c49..1c1841603 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -1,33 +1,88 @@ -Import = Struct.new("Import", :import_class, :import_method, :folder) - -namespace :core do - desc "Import all data XMLs from legacy CORE" - task :full_import, %i[archive_path] => :environment do |_task, args| - archive_path = args[:archive_path] - raise "Usage: rake core:full_import['path/to/archive']" if archive_path.blank? +namespace :import do + task :full, %i[institutions_csv_path] => :environment do |_task, args| + institutions_csv_path = args[:institutions_csv_path] + raise "Usage: rake core:full_import['path/to/institutions_csv']" if institutions_csv_path.blank? s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - archive_io = s3_service.get_file_io(archive_path) - archive_service = Storage::ArchiveService.new(archive_io) + csv = CSV.parse(s3_service.get_file_io(institutions_csv_path), headers: true) + org_count = csv.length - import_list = [ + initial_import_list = [ Import.new(Imports::OrganisationImportService, :create_organisations, "institution"), Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"), Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"), Import.new(Imports::UserImportService, :create_users, "user"), Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"), Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"), + ] + + Rails.logger.info("Beginning initial imports for #{org_count} organisations") + + csv.each { |row| + archive_path = row[1] + archive_io = s3_service.get_file_io(archive_path) + archive_service = Storage::ArchiveService.new(archive_io) + + Rails.logger.info("Performing initial imports for organisation #{row[0]}") + + initial_import_list.each do |step| + if archive_service.folder_present?(step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder) + end + end + } + + logs_import_list = [ Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"), - #Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), + Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), ] - import_list.each do |step| - if archive_service.folder_present?(step.folder) - Rails.logger.info("Start importing folder #{step.folder}") - step.import_class.new(archive_service).send(step.import_method, step.folder) - else - Rails.logger.info("#{step.folder} does not exist, skipping #{step.import_class}") + Rails.logger.info("Initial imports complete, beginning log imports for #{org_count} organisations") + + csv.each { |row| + archive_path = row[1] + archive_io = s3_service.get_file_io(archive_path) + archive_service = Storage::ArchiveService.new(archive_io) + + log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i + Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") + + logs_import_list.each do |step| + if archive_service.folder_present?(step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder) + end end + } + + Rails.logger.info("Log import complete, triggering user invite emails") + + csv.each { |row| + organisation = Organisation.find_by(name: row[0]) + next unless organisation + users = User.where(organisation: organisation, active: true, initial_confirmation_sent: false) + users.each { |user| ResendInvitationMailer.resend_invitation_email(user).deliver_later } + } + + Rails.logger.info("Invite emails triggered, generating report") + + rep = CSV.generate do |report| + headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"] + report << headers + + csv.each { |row| + name = row[0] + organisation = Organisation.find_by(name: name) + next unless organisation + completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count + in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count + completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count + in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count + report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs) + } end + report_name = "MigratedLogsReport_#{DateTime.now.strftime('%Y-%m-%dT%H:%M:%S')}.csv" + s3_service.write_file(report_name, rep) + + Rails.logger.info("Logs report available in s3 import bucket at #{report_name}") end end diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake deleted file mode 100644 index 13e9549f2..000000000 --- a/lib/tasks/import.rake +++ /dev/null @@ -1,88 +0,0 @@ -namespace :import do - task :full, %i[institutions_csv_path] => :environment do |_task, args| - institutions_csv_path = args[:institutions_csv_path] - raise "Must provide institutions csv path" if institutions_csv_path.blank? - - s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) - csv = CSV.parse(s3_service.get_file_io(institutions_csv_path), headers: true) - org_count = csv.length - - initial_import_list = [ - Import.new(Imports::OrganisationImportService, :create_organisations, "institution"), - Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"), - Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"), - Import.new(Imports::UserImportService, :create_users, "user"), - Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"), - Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"), - ] - - Rails.logger.info("Beginning initial imports for #{org_count} organisations") - - csv.each { |row| - archive_path = row[1] - archive_io = s3_service.get_file_io(archive_path) - archive_service = Storage::ArchiveService.new(archive_io) - - Rails.logger.info("Performing initial imports for organisation #{row[0]}") - - initial_import_list.each do |step| - if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) - end - end - } - - logs_import_list = [ - Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"), - Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), - ] - - Rails.logger.info("Initial imports complete, beginning log imports for #{org_count} organisations") - - csv.each { |row| - archive_path = row[1] - archive_io = s3_service.get_file_io(archive_path) - archive_service = Storage::ArchiveService.new(archive_io) - - log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i - Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") - - logs_import_list.each do |step| - if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service).send(step.import_method, step.folder) - end - end - } - - Rails.logger.info("Log import complete, triggering user invite emails") - - csv.each { |row| - organisation = Organisation.find_by(name: row[0]) - next unless organisation - users = User.where(organisation: organisation, active: true) - users.each { |user| ResendInvitationMailer.resend_invitation_email(user).deliver_later } - } - - Rails.logger.info("Invite emails triggered, generating report") - - rep = CSV.generate do |report| - headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"] - report << headers - - csv.each { |row| - name = row[0] - organisation = Organisation.find_by(name: name) - next unless organisation - completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count - in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count - completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count - in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count - report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs) - } - end - report_name = "LogsReport-#{DateTime.now()}" - s3_service.write_file(report_name, rep) - - Rails.logger.info("Logs report available in s3 import bucket at #{report_name}") - end -end diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb index 67e8ca715..4298004e9 100644 --- a/spec/lib/tasks/full_import_spec.rb +++ b/spec/lib/tasks/full_import_spec.rb @@ -2,8 +2,8 @@ require "rails_helper" require "rake" require "zip" -describe "rake core:full_import", type: :task do - subject(:task) { Rake::Task["core:full_import"] } +describe "rake import:full", type: :task do + subject(:task) { Rake::Task["import:full"] } let(:s3_service) { instance_double(Storage::S3Service) } let(:archive_service) { instance_double(Storage::ArchiveService) } @@ -17,6 +17,7 @@ describe "rake core:full_import", type: :task do allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) allow(Storage::S3Service).to receive(:new).and_return(s3_service) allow(s3_service).to receive(:get_file_io) + allow(s3_service).to receive(:write_file) allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) end @@ -56,7 +57,7 @@ describe "rake core:full_import", type: :task do expect(data_protection_service).to receive(:create_data_protection_confirmations).with("dataprotect") expect(rent_period_service).to receive(:create_organisation_rent_periods).with("rent-period") expect(lettings_logs_service).to receive(:create_logs).with("logs") - # expect(sales_logs_service).to receive(:create_logs).with("logs") + expect(sales_logs_service).to receive(:create_logs).with("logs") task.invoke(fixture_path) end @@ -76,12 +77,10 @@ describe "rake core:full_import", type: :task do expect(data_protection_service).to receive(:create_data_protection_confirmations) expect(rent_period_service).to receive(:create_organisation_rent_periods) expect(lettings_logs_service).to receive(:create_logs) - # expect(sales_logs_service).to receive(:create_logs) + expect(sales_logs_service).to receive(:create_logs) expect(scheme_service).not_to receive(:create_schemes) expect(location_service).not_to receive(:create_scheme_locations) - expect(Rails.logger).to receive(:info).with("mgmtgroups does not exist, skipping Imports::SchemeImportService") - expect(Rails.logger).to receive(:info).with("schemes does not exist, skipping Imports::SchemeLocationImportService") task.invoke(fixture_path) end