Tidy

app/mailers/devise_notify_mailer.rb

@@ -6,11 +6,7 @@ class DeviseNotifyMailer < Devise::Mailer
   end
 
   def send_email(email, template_id, personalisation)
-    if intercept_send?(email)
+    return true if intercept_send?(email)
-      Rails.logger.info("Intercepted send to #{email}")
-      return true
-    end
-    #return true if intercept_send?(email)
 
     notify_client.send_email(
       email_address: email,

app/mailers/notify_mailer.rb

@@ -6,10 +6,7 @@ class NotifyMailer < ApplicationMailer
   end
 
   def send_email(email, template_id, personalisation)
-    if intercept_send?(email)
+    return true if intercept_send?(email)
-      Rails.logger.info("Intercepted email send to #{email}")
-      return true
-    end
 
     notify_client.send_email(
       email_address: email,

app/services/imports/user_import_service.rb

@@ -18,15 +18,14 @@ module Imports
       deleted = user_field_value(xml_document, "deleted")
 
       if LegacyUser.find_by(old_user_id:)
-        #@logger.warn("User #{name} with old user id #{old_user_id} is already present, skipping.")
+        @logger.warn("User #{name} with old user id #{old_user_id} is already present, skipping.")
       elsif deleted == "true"
-        #@logger.warn("User #{name} with old user id #{old_user_id} is deleted, skipping.")
+        @logger.warn("User #{name} with old user id #{old_user_id} is deleted, skipping.")
       elsif (user = User.find_by(email:, organisation:))
         is_dpo = user.is_data_protection_officer? || is_dpo?(user_field_value(xml_document, "user-type"))
         role = highest_role(user.role, role(user_field_value(xml_document, "user-type")))
         user.update!(role:, is_dpo:)
         user.legacy_users.create!(old_user_id:)
-        #@logger.info("Found duplicated email, updating user #{user.id} with role #{role} and is_dpo #{is_dpo}")
       else
         user = User.new
         user.email = email

lib/tasks/full_import.rake

@@ -1,33 +1,88 @@
-Import = Struct.new("Import", :import_class, :import_method, :folder)
+namespace :import do
-
+  task :full, %i[institutions_csv_path] => :environment do |_task, args|
-namespace :core do
+    institutions_csv_path = args[:institutions_csv_path]
-  desc "Import all data XMLs from legacy CORE"
+    raise "Usage: rake core:full_import['path/to/institutions_csv']" if institutions_csv_path.blank?
-  task :full_import, %i[archive_path] => :environment do |_task, args|
-    archive_path = args[:archive_path]
-    raise "Usage: rake core:full_import['path/to/archive']" if archive_path.blank?
 
     s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    archive_io = s3_service.get_file_io(archive_path)
+    csv = CSV.parse(s3_service.get_file_io(institutions_csv_path), headers: true)
-    archive_service = Storage::ArchiveService.new(archive_io)
+    org_count = csv.length
 
-    import_list = [
+    initial_import_list = [
       Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),
       Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"),
       Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"),
       Import.new(Imports::UserImportService, :create_users, "user"),
       Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"),
       Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"),
+    ]
+
+    Rails.logger.info("Beginning initial imports for #{org_count} organisations")
+
+    csv.each { |row|
+      archive_path = row[1]
+      archive_io = s3_service.get_file_io(archive_path)
+      archive_service = Storage::ArchiveService.new(archive_io)
+
+      Rails.logger.info("Performing initial imports for organisation #{row[0]}")
+
+      initial_import_list.each do |step|
+        if archive_service.folder_present?(step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder)
+        end
+      end
+    }
+
+    logs_import_list = [
       Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"),
-      #Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
+      Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
     ]
 
-    import_list.each do |step|
+    Rails.logger.info("Initial imports complete, beginning log imports for #{org_count} organisations")
-      if archive_service.folder_present?(step.folder)
+
-        Rails.logger.info("Start importing folder #{step.folder}")
+    csv.each { |row|
-        step.import_class.new(archive_service).send(step.import_method, step.folder)
+      archive_path = row[1]
-      else
+      archive_io = s3_service.get_file_io(archive_path)
-        Rails.logger.info("#{step.folder} does not exist, skipping #{step.import_class}")
+      archive_service = Storage::ArchiveService.new(archive_io)
+
+      log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i
+      Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
+
+      logs_import_list.each do |step|
+        if archive_service.folder_present?(step.folder)
+          step.import_class.new(archive_service).send(step.import_method, step.folder)
+        end
       end
+    }
+
+    Rails.logger.info("Log import complete, triggering user invite emails")
+
+    csv.each { |row|
+      organisation = Organisation.find_by(name: row[0])
+      next unless organisation
+      users = User.where(organisation: organisation, active: true, initial_confirmation_sent: false)
+      users.each { |user| ResendInvitationMailer.resend_invitation_email(user).deliver_later }
+    }
+
+    Rails.logger.info("Invite emails triggered, generating report")
+
+    rep = CSV.generate do |report|
+      headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"]
+      report << headers
+
+      csv.each { |row|
+        name = row[0]
+        organisation = Organisation.find_by(name: name)
+        next unless organisation
+        completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count
+        in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count
+        completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count
+        in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count
+        report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs)
+      }
     end
+    report_name = "MigratedLogsReport_#{DateTime.now.strftime('%Y-%m-%dT%H:%M:%S')}.csv"
+    s3_service.write_file(report_name, rep)
+
+    Rails.logger.info("Logs report available in s3 import bucket at #{report_name}")
   end
 end

lib/tasks/import.rake

@@ -1,88 +0,0 @@
-namespace :import do
-  task :full, %i[institutions_csv_path] => :environment do |_task, args|
-    institutions_csv_path = args[:institutions_csv_path]
-    raise "Must provide institutions csv path" if institutions_csv_path.blank?
-
-    s3_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"])
-    csv = CSV.parse(s3_service.get_file_io(institutions_csv_path), headers: true)
-    org_count = csv.length
-
-    initial_import_list = [
-      Import.new(Imports::OrganisationImportService, :create_organisations, "institution"),
-      Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"),
-      Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"),
-      Import.new(Imports::UserImportService, :create_users, "user"),
-      Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"),
-      Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"),
-    ]
-
-    Rails.logger.info("Beginning initial imports for #{org_count} organisations")
-
-    csv.each { |row|
-      archive_path = row[1]
-      archive_io = s3_service.get_file_io(archive_path)
-      archive_service = Storage::ArchiveService.new(archive_io)
-
-      Rails.logger.info("Performing initial imports for organisation #{row[0]}")
-
-      initial_import_list.each do |step|
-        if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
-        end
-      end
-    }
-
-    logs_import_list = [
-      Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"),
-      Import.new(Imports::SalesLogsImportService, :create_logs, "logs"),
-    ]
-
-    Rails.logger.info("Initial imports complete, beginning log imports for #{org_count} organisations")
-
-    csv.each { |row|
-      archive_path = row[1]
-      archive_io = s3_service.get_file_io(archive_path)
-      archive_service = Storage::ArchiveService.new(archive_io)
-
-      log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i
-      Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs")
-
-      logs_import_list.each do |step|
-        if archive_service.folder_present?(step.folder)
-          step.import_class.new(archive_service).send(step.import_method, step.folder)
-        end
-      end
-    }
-
-    Rails.logger.info("Log import complete, triggering user invite emails")
-
-    csv.each { |row|
-      organisation = Organisation.find_by(name: row[0])
-      next unless organisation
-      users = User.where(organisation: organisation, active: true)
-      users.each { |user| ResendInvitationMailer.resend_invitation_email(user).deliver_later }
-    }
-
-    Rails.logger.info("Invite emails triggered, generating report")
-
-    rep = CSV.generate do |report|
-      headers = ["Institution name", "Id", "Old Completed lettings logs", "Old In progress lettings logs", "Old Completed sales logs", "Old In progress sales logs", "New Completed lettings logs", "New In Progress lettings logs", "New Completed sales logs", "New In Progress sales logs"]
-      report << headers
-
-      csv.each { |row|
-        name = row[0]
-        organisation = Organisation.find_by(name: name)
-        next unless organisation
-        completed_sales_logs = organisation.owned_sales_logs.where(status: "completed").count
-        in_progress_sales_logs = organisation.owned_sales_logs.where(status: "in_progress").count
-        completed_lettings_logs = organisation.owned_lettings_logs.where(status: "completed").count
-        in_progress_lettings_logs = organisation.owned_lettings_logs.where(status: "in_progress").count
-        report << row.push(completed_lettings_logs, in_progress_lettings_logs, completed_sales_logs, in_progress_sales_logs)
-      }
-    end
-    report_name = "LogsReport-#{DateTime.now()}"
-    s3_service.write_file(report_name, rep)
-
-    Rails.logger.info("Logs report available in s3 import bucket at #{report_name}")
-  end
-end

spec/lib/tasks/full_import_spec.rb

@@ -2,8 +2,8 @@ require "rails_helper"
 require "rake"
 require "zip"
 
-describe "rake core:full_import", type: :task do
+describe "rake import:full", type: :task do
-  subject(:task) { Rake::Task["core:full_import"] }
+  subject(:task) { Rake::Task["import:full"] }
 
   let(:s3_service) { instance_double(Storage::S3Service) }
   let(:archive_service) { instance_double(Storage::ArchiveService) }
@@ -17,6 +17,7 @@ describe "rake core:full_import", type: :task do
     allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service)
     allow(Storage::S3Service).to receive(:new).and_return(s3_service)
     allow(s3_service).to receive(:get_file_io)
+    allow(s3_service).to receive(:write_file)
     allow(Storage::ArchiveService).to receive(:new).and_return(archive_service)
   end
 
@@ -56,7 +57,7 @@ describe "rake core:full_import", type: :task do
         expect(data_protection_service).to receive(:create_data_protection_confirmations).with("dataprotect")
         expect(rent_period_service).to receive(:create_organisation_rent_periods).with("rent-period")
         expect(lettings_logs_service).to receive(:create_logs).with("logs")
-        # expect(sales_logs_service).to receive(:create_logs).with("logs")
+        expect(sales_logs_service).to receive(:create_logs).with("logs")
 
         task.invoke(fixture_path)
       end
@@ -76,12 +77,10 @@ describe "rake core:full_import", type: :task do
         expect(data_protection_service).to receive(:create_data_protection_confirmations)
         expect(rent_period_service).to receive(:create_organisation_rent_periods)
         expect(lettings_logs_service).to receive(:create_logs)
-        # expect(sales_logs_service).to receive(:create_logs)
+        expect(sales_logs_service).to receive(:create_logs)
 
         expect(scheme_service).not_to receive(:create_schemes)
         expect(location_service).not_to receive(:create_scheme_locations)
-        expect(Rails.logger).to receive(:info).with("mgmtgroups does not exist, skipping Imports::SchemeImportService")
-        expect(Rails.logger).to receive(:info).with("schemes does not exist, skipping Imports::SchemeLocationImportService")
 
         task.invoke(fixture_path)
       end