From a16c66295eb723586bd136c3d51e4ac146365e7f Mon Sep 17 00:00:00 2001 From: Kat Date: Tue, 15 Aug 2023 17:35:14 +0100 Subject: [PATCH] Revert "CLDC-2602 Save import logs to S3 (#1829)" This reverts commit b93a6456d4e7240ecb654c7c5d8d4d92a6c14e38. --- config/initializers/multi_logger.rb | 17 ----- lib/tasks/full_import.rake | 49 ++++++--------- spec/lib/tasks/full_import_spec.rb | 62 ------------------- .../organisation_import_service_spec.rb | 7 +-- 4 files changed, 20 insertions(+), 115 deletions(-) delete mode 100644 config/initializers/multi_logger.rb diff --git a/config/initializers/multi_logger.rb b/config/initializers/multi_logger.rb deleted file mode 100644 index d39d786c1..000000000 --- a/config/initializers/multi_logger.rb +++ /dev/null @@ -1,17 +0,0 @@ -class MultiLogger - def initialize(*targets) - @targets = targets - end - - def info(data) - @targets.each { |t| t.info(data) } - end - - def warn(data) - @targets.each { |t| t.warn(data) } - end - - def error(data) - @targets.each { |t| t.error(data) } - end -end diff --git a/lib/tasks/full_import.rake b/lib/tasks/full_import.rake index c1fe146d8..4fea7ef93 100644 --- a/lib/tasks/full_import.rake +++ b/lib/tasks/full_import.rake @@ -1,4 +1,4 @@ -Import = Struct.new("Import", :import_class, :import_method, :folder, :logger) +Import = Struct.new("Import", :import_class, :import_method, :folder) namespace :import do desc "Run initial import steps - orgs, schemes, users etc (without triggering user invite emails)" @@ -8,40 +8,34 @@ namespace :import do s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) - logs_string = StringIO.new - logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) org_count = csv.length initial_import_list = [ - Import.new(Imports::OrganisationImportService, :create_organisations, "institution", logger), - Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups", logger), - Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes", logger), - Import.new(Imports::UserImportService, :create_users, "user", logger), - Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect", logger), - Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period", logger), + Import.new(Imports::OrganisationImportService, :create_organisations, "institution"), + Import.new(Imports::SchemeImportService, :create_schemes, "mgmtgroups"), + Import.new(Imports::SchemeLocationImportService, :create_scheme_locations, "schemes"), + Import.new(Imports::UserImportService, :create_users, "user"), + Import.new(Imports::DataProtectionConfirmationImportService, :create_data_protection_confirmations, "dataprotect"), + Import.new(Imports::OrganisationRentPeriodImportService, :create_organisation_rent_periods, "rent-period"), ] - logger.info("Beginning initial imports for #{org_count} organisations") + Rails.logger.info("Beginning initial imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] archive_io = s3_service.get_file_io(archive_path) archive_service = Storage::ArchiveService.new(archive_io) - logger.info("Performing initial imports for organisation #{row[0]}") + + Rails.logger.info("Performing initial imports for organisation #{row[0]}") initial_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service, step.logger).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder) end end - - log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_initial.log" - s3_service.write_file(log_file, logs_string.string) - logs_string.rewind - logs_string.truncate(0) end - logger.info("Finished initial imports") + Rails.logger.info("Finished initial imports") end desc "Run logs import steps" @@ -52,15 +46,13 @@ namespace :import do s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) csv = CSV.parse(s3_service.get_file_io(institutions_csv_name), headers: true) org_count = csv.length - logs_string = StringIO.new - logger = MultiLogger.new(Rails.logger, Logger.new(logs_string)) logs_import_list = [ - Import.new(Imports::LettingsLogsImportService, :create_logs, "logs", logger), - Import.new(Imports::SalesLogsImportService, :create_logs, "logs", logger), + Import.new(Imports::LettingsLogsImportService, :create_logs, "logs"), + Import.new(Imports::SalesLogsImportService, :create_logs, "logs"), ] - logger.info("Beginning log imports for #{org_count} organisations") + Rails.logger.info("Beginning log imports for #{org_count} organisations") csv.each do |row| archive_path = row[1] @@ -68,21 +60,16 @@ namespace :import do archive_service = Storage::ArchiveService.new(archive_io) log_count = row[2].to_i + row[3].to_i + row[4].to_i + row[5].to_i - logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") + Rails.logger.info("Importing logs for organisation #{row[0]}, expecting #{log_count} logs") logs_import_list.each do |step| if archive_service.folder_present?(step.folder) - step.import_class.new(archive_service, step.logger).send(step.import_method, step.folder) + step.import_class.new(archive_service).send(step.import_method, step.folder) end end - - log_file = "#{File.basename(institutions_csv_name, File.extname(institutions_csv_name))}_#{File.basename(archive_path, File.extname(archive_path))}_logs.log" - s3_service.write_file(log_file, logs_string.string) - logs_string.rewind - logs_string.truncate(0) end - logger.info("Log import complete") + Rails.logger.info("Log import complete") end desc "Trigger invite emails for imported users" diff --git a/spec/lib/tasks/full_import_spec.rb b/spec/lib/tasks/full_import_spec.rb index 0b8d3bd9a..4c123d5b5 100644 --- a/spec/lib/tasks/full_import_spec.rb +++ b/spec/lib/tasks/full_import_spec.rb @@ -53,66 +53,4 @@ describe "full import", type: :task do end end end - - describe "import:initial" do - subject(:task) { Rake::Task["import:initial"] } - - let(:archive_service) { instance_double(Storage::ArchiveService) } - - before do - Rake.application.rake_require("tasks/full_import") - Rake::Task.define_task(:environment) - task.reenable - end - - context "when calling the initial import" do - before do - allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) - allow(archive_service).to receive(:folder_present?).and_return(false) - allow(Imports::OrganisationImportService).to receive(:new).and_return(instance_double(Imports::OrganisationImportService)) - allow(Imports::SchemeImportService).to receive(:new).and_return(instance_double(Imports::SchemeImportService)) - allow(Imports::SchemeLocationImportService).to receive(:new).and_return(instance_double(Imports::SchemeLocationImportService)) - allow(Imports::UserImportService).to receive(:new).and_return(instance_double(Imports::UserImportService)) - allow(Imports::DataProtectionConfirmationImportService).to receive(:new).and_return(instance_double(Imports::DataProtectionConfirmationImportService)) - allow(Imports::OrganisationRentPeriodImportService).to receive(:new).and_return(instance_double(Imports::OrganisationRentPeriodImportService)) - end - - it "creates a report using given organisation csv" do - expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name) - expect(storage_service).to receive(:write_file).with("some_name_1_initial.log", / INFO -- : Performing initial imports for organisation org1/) - expect(storage_service).to receive(:write_file).with("some_name_2_initial.log", / INFO -- : Performing initial imports for organisation org2/) - - task.invoke("some_name.csv") - end - end - end - - describe "import:logs" do - subject(:task) { Rake::Task["import:logs"] } - - let(:archive_service) { instance_double(Storage::ArchiveService) } - - before do - Rake.application.rake_require("tasks/full_import") - Rake::Task.define_task(:environment) - task.reenable - end - - context "when calling the logs import" do - before do - allow(Storage::ArchiveService).to receive(:new).and_return(archive_service) - allow(archive_service).to receive(:folder_present?).and_return(false) - allow(Imports::LettingsLogsImportService).to receive(:new).and_return(instance_double(Imports::LettingsLogsImportService)) - allow(Imports::SalesLogsImportService).to receive(:new).and_return(instance_double(Imports::SalesLogsImportService)) - end - - it "creates a report using given organisation csv" do - expect(Storage::S3Service).to receive(:new).with(env_config_service, instance_name) - expect(storage_service).to receive(:write_file).with("some_name_1_logs.log", / INFO -- : Importing logs for organisation org1, expecting 0 logs/) - expect(storage_service).to receive(:write_file).with("some_name_2_logs.log", / INFO -- : Importing logs for organisation org2, expecting 0 logs/) - - task.invoke("some_name.csv") - end - end - end end diff --git a/spec/services/imports/organisation_import_service_spec.rb b/spec/services/imports/organisation_import_service_spec.rb index 8496c046f..c8fcd4829 100644 --- a/spec/services/imports/organisation_import_service_spec.rb +++ b/spec/services/imports/organisation_import_service_spec.rb @@ -2,8 +2,7 @@ require "rails_helper" RSpec.describe Imports::OrganisationImportService do let(:storage_service) { instance_double(Storage::S3Service) } - let(:logs_string) { StringIO.new } - let(:logger) { MultiLogger.new(Rails.logger, Logger.new(logs_string)) } + let(:logger) { instance_double(Rails::Rack::Logger) } let(:folder_name) { "organisations" } let(:filenames) { %w[my_folder/my_file1.xml my_folder/my_file2.xml] } let(:fixture_directory) { "spec/fixtures/imports/institution" } @@ -82,12 +81,10 @@ RSpec.describe Imports::OrganisationImportService do it "successfully create an organisation the first time, and does not update it" do expect(storage_service).to receive(:list_files).with(folder_name).twice expect(storage_service).to receive(:get_file_io).with(filenames[0]).twice - expect(logger).to receive(:warn).once.and_call_original - expect(Rails.logger).to receive(:warn).with("Organisation my_new_organisation is already present with old visible ID 1, skipping.") + expect(logger).to receive(:warn).once expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(1) expect { import_service.create_organisations(folder_name) }.to change(Organisation, :count).by(0) - expect(logs_string.string).to include("Organisation my_new_organisation is already present with old visible ID 1, skipping.") expect(Organisation).to exist(old_visible_id: "1", name: "HA Ltd") end