diff --git a/lib/tasks/import_address_from_csv.rake b/lib/tasks/import_address_from_csv.rake index e910f3a77..e30892e8b 100644 --- a/lib/tasks/import_address_from_csv.rake +++ b/lib/tasks/import_address_from_csv.rake @@ -1,18 +1,35 @@ namespace :data_import do - desc "Import address data from a csv file" - task :import_address_from_csv, %i[file_name] => :environment do |_task, args| + desc "Import lettings address data from a csv file" + task :import_lettings_addresses_from_csv, %i[file_name] => :environment do |_task, args| file_name = args[:file_name] - raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank? + raise "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']" if file_name.blank? s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + contains_issue_type = addresses_csv.headers.include?("Issue type") addresses_csv.each do |row| - lettings_log_id = row[0] + lettings_log_id = contains_issue_type ? row[1] : row[0] + uprn = contains_issue_type ? row[8] : row[7] + address_line1 = contains_issue_type ? row[9] : row[8] + address_line2 = contains_issue_type ? row[10] : row[9] + town_or_city = contains_issue_type ? row[11] : row[10] + county = contains_issue_type ? row[12] : row[11] + postcode_full = contains_issue_type ? row[13] : row[12] if lettings_log_id.blank? - Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}") + Rails.logger.info("Lettings log ID not provided for address: #{[address_line1, address_line2, town_or_city, county, postcode_full].join(', ')}") + next + end + + if uprn.present? + Rails.logger.info("Lettings log with ID #{lettings_log_id} contains uprn, skipping log") + next + end + + if address_line1.blank? || town_or_city.blank? || postcode_full.blank? + Rails.logger.info("Lettings log with ID #{lettings_log_id} is missing required address data, skipping log") next end @@ -25,18 +42,76 @@ namespace :data_import do lettings_log.uprn_known = 0 lettings_log.uprn = nil lettings_log.uprn_confirmed = nil - lettings_log.address_line1 = row[1] - lettings_log.address_line2 = row[2] - lettings_log.town_or_city = row[3] - lettings_log.postcode_full = row[4] + lettings_log.address_line1 = address_line1 + lettings_log.address_line2 = address_line2 + lettings_log.town_or_city = town_or_city + lettings_log.county = county + lettings_log.postcode_full = postcode_full lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil - lettings_log.county = nil lettings_log.is_la_inferred = nil lettings_log.la = nil lettings_log.values_updated_at = Time.zone.now lettings_log.save! - Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}") + Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.county, lettings_log.postcode_full].join(', ')}") + end + end + + desc "Import sales address data from a csv file" + task :import_sales_addresses_from_csv, %i[file_name] => :environment do |_task, args| + file_name = args[:file_name] + + raise "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']" if file_name.blank? + + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + contains_issue_type = addresses_csv.headers.include?("Issue type") + + addresses_csv.each do |row| + sales_log_id = contains_issue_type ? row[1] : row[0] + uprn = contains_issue_type ? row[6] : row[5] + address_line1 = contains_issue_type ? row[7] : row[6] + address_line2 = contains_issue_type ? row[8] : row[7] + town_or_city = contains_issue_type ? row[9] : row[8] + county = contains_issue_type ? row[10] : row[9] + postcode_full = contains_issue_type ? row[11] : row[10] + + if sales_log_id.blank? + Rails.logger.info("Sales log ID not provided for address: #{[address_line1, address_line2, town_or_city, county, postcode_full].join(', ')}") + next + end + + if uprn.present? + Rails.logger.info("Sales log with ID #{sales_log_id} contains uprn, skipping log") + next + end + + if address_line1.blank? || town_or_city.blank? || postcode_full.blank? + Rails.logger.info("Sales log with ID #{sales_log_id} is missing required address data, skipping log") + next + end + + sales_log = SalesLog.find_by(id: sales_log_id) + if sales_log.blank? + Rails.logger.info("Could not find a sales log with id #{sales_log_id}") + next + end + + sales_log.uprn_known = 0 + sales_log.uprn = nil + sales_log.uprn_confirmed = nil + sales_log.address_line1 = address_line1 + sales_log.address_line2 = address_line2 + sales_log.town_or_city = town_or_city + sales_log.county = county + sales_log.postcode_full = postcode_full + sales_log.pcodenk = sales_log.postcode_full.present? ? 0 : nil + sales_log.is_la_inferred = nil + sales_log.la = nil + sales_log.values_updated_at = Time.zone.now + + sales_log.save! + Rails.logger.info("Updated sales log #{sales_log_id}, with address: #{[sales_log.address_line1, sales_log.address_line2, sales_log.town_or_city, sales_log.county, sales_log.postcode_full].join(', ')}") end end end diff --git a/spec/fixtures/files/addresses_reimport.csv b/spec/fixtures/files/addresses_reimport.csv index 998d2fbfa..d413b3ffc 100644 --- a/spec/fixtures/files/addresses_reimport.csv +++ b/spec/fixtures/files/addresses_reimport.csv @@ -1,6 +1,7 @@ -lettings_log_id,address_line1,address_line2,town_or_city,postcode_full -{id},address 1,address 2,town,B1 1BB -{id2},address 3,address 4,city,B1 1BB -{id3},,,,C1 1CC -,address 5,address 6,city,D1 1DD -fake_id,address 7,address 8,city,D1 1DD +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property’s postcode +Full address required,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 1,address 2,town,county,B1 1BB +Missing town or city,{id2},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 3,,city,,B1 1BB +UPRN issues,{id3},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,123,Some Place,,Bristol,,BS1 1AD +UPRN issues,{id4},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,,,BS1 1AD +UPRN issues,,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD +UPRN issues,fake_id,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/addresses_reimport_all_logs.csv b/spec/fixtures/files/addresses_reimport_all_logs.csv new file mode 100644 index 000000000..9bf372231 --- /dev/null +++ b/spec/fixtures/files/addresses_reimport_all_logs.csv @@ -0,0 +1,7 @@ +Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property’s postcode +{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 1,address 2,town,county,B1 1BB +{id2},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 3,,city,,B1 1BB +{id3},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,123,Some Place,,Bristol,,BS1 1AD +{id4},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,,,BS1 1AD +,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD +fake_id,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/sales_addresses_reimport.csv b/spec/fixtures/files/sales_addresses_reimport.csv new file mode 100644 index 000000000..b613e70c2 --- /dev/null +++ b/spec/fixtures/files/sales_addresses_reimport.csv @@ -0,0 +1,7 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property’s postcode +Full address required,{id},2023-04-05,purchid,testy@example.com,Address org,,address 1,address 2,town,county,B1 1BB +Missing town or city,{id2},2023-04-05,purchid,testy@example.com,Address org,,address 3,,city,,B1 1BB +UPRN issues,{id3},2023-04-05,purchid,testy@example.com,Address org,123,Some Place,,Bristol,,BS1 1AD +UPRN issues,{id4},2023-04-05,purchid,testy@example.com,Address org,,Some Place,,,,BS1 1AD +UPRN issues,,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD +UPRN issues,fake_id,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/sales_addresses_reimport_all_logs.csv b/spec/fixtures/files/sales_addresses_reimport_all_logs.csv new file mode 100644 index 000000000..4ba69002b --- /dev/null +++ b/spec/fixtures/files/sales_addresses_reimport_all_logs.csv @@ -0,0 +1,7 @@ +Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property’s postcode +{id},2023-04-05,purchid,testy@example.com,Address org,,address 1,address 2,town,county,B1 1BB +{id2},2023-04-05,purchid,testy@example.com,Address org,,address 3,,city,,B1 1BB +{id3},2023-04-05,purchid,testy@example.com,Address org,123,Some Place,,Bristol,,BS1 1AD +{id4},2023-04-05,purchid,testy@example.com,Address org,,Some Place,,,,BS1 1AD +,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD +fake_id,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/lib/tasks/correct_address_from_csv_spec.rb b/spec/lib/tasks/correct_address_from_csv_spec.rb index 0a3d1c563..4924b3a25 100644 --- a/spec/lib/tasks/correct_address_from_csv_spec.rb +++ b/spec/lib/tasks/correct_address_from_csv_spec.rb @@ -2,14 +2,45 @@ require "rails_helper" require "rake" RSpec.describe "data_import" do - def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template) - export_template.sub!(/\{id\}/, lettings_log.id.to_s) - export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s) - export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s) + def replace_entity_ids(log, second_log, third_log, fourth_log, export_template) + export_template.sub!(/\{id\}/, log.id.to_s) + export_template.sub!(/\{id2\}/, second_log.id.to_s) + export_template.sub!(/\{id3\}/, third_log.id.to_s) + export_template.sub!(/\{id4\}/, fourth_log.id.to_s) end - describe ":import_address_from_csv", type: :task do - subject(:task) { Rake::Task["data_import:import_address_from_csv"] } + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) + allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") + + WebMock.stub_request(:get, /api\.postcodes\.io/) + .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) + WebMock.stub_request(:get, /api\.postcodes\.io\/postcodes\/B11BB/) + .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) + + body = { + results: [ + { + DPA: { + "POSTCODE": "LS16 6FT", + "POST_TOWN": "Westminster", + "PO_BOX_NUMBER": "Wrong Address Line1", + "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key&uprn=1") + .to_return(status: 200, body:, headers: {}) + end + + describe ":import_lettings_addresses_from_csv", type: :task do + subject(:task) { Rake::Task["data_import:import_lettings_addresses_from_csv"] } let(:instance_name) { "paas_import_instance" } let(:storage_service) { instance_double(Storage::S3Service) } @@ -17,25 +48,14 @@ RSpec.describe "data_import" do let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } before do - allow(Storage::S3Service).to receive(:new).and_return(storage_service) - allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) - allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) - allow(ENV).to receive(:[]) - allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) - allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") - Rake.application.rake_require("tasks/import_address_from_csv") Rake::Task.define_task(:environment) task.reenable - - WebMock.stub_request(:get, /api.postcodes.io/) - .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) - WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/) - .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) end context "when the rake task is run" do let(:addresses_csv_path) { "addresses_reimport_123.csv" } + let(:all_addresses_csv_path) { "all_addresses_reimport_123.csv" } let(:wrong_file_path) { "/test/no_csv_here.csv" } let!(:lettings_log) do create(:lettings_log, @@ -52,102 +72,391 @@ RSpec.describe "data_import" do is_la_inferred: true) end - let!(:second_lettings_log) do - create(:lettings_log, - uprn_known: 1, - uprn: "1", - uprn_confirmed: nil, - address_line1: "wrong address line1", - address_line2: "wrong address 2", - town_or_city: "wrong town", - county: "wrong city", - postcode_known: 1, - postcode_full: "A1 1AA", - la: "E06000064", - is_la_inferred: true) + let!(:lettings_logs) do + create_list(:lettings_log, + 3, + uprn_known: 1, + uprn: "1", + uprn_confirmed: nil, + address_line1: "wrong address line1", + address_line2: "wrong address 2", + town_or_city: "wrong town", + county: "wrong city", + postcode_known: 1, + postcode_full: "A1 1AA", + la: "E06000064", + is_la_inferred: true) end - let!(:third_lettings_log) do - create(:lettings_log, - uprn_known: 1, - uprn: "1", + before do + allow(storage_service).to receive(:get_file_io) + .with("addresses_reimport_123.csv") + .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read)) + + allow(storage_service).to receive(:get_file_io) + .with("all_addresses_reimport_123.csv") + .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read)) + end + + context "when the file contains issue type column" do + it "updates the log address when old address was not given" do + task.invoke(addresses_csv_path) + lettings_log.reload + expect(lettings_log.uprn_known).to eq(0) + expect(lettings_log.uprn).to eq(nil) + expect(lettings_log.uprn_confirmed).to eq(nil) + expect(lettings_log.address_line1).to eq("address 1") + expect(lettings_log.address_line2).to eq("address 2") + expect(lettings_log.town_or_city).to eq("town") + expect(lettings_log.county).to eq("county") + expect(lettings_log.postcode_known).to eq(1) + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(addresses_csv_path) + lettings_logs[0].reload + expect(lettings_logs[0].uprn_known).to eq(0) + expect(lettings_logs[0].uprn).to eq(nil) + expect(lettings_logs[0].uprn_confirmed).to eq(nil) + expect(lettings_logs[0].address_line1).to eq("address 3") + expect(lettings_logs[0].address_line2).to eq(nil) + expect(lettings_logs[0].town_or_city).to eq("city") + expect(lettings_logs[0].county).to eq(nil) + expect(lettings_logs[0].postcode_known).to eq(1) + expect(lettings_logs[0].postcode_full).to eq("B1 1BB") + expect(lettings_logs[0].la).to eq("E08000035") + expect(lettings_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(addresses_csv_path) + lettings_logs[1].reload + expect(lettings_logs[1].uprn_known).to eq(1) + expect(lettings_logs[1].uprn).to eq("1") + expect(lettings_logs[1].uprn_confirmed).to eq(nil) + expect(lettings_logs[1].address_line1).to eq("wrong address line1") + expect(lettings_logs[1].address_line2).to eq("wrong address 2") + expect(lettings_logs[1].town_or_city).to eq("wrong town") + expect(lettings_logs[1].county).to eq("wrong city") + expect(lettings_logs[1].postcode_known).to eq(1) + expect(lettings_logs[1].postcode_full).to eq("A1 1AA") + expect(lettings_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(addresses_csv_path) + lettings_logs[2].reload + expect(lettings_logs[2].uprn_known).to eq(1) + expect(lettings_logs[2].uprn).to eq("1") + expect(lettings_logs[2].uprn_confirmed).to eq(nil) + expect(lettings_logs[2].address_line1).to eq("wrong address line1") + expect(lettings_logs[2].address_line2).to eq("wrong address 2") + expect(lettings_logs[2].town_or_city).to eq("wrong town") + expect(lettings_logs[2].county).to eq("wrong city") + expect(lettings_logs[2].postcode_known).to eq(1) + expect(lettings_logs[2].postcode_full).to eq("A1 1AA") + expect(lettings_logs[2].la).to eq("E06000064") + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + + task.invoke(addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']") + end + end + + context "when the file does not contain issue type column" do + it "updates the log address when old address was not given" do + task.invoke(all_addresses_csv_path) + lettings_log.reload + expect(lettings_log.uprn_known).to eq(0) + expect(lettings_log.uprn).to eq(nil) + expect(lettings_log.uprn_confirmed).to eq(nil) + expect(lettings_log.address_line1).to eq("address 1") + expect(lettings_log.address_line2).to eq("address 2") + expect(lettings_log.town_or_city).to eq("town") + expect(lettings_log.county).to eq("county") + expect(lettings_log.postcode_known).to eq(1) + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(all_addresses_csv_path) + lettings_logs[0].reload + expect(lettings_logs[0].uprn_known).to eq(0) + expect(lettings_logs[0].uprn).to eq(nil) + expect(lettings_logs[0].uprn_confirmed).to eq(nil) + expect(lettings_logs[0].address_line1).to eq("address 3") + expect(lettings_logs[0].address_line2).to eq(nil) + expect(lettings_logs[0].town_or_city).to eq("city") + expect(lettings_logs[0].county).to eq(nil) + expect(lettings_logs[0].postcode_known).to eq(1) + expect(lettings_logs[0].postcode_full).to eq("B1 1BB") + expect(lettings_logs[0].la).to eq("E08000035") + expect(lettings_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(all_addresses_csv_path) + lettings_logs[1].reload + expect(lettings_logs[1].uprn_known).to eq(1) + expect(lettings_logs[1].uprn).to eq("1") + expect(lettings_logs[1].uprn_confirmed).to eq(nil) + expect(lettings_logs[1].address_line1).to eq("wrong address line1") + expect(lettings_logs[1].address_line2).to eq("wrong address 2") + expect(lettings_logs[1].town_or_city).to eq("wrong town") + expect(lettings_logs[1].county).to eq("wrong city") + expect(lettings_logs[1].postcode_known).to eq(1) + expect(lettings_logs[1].postcode_full).to eq("A1 1AA") + expect(lettings_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(all_addresses_csv_path) + lettings_logs[2].reload + expect(lettings_logs[2].uprn_known).to eq(1) + expect(lettings_logs[2].uprn).to eq("1") + expect(lettings_logs[2].uprn_confirmed).to eq(nil) + expect(lettings_logs[2].address_line1).to eq("wrong address line1") + expect(lettings_logs[2].address_line2).to eq("wrong address 2") + expect(lettings_logs[2].town_or_city).to eq("wrong town") + expect(lettings_logs[2].county).to eq("wrong city") + expect(lettings_logs[2].postcode_known).to eq(1) + expect(lettings_logs[2].postcode_full).to eq("A1 1AA") + expect(lettings_logs[2].la).to eq("E06000064") + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + + task.invoke(all_addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']") + end + end + end + end + + describe ":import_sales_addresses_from_csv", type: :task do + subject(:task) { Rake::Task["data_import:import_sales_addresses_from_csv"] } + + let(:instance_name) { "paas_import_instance" } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } + let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } + + before do + Rake.application.rake_require("tasks/import_address_from_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:addresses_csv_path) { "addresses_reimport_123.csv" } + let(:all_addresses_csv_path) { "all_addresses_reimport_123.csv" } + let(:wrong_file_path) { "/test/no_csv_here.csv" } + let!(:sales_log) do + create(:sales_log, + :completed, + uprn_known: nil, + uprn: nil, uprn_confirmed: nil, - address_line1: "wrong address line1", - address_line2: "wrong address 2", - town_or_city: "wrong town", - county: "wrong city", - postcode_known: 1, - postcode_full: "A1 1AA", + address_line1: nil, + address_line2: nil, + town_or_city: nil, la: "E06000064", is_la_inferred: true) end + let!(:sales_logs) { create_list(:sales_log, 3, :completed, uprn_known: 1, uprn: "1", la: "E06000064", is_la_inferred: true) } + before do allow(storage_service).to receive(:get_file_io) .with("addresses_reimport_123.csv") - .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read)) - end + .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read)) - it "updates the log address when old address was not given" do - task.invoke(addresses_csv_path) - lettings_log.reload - expect(lettings_log.uprn_known).to eq(0) - expect(lettings_log.uprn).to eq(nil) - expect(lettings_log.uprn_confirmed).to eq(nil) - expect(lettings_log.address_line1).to eq("address 1") - expect(lettings_log.address_line2).to eq("address 2") - expect(lettings_log.town_or_city).to eq("town") - expect(lettings_log.county).to eq(nil) - expect(lettings_log.postcode_known).to eq(1) - expect(lettings_log.postcode_full).to eq("B1 1BB") - expect(lettings_log.la).to eq("E08000035") - expect(lettings_log.is_la_inferred).to eq(true) + allow(storage_service).to receive(:get_file_io) + .with("all_addresses_reimport_123.csv") + .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read)) end - it "updates the log address when old address was given" do - task.invoke(addresses_csv_path) - second_lettings_log.reload - expect(second_lettings_log.uprn_known).to eq(0) - expect(second_lettings_log.uprn).to eq(nil) - expect(second_lettings_log.uprn_confirmed).to eq(nil) - expect(second_lettings_log.address_line1).to eq("address 3") - expect(second_lettings_log.address_line2).to eq("address 4") - expect(second_lettings_log.town_or_city).to eq("city") - expect(second_lettings_log.county).to eq(nil) - expect(second_lettings_log.postcode_known).to eq(1) - expect(second_lettings_log.postcode_full).to eq("B1 1BB") - expect(second_lettings_log.la).to eq("E08000035") - expect(second_lettings_log.is_la_inferred).to eq(true) - end + context "when the file contains issue type column" do + it "updates the log address when old address was not given" do + task.invoke(addresses_csv_path) + sales_log.reload + expect(sales_log.uprn_known).to eq(0) + expect(sales_log.uprn).to eq(nil) + expect(sales_log.uprn_confirmed).to eq(nil) + expect(sales_log.address_line1).to eq("address 1") + expect(sales_log.address_line2).to eq("address 2") + expect(sales_log.town_or_city).to eq("town") + expect(sales_log.county).to eq("county") + expect(sales_log.pcodenk).to eq(0) + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end - it "updates the log address when new address is not given" do - task.invoke(addresses_csv_path) - third_lettings_log.reload - expect(third_lettings_log.uprn_known).to eq(0) - expect(third_lettings_log.uprn).to eq(nil) - expect(third_lettings_log.uprn_confirmed).to eq(nil) - expect(third_lettings_log.address_line1).to eq(nil) - expect(third_lettings_log.address_line2).to eq(nil) - expect(third_lettings_log.town_or_city).to eq(nil) - expect(third_lettings_log.county).to eq(nil) - expect(third_lettings_log.postcode_known).to eq(1) - expect(third_lettings_log.postcode_full).to eq("C11CC") - expect(third_lettings_log.la).to eq(nil) - expect(third_lettings_log.is_la_inferred).to eq(false) - end + it "updates the log address when old address was given" do + task.invoke(addresses_csv_path) + sales_logs[0].reload + expect(sales_logs[0].uprn_known).to eq(0) + expect(sales_logs[0].uprn).to eq(nil) + expect(sales_logs[0].uprn_confirmed).to eq(nil) + expect(sales_logs[0].address_line1).to eq("address 3") + expect(sales_logs[0].address_line2).to eq(nil) + expect(sales_logs[0].town_or_city).to eq("city") + expect(sales_logs[0].county).to eq(nil) + expect(sales_logs[0].pcodenk).to eq(0) + expect(sales_logs[0].postcode_full).to eq("B1 1BB") + expect(sales_logs[0].la).to eq("E08000035") + expect(sales_logs[0].is_la_inferred).to eq(true) + end - it "logs the progress of the update" do - expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB") - expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB") - expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC") - expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD") - expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + it "does not update log address when uprn is given" do + task.invoke(addresses_csv_path) + sales_logs[1].reload + expect(sales_logs[1].uprn_known).to eq(1) + expect(sales_logs[1].uprn).to eq("1") + expect(sales_logs[1].uprn_confirmed).to eq(nil) + expect(sales_logs[1].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[1].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[1].town_or_city).to eq("Westminster") + expect(sales_logs[1].county).to eq(nil) + expect(sales_logs[1].pcodenk).to eq(0) + expect(sales_logs[1].postcode_full).to eq("LS16 6FT") + expect(sales_logs[1].la).to eq("E06000064") + end - task.invoke(addresses_csv_path) + it "does not update log address when all required address fields are not present" do + task.invoke(addresses_csv_path) + sales_logs[2].reload + expect(sales_logs[2].uprn_known).to eq(1) + expect(sales_logs[2].uprn).to eq("1") + expect(sales_logs[2].uprn_confirmed).to eq(nil) + expect(sales_logs[2].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[2].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[2].town_or_city).to eq("Westminster") + expect(sales_logs[2].county).to eq(nil) + expect(sales_logs[2].pcodenk).to eq(0) + expect(sales_logs[2].postcode_full).to eq("LS16 6FT") + expect(sales_logs[2].la).to eq("E06000064") + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a sales log with id fake_id") + + task.invoke(addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']") + end end - it "raises an error when no path is given" do - expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']") + context "when the file does not contain issue type column" do + it "updates the log address when old address was not given" do + task.invoke(all_addresses_csv_path) + sales_log.reload + expect(sales_log.uprn_known).to eq(0) + expect(sales_log.uprn).to eq(nil) + expect(sales_log.uprn_confirmed).to eq(nil) + expect(sales_log.address_line1).to eq("address 1") + expect(sales_log.address_line2).to eq("address 2") + expect(sales_log.town_or_city).to eq("town") + expect(sales_log.county).to eq("county") + expect(sales_log.pcodenk).to eq(0) + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(all_addresses_csv_path) + sales_logs[0].reload + expect(sales_logs[0].uprn_known).to eq(0) + expect(sales_logs[0].uprn).to eq(nil) + expect(sales_logs[0].uprn_confirmed).to eq(nil) + expect(sales_logs[0].address_line1).to eq("address 3") + expect(sales_logs[0].address_line2).to eq(nil) + expect(sales_logs[0].town_or_city).to eq("city") + expect(sales_logs[0].county).to eq(nil) + expect(sales_logs[0].pcodenk).to eq(0) + expect(sales_logs[0].postcode_full).to eq("B1 1BB") + expect(sales_logs[0].la).to eq("E08000035") + expect(sales_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(all_addresses_csv_path) + sales_logs[1].reload + expect(sales_logs[1].uprn_known).to eq(1) + expect(sales_logs[1].uprn).to eq("1") + expect(sales_logs[1].uprn_confirmed).to eq(nil) + expect(sales_logs[1].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[1].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[1].town_or_city).to eq("Westminster") + expect(sales_logs[1].county).to eq(nil) + expect(sales_logs[1].pcodenk).to eq(0) + expect(sales_logs[1].postcode_full).to eq("LS16 6FT") + expect(sales_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(all_addresses_csv_path) + sales_logs[2].reload + expect(sales_logs[2].uprn_known).to eq(1) + expect(sales_logs[2].uprn).to eq("1") + expect(sales_logs[2].uprn_confirmed).to eq(nil) + expect(sales_logs[2].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[2].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[2].town_or_city).to eq("Westminster") + expect(sales_logs[2].county).to eq(nil) + expect(sales_logs[2].pcodenk).to eq(0) + expect(sales_logs[2].postcode_full).to eq("LS16 6FT") + expect(sales_logs[2].la).to eq("E06000064") + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a sales log with id fake_id") + + task.invoke(all_addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']") + end end end end