diff --git a/lib/tasks/update_schemes_and_locations_from_csv.rake b/lib/tasks/update_schemes_and_locations_from_csv.rake index a52d600c5..d220d8069 100644 --- a/lib/tasks/update_schemes_and_locations_from_csv.rake +++ b/lib/tasks/update_schemes_and_locations_from_csv.rake @@ -94,4 +94,106 @@ namespace :bulk_update do end end end + + desc "Bulk update location data from a csv file" + task :update_locations_from_csv, %i[original_file_name updated_file_name] => :environment do |_task, args| + original_file_name = args[:original_file_name] + updated_file_name = args[:updated_file_name] + + raise "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']" if original_file_name.blank? || updated_file_name.blank? + + s3_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + original_file_io = s3_service.get_file_io(original_file_name) + original_file_io.set_encoding_by_bom + original_locations_csv = CSV.parse(original_file_io, headers: true) + + updated_file_io = s3_service.get_file_io(updated_file_name) + updated_file_io.set_encoding_by_bom + updated_locations_csv = CSV.parse(updated_file_io, headers: true) + + updated_locations_csv.each do |row| + original_attributes = {} + updated_attributes = {} + + updated_attributes["scheme_code"] = row[0] + updated_attributes["location_code"] = row[1] + updated_attributes["postcode"] = row[2] + updated_attributes["name"] = row[3] + updated_attributes["status"] = row[4] + updated_attributes["location_admin_district"] = row[5] + updated_attributes["units"] = row[6] + updated_attributes["type_of_unit"] = row[7] + updated_attributes["mobility_type"] = row[8] + updated_attributes["active_dates"] = row[9] + + original_row = original_locations_csv.find { |original_locations_row| original_locations_row[1] == updated_attributes["location_code"] } + if original_row.blank? || original_row["location_code"].nil? + Rails.logger.info("Location with id #{updated_attributes['location_code']} is not in the original location csv") + next + end + + original_attributes["scheme_code"] = original_row[0] + original_attributes["location_code"] = original_row[1] + original_attributes["postcode"] = original_row[2] + original_attributes["name"] = original_row[3] + original_attributes["status"] = original_row[4] + original_attributes["location_admin_district"] = original_row[5] + original_attributes["units"] = original_row[6] + original_attributes["type_of_unit"] = original_row[7] + original_attributes["mobility_type"] = original_row[8] + original_attributes["active_dates"] = original_row[9] + + location = Location.find_by(id: original_attributes["location_code"]) + if location.blank? + Rails.logger.info("Location with id #{original_attributes['location_code']} is not in the database") + next + end + + updated_attributes.each do |key, value| + next unless value != original_attributes[key] && value.present? + + case key + when "location_admin_district" + location_code = Location.local_authorities_for_current_year.key(value) + if location_code.present? + location.location_code = location_code + location.location_admin_district = value + Rails.logger.info("Updating location #{original_attributes['location_code']}, with location_code: #{location_code}") + else + Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. Location admin distrint #{value} is not a valid option") + end + when "postcode" + if !value&.match(POSTCODE_REGEXP) + Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. #{I18n.t('validations.postcode')}") + else + location.postcode = PostcodeService.clean(value) + Rails.logger.info("Updating location #{original_attributes['location_code']}, with postcode: #{value}") + end + when "name", "units", "type_of_unit", "mobility_type" + begin + location[key] = value + Rails.logger.info("Updating location #{original_attributes['location_code']}, with #{key}: #{value}") + rescue ArgumentError => e + Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. #{e.message}") + end + when "scheme_code" + scheme = Scheme.find_by(id: value.delete("S")) + if scheme.present? + location["scheme_id"] = scheme.id + Rails.logger.info("Updating location #{original_attributes['location_code']}, with scheme: S#{scheme.id}") + else + Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. Scheme with id #{value} is not in the database") + end + when "location_code", "status", "active_dates" + Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key} as it it not a permitted field") + end + end + begin + location.save! + Rails.logger.info("Saved location #{original_attributes['location_code']}.") + rescue ActiveRecord::RecordInvalid => e + Rails.logger.error("Cannot update location #{original_attributes['location_code']}. #{e.message}") + end + end + end end diff --git a/spec/fixtures/files/original_locations.csv b/spec/fixtures/files/original_locations.csv new file mode 100644 index 000000000..13370ec13 --- /dev/null +++ b/spec/fixtures/files/original_locations.csv @@ -0,0 +1,5 @@ +scheme_code,location_code,location_postcode,location_name,location_status,location_local_authority,location_units,location_type_of_unit,location_mobility_type,location_active_dates +{scheme_id1},{id1},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022" +{scheme_id2},{id2},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022" +{scheme_id3},{id3},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022" +1,SWrong_id,SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022" diff --git a/spec/fixtures/files/updated_locations.csv b/spec/fixtures/files/updated_locations.csv new file mode 100644 index 000000000..7f5237fab --- /dev/null +++ b/spec/fixtures/files/updated_locations.csv @@ -0,0 +1,6 @@ +scheme_code,location_code,location_postcode,location_name,location_status,location_local_authority,location_units,location_type_of_unit,location_mobility_type,location_active_dates +{scheme_id1},{id1},B11BB,Updated name,deactivating_soon,Westminster,10,Bungalow,Wheelchair-user standard,"Active from 1 April 2028" +{scheme_id2},{id2},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022" +{scheme_id3},{id3},SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022" +x,Wrong_id,SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022" +x,SWrong_id,SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022" diff --git a/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb b/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb index eb98a1ab6..9206fbe47 100644 --- a/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb +++ b/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb @@ -2,11 +2,19 @@ require "rails_helper" require "rake" RSpec.describe "bulk_update" do - def replace_entity_ids(scheme_1, scheme_2, scheme_3, _incomplete_scheme, export_template) + def replace_entity_ids(scheme_1, scheme_2, scheme_3, export_template) export_template.sub!(/\{id1\}/, "S#{scheme_1.id}") export_template.sub!(/\{id2\}/, "S#{scheme_2.id}") export_template.sub!(/\{id3\}/, "S#{scheme_3.id}") - # export_template.sub!(/\{id4\}/, "S#{incomplete_scheme.id}") + end + + def replace_entity_ids_for_locations(location_1, location_2, location_3, scheme_1, scheme_2, scheme_3, export_template) + export_template.sub!(/\{id1\}/, location_1.id.to_s) + export_template.sub!(/\{id2\}/, location_2.id.to_s) + export_template.sub!(/\{id3\}/, location_3.id.to_s) + export_template.sub!(/\{scheme_id1\}/, "S#{scheme_1['id']}") + export_template.sub!(/\{scheme_id2\}/, "S#{scheme_2['id']}") + export_template.sub!(/\{scheme_id3\}/, "S#{scheme_3['id']}") end before do @@ -58,32 +66,14 @@ RSpec.describe "bulk_update" do total_units: 2) end - let(:incomplete_scheme) do - build(:scheme, - service_name: "Incomplete scheme", - sensitive: 1, - registered_under_care_act: 4, - support_type: nil, - scheme_type: 7, - arrangement_type: "D", - intended_stay: nil, - primary_client_group: "G", - secondary_client_group: nil, - has_other_client_group: nil, - owning_organisation: FactoryBot.create(:organisation), - created_at: Time.zone.local(2021, 4, 1), - total_units: 2) - end - before do - incomplete_scheme.save!(validate: false) allow(storage_service).to receive(:get_file_io) .with("original_schemes.csv") - .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/original_schemes.csv").read))) + .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], File.open("./spec/fixtures/files/original_schemes.csv").read))) allow(storage_service).to receive(:get_file_io) .with("updated_schemes.csv") - .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/updated_schemes.csv").read))) + .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], File.open("./spec/fixtures/files/updated_schemes.csv").read))) end it "updates the allowed scheme fields if they have changed and doesn't update other fields" do @@ -228,4 +218,130 @@ RSpec.describe "bulk_update" do end end end + + describe ":update_locations_from_csv", type: :task do + subject(:task) { Rake::Task["bulk_update:update_locations_from_csv"] } + + let(:instance_name) { "import_instance" } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } + + before do + Rake.application.rake_require("tasks/update_schemes_and_locations_from_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:original_locations_csv_path) { "original_locations.csv" } + let(:updated_locations_csv_path) { "updated_locations.csv" } + let(:wrong_file_path) { "/test/no_csv_here.csv" } + let!(:scheme) { FactoryBot.create(:scheme, service_name: "Scheme 1") } + let!(:different_scheme) { FactoryBot.create(:scheme, service_name: "Different scheme") } + + let(:locations) do + create_list(:location, + 3, + postcode: "SW1A 2AA", + name: "Downing Street", + type_of_unit: "Self-contained house", + units: 20, + mobility_type: "Fitted with equipment and adaptations", + location_code: "E09000033", + location_admin_district: "Westminster", + startdate: Time.zone.local(2022, 4, 1), + confirmed: true, + scheme:) + end + + before do + allow(storage_service).to receive(:get_file_io) + .with("original_locations.csv") + .and_return(StringIO.new(replace_entity_ids_for_locations(locations[0], locations[1], locations[2], scheme, scheme, scheme, File.open("./spec/fixtures/files/original_locations.csv").read))) + + allow(storage_service).to receive(:get_file_io) + .with("updated_locations.csv") + .and_return(StringIO.new(replace_entity_ids_for_locations(locations[0], locations[1], locations[2], different_scheme, scheme, { id: "non existent scheme id" }, File.open("./spec/fixtures/files/updated_locations.csv").read))) + end + + it "updates the allowed location fields if they have changed and doesn't update other fields" do + task.invoke(original_locations_csv_path, updated_locations_csv_path) + locations[0].reload + expect(locations[0].postcode).to eq("B1 1BB") + expect(locations[0].name).to eq("Updated name") + expect(locations[0].type_of_unit).to eq("Bungalow") + expect(locations[0].units).to eq(10) + expect(locations[0].mobility_type).to eq("Wheelchair-user standard") + expect(locations[0].location_code).to eq("E08000035") + expect(locations[0].location_admin_district).to eq("Westminster") + expect(locations[0].scheme).to eq(different_scheme) + end + + it "does not update the location if it hasn't changed" do + task.invoke(original_locations_csv_path, updated_locations_csv_path) + locations[1].reload + expect(locations[1].postcode).to eq("SW1A 2AA") + expect(locations[1].name).to eq("Downing Street") + expect(locations[1].type_of_unit).to eq("Self-contained house") + expect(locations[1].units).to eq(20) + expect(locations[1].mobility_type).to eq("Fitted with equipment and adaptations") + expect(locations[1].location_code).to eq("E09000033") + expect(locations[1].location_admin_district).to eq("Westminster") + expect(locations[1].scheme).to eq(scheme) + end + + it "does not update the location with invalid values" do + task.invoke(original_locations_csv_path, updated_locations_csv_path) + locations[2].reload + expect(locations[2].postcode).to eq("SW1A 2AA") + expect(locations[2].name).to eq("Downing Street") + expect(locations[2].type_of_unit).to eq("Self-contained house") + expect(locations[2].units).to eq(20) + expect(locations[2].mobility_type).to eq("Fitted with equipment and adaptations") + expect(locations[2].location_code).to eq("E09000033") + expect(locations[2].location_admin_district).to eq("Westminster") + expect(locations[2].scheme).to eq(scheme) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with postcode: B11BB") + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with name: Updated name") + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with type_of_unit: Bungalow") + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with units: 10") + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with mobility_type: Wheelchair-user standard") + expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with scheme: S#{different_scheme.id}") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[0].id} with status as it it not a permitted field") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[0].id} with active_dates as it it not a permitted field") + expect(Rails.logger).to receive(:info).with("Saved location #{locations[0].id}.") + + expect(Rails.logger).to receive(:info).with("Saved location #{locations[1].id}.") + + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with postcode: SWAAA. Enter a postcode in the correct format, for example AA1 1AA") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with scheme_code: S. Scheme with id S is not in the database") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with location_admin_district: Westminst. Location admin distrint Westminst is not a valid option") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with type_of_unit: elf-contained house. 'elf-contained house' is not a valid type_of_unit") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with mobility_type: 55. '55' is not a valid mobility_type") + expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with status as it it not a permitted field") + + expect(Rails.logger).to receive(:info).with("Saved location #{locations[2].id}.") + + expect(Rails.logger).to receive(:info).with("Location with id Wrong_id is not in the original location csv") + expect(Rails.logger).to receive(:info).with("Location with id SWrong_id is not in the database") + + task.invoke(original_locations_csv_path, updated_locations_csv_path) + end + + it "raises an error when no paths are given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']") + end + + it "raises an error when no original path is given" do + expect { task.invoke(nil, updated_locations_csv_path) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']") + end + + it "raises an error when no updated path is given" do + expect { task.invoke(original_locations_csv_path, nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']") + end + end + end end