update locations from csv

lib/tasks/update_schemes_and_locations_from_csv.rake

@@ -94,4 +94,106 @@ namespace :bulk_update do
       end
     end
   end
+
+  desc "Bulk update location data from a csv file"
+  task :update_locations_from_csv, %i[original_file_name updated_file_name] => :environment do |_task, args|
+    original_file_name = args[:original_file_name]
+    updated_file_name = args[:updated_file_name]
+
+    raise "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']" if original_file_name.blank? || updated_file_name.blank?
+
+    s3_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"])
+    original_file_io = s3_service.get_file_io(original_file_name)
+    original_file_io.set_encoding_by_bom
+    original_locations_csv = CSV.parse(original_file_io, headers: true)
+
+    updated_file_io = s3_service.get_file_io(updated_file_name)
+    updated_file_io.set_encoding_by_bom
+    updated_locations_csv = CSV.parse(updated_file_io, headers: true)
+
+    updated_locations_csv.each do |row|
+      original_attributes = {}
+      updated_attributes = {}
+
+      updated_attributes["scheme_code"] = row[0]
+      updated_attributes["location_code"] = row[1]
+      updated_attributes["postcode"] = row[2]
+      updated_attributes["name"] = row[3]
+      updated_attributes["status"] = row[4]
+      updated_attributes["location_admin_district"] = row[5]
+      updated_attributes["units"] = row[6]
+      updated_attributes["type_of_unit"] = row[7]
+      updated_attributes["mobility_type"] = row[8]
+      updated_attributes["active_dates"] = row[9]
+
+      original_row = original_locations_csv.find { |original_locations_row| original_locations_row[1] == updated_attributes["location_code"] }
+      if original_row.blank? || original_row["location_code"].nil?
+        Rails.logger.info("Location with id #{updated_attributes['location_code']} is not in the original location csv")
+        next
+      end
+
+      original_attributes["scheme_code"] = original_row[0]
+      original_attributes["location_code"] = original_row[1]
+      original_attributes["postcode"] = original_row[2]
+      original_attributes["name"] = original_row[3]
+      original_attributes["status"] = original_row[4]
+      original_attributes["location_admin_district"] = original_row[5]
+      original_attributes["units"] = original_row[6]
+      original_attributes["type_of_unit"] = original_row[7]
+      original_attributes["mobility_type"] = original_row[8]
+      original_attributes["active_dates"] = original_row[9]
+
+      location = Location.find_by(id: original_attributes["location_code"])
+      if location.blank?
+        Rails.logger.info("Location with id #{original_attributes['location_code']} is not in the database")
+        next
+      end
+
+      updated_attributes.each do |key, value|
+        next unless value != original_attributes[key] && value.present?
+
+        case key
+        when "location_admin_district"
+          location_code = Location.local_authorities_for_current_year.key(value)
+          if location_code.present?
+            location.location_code = location_code
+            location.location_admin_district = value
+            Rails.logger.info("Updating location #{original_attributes['location_code']}, with location_code: #{location_code}")
+          else
+            Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. Location admin distrint #{value} is not a valid option")
+          end
+        when "postcode"
+          if !value&.match(POSTCODE_REGEXP)
+            Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. #{I18n.t('validations.postcode')}")
+          else
+            location.postcode = PostcodeService.clean(value)
+            Rails.logger.info("Updating location #{original_attributes['location_code']}, with postcode: #{value}")
+          end
+        when "name", "units", "type_of_unit", "mobility_type"
+          begin
+            location[key] = value
+            Rails.logger.info("Updating location #{original_attributes['location_code']}, with #{key}: #{value}")
+          rescue ArgumentError => e
+            Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. #{e.message}")
+          end
+        when "scheme_code"
+          scheme = Scheme.find_by(id: value.delete("S"))
+          if scheme.present?
+            location["scheme_id"] = scheme.id
+            Rails.logger.info("Updating location #{original_attributes['location_code']}, with scheme: S#{scheme.id}")
+          else
+            Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key}: #{value}. Scheme with id #{value} is not in the database")
+          end
+        when "location_code", "status", "active_dates"
+          Rails.logger.info("Cannot update location #{original_attributes['location_code']} with #{key} as it it not a permitted field")
+        end
+      end
+      begin
+        location.save!
+        Rails.logger.info("Saved location #{original_attributes['location_code']}.")
+      rescue ActiveRecord::RecordInvalid => e
+        Rails.logger.error("Cannot update location #{original_attributes['location_code']}. #{e.message}")
+      end
+    end
+  end
 end

spec/fixtures/files/original_locations.csv

@@ -0,0 +1,5 @@
+scheme_code,location_code,location_postcode,location_name,location_status,location_local_authority,location_units,location_type_of_unit,location_mobility_type,location_active_dates
+{scheme_id1},{id1},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022"
+{scheme_id2},{id2},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022"
+{scheme_id3},{id3},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022"
+1,SWrong_id,SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022"

spec/fixtures/files/updated_locations.csv

@@ -0,0 +1,6 @@
+scheme_code,location_code,location_postcode,location_name,location_status,location_local_authority,location_units,location_type_of_unit,location_mobility_type,location_active_dates
+{scheme_id1},{id1},B11BB,Updated name,deactivating_soon,Westminster,10,Bungalow,Wheelchair-user standard,"Active from 1 April 2028"
+{scheme_id2},{id2},SW1A 2AA,Downing Street,active,Westminster,20,Self-contained house,Fitted with equipment and adaptations,"Active from 1 April 2022"
+{scheme_id3},{id3},SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022"
+x,Wrong_id,SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022"
+x,SWrong_id,SWAAA,,deactivating_soon,Westminst,,elf-contained house,55,"Active from 1 April 2022"

spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb

@@ -2,11 +2,19 @@ require "rails_helper"
 require "rake"
 
 RSpec.describe "bulk_update" do
-  def replace_entity_ids(scheme_1, scheme_2, scheme_3, _incomplete_scheme, export_template)
+  def replace_entity_ids(scheme_1, scheme_2, scheme_3, export_template)
     export_template.sub!(/\{id1\}/, "S#{scheme_1.id}")
     export_template.sub!(/\{id2\}/, "S#{scheme_2.id}")
     export_template.sub!(/\{id3\}/, "S#{scheme_3.id}")
-    # export_template.sub!(/\{id4\}/, "S#{incomplete_scheme.id}")
+  end
+
+  def replace_entity_ids_for_locations(location_1, location_2, location_3, scheme_1, scheme_2, scheme_3, export_template)
+    export_template.sub!(/\{id1\}/, location_1.id.to_s)
+    export_template.sub!(/\{id2\}/, location_2.id.to_s)
+    export_template.sub!(/\{id3\}/, location_3.id.to_s)
+    export_template.sub!(/\{scheme_id1\}/, "S#{scheme_1['id']}")
+    export_template.sub!(/\{scheme_id2\}/, "S#{scheme_2['id']}")
+    export_template.sub!(/\{scheme_id3\}/, "S#{scheme_3['id']}")
   end
 
   before do
@@ -58,32 +66,14 @@ RSpec.describe "bulk_update" do
                     total_units: 2)
       end
 
-      let(:incomplete_scheme) do
-        build(:scheme,
-              service_name: "Incomplete scheme",
-              sensitive: 1,
-              registered_under_care_act: 4,
-              support_type: nil,
-              scheme_type: 7,
-              arrangement_type: "D",
-              intended_stay: nil,
-              primary_client_group: "G",
-              secondary_client_group: nil,
-              has_other_client_group: nil,
-              owning_organisation: FactoryBot.create(:organisation),
-              created_at: Time.zone.local(2021, 4, 1),
-              total_units: 2)
-      end
-
       before do
-        incomplete_scheme.save!(validate: false)
         allow(storage_service).to receive(:get_file_io)
         .with("original_schemes.csv")
-        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/original_schemes.csv").read)))
+        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], File.open("./spec/fixtures/files/original_schemes.csv").read)))
 
         allow(storage_service).to receive(:get_file_io)
         .with("updated_schemes.csv")
-        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/updated_schemes.csv").read)))
+        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], File.open("./spec/fixtures/files/updated_schemes.csv").read)))
       end
 
       it "updates the allowed scheme fields if they have changed and doesn't update other fields" do
@@ -228,4 +218,130 @@ RSpec.describe "bulk_update" do
       end
     end
   end
+
+  describe ":update_locations_from_csv", type: :task do
+    subject(:task) { Rake::Task["bulk_update:update_locations_from_csv"] }
+
+    let(:instance_name) { "import_instance" }
+    let(:storage_service) { instance_double(Storage::S3Service) }
+    let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
+
+    before do
+      Rake.application.rake_require("tasks/update_schemes_and_locations_from_csv")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when the rake task is run" do
+      let(:original_locations_csv_path) { "original_locations.csv" }
+      let(:updated_locations_csv_path) { "updated_locations.csv" }
+      let(:wrong_file_path) { "/test/no_csv_here.csv" }
+      let!(:scheme) { FactoryBot.create(:scheme, service_name: "Scheme 1") }
+      let!(:different_scheme) { FactoryBot.create(:scheme, service_name: "Different scheme") }
+
+      let(:locations) do
+        create_list(:location,
+                    3,
+                    postcode: "SW1A 2AA",
+                    name: "Downing Street",
+                    type_of_unit: "Self-contained house",
+                    units: 20,
+                    mobility_type: "Fitted with equipment and adaptations",
+                    location_code: "E09000033",
+                    location_admin_district: "Westminster",
+                    startdate: Time.zone.local(2022, 4, 1),
+                    confirmed: true,
+                    scheme:)
+      end
+
+      before do
+        allow(storage_service).to receive(:get_file_io)
+        .with("original_locations.csv")
+        .and_return(StringIO.new(replace_entity_ids_for_locations(locations[0], locations[1], locations[2], scheme, scheme, scheme, File.open("./spec/fixtures/files/original_locations.csv").read)))
+
+        allow(storage_service).to receive(:get_file_io)
+        .with("updated_locations.csv")
+        .and_return(StringIO.new(replace_entity_ids_for_locations(locations[0], locations[1], locations[2], different_scheme, scheme, { id: "non existent scheme id" }, File.open("./spec/fixtures/files/updated_locations.csv").read)))
+      end
+
+      it "updates the allowed location fields if they have changed and doesn't update other fields" do
+        task.invoke(original_locations_csv_path, updated_locations_csv_path)
+        locations[0].reload
+        expect(locations[0].postcode).to eq("B1 1BB")
+        expect(locations[0].name).to eq("Updated name")
+        expect(locations[0].type_of_unit).to eq("Bungalow")
+        expect(locations[0].units).to eq(10)
+        expect(locations[0].mobility_type).to eq("Wheelchair-user standard")
+        expect(locations[0].location_code).to eq("E08000035")
+        expect(locations[0].location_admin_district).to eq("Westminster")
+        expect(locations[0].scheme).to eq(different_scheme)
+      end
+
+      it "does not update the location if it hasn't changed" do
+        task.invoke(original_locations_csv_path, updated_locations_csv_path)
+        locations[1].reload
+        expect(locations[1].postcode).to eq("SW1A 2AA")
+        expect(locations[1].name).to eq("Downing Street")
+        expect(locations[1].type_of_unit).to eq("Self-contained house")
+        expect(locations[1].units).to eq(20)
+        expect(locations[1].mobility_type).to eq("Fitted with equipment and adaptations")
+        expect(locations[1].location_code).to eq("E09000033")
+        expect(locations[1].location_admin_district).to eq("Westminster")
+        expect(locations[1].scheme).to eq(scheme)
+      end
+
+      it "does not update the location with invalid values" do
+        task.invoke(original_locations_csv_path, updated_locations_csv_path)
+        locations[2].reload
+        expect(locations[2].postcode).to eq("SW1A 2AA")
+        expect(locations[2].name).to eq("Downing Street")
+        expect(locations[2].type_of_unit).to eq("Self-contained house")
+        expect(locations[2].units).to eq(20)
+        expect(locations[2].mobility_type).to eq("Fitted with equipment and adaptations")
+        expect(locations[2].location_code).to eq("E09000033")
+        expect(locations[2].location_admin_district).to eq("Westminster")
+        expect(locations[2].scheme).to eq(scheme)
+      end
+
+      it "logs the progress of the update" do
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with postcode: B11BB")
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with name: Updated name")
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with type_of_unit: Bungalow")
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with units: 10")
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with mobility_type: Wheelchair-user standard")
+        expect(Rails.logger).to receive(:info).with("Updating location #{locations[0].id}, with scheme: S#{different_scheme.id}")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[0].id} with status as it it not a permitted field")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[0].id} with active_dates as it it not a permitted field")
+        expect(Rails.logger).to receive(:info).with("Saved location #{locations[0].id}.")
+
+        expect(Rails.logger).to receive(:info).with("Saved location #{locations[1].id}.")
+
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with postcode: SWAAA. Enter a postcode in the correct format, for example AA1 1AA")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with scheme_code: S. Scheme with id S is not in the database")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with location_admin_district: Westminst. Location admin distrint Westminst is not a valid option")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with type_of_unit: elf-contained house. 'elf-contained house' is not a valid type_of_unit")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with mobility_type: 55. '55' is not a valid mobility_type")
+        expect(Rails.logger).to receive(:info).with("Cannot update location #{locations[2].id} with status as it it not a permitted field")
+
+        expect(Rails.logger).to receive(:info).with("Saved location #{locations[2].id}.")
+
+        expect(Rails.logger).to receive(:info).with("Location with id Wrong_id is not in the original location csv")
+        expect(Rails.logger).to receive(:info).with("Location with id SWrong_id is not in the database")
+
+        task.invoke(original_locations_csv_path, updated_locations_csv_path)
+      end
+
+      it "raises an error when no paths are given" do
+        expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']")
+      end
+
+      it "raises an error when no original path is given" do
+        expect { task.invoke(nil, updated_locations_csv_path) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']")
+      end
+
+      it "raises an error when no updated path is given" do
+        expect { task.invoke(original_locations_csv_path, nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_locations_from_csv['original_file_name','updated_file_name']")
+      end
+    end
+  end
 end