Update schemes from csv

lib/tasks/update_schemes_and_locations_from_csv.rake

@@ -0,0 +1,92 @@
+namespace :bulk_update do
+  desc "Bulk update scheme data from a csv file"
+  task :update_schemes_from_csv, %i[original_file_name updated_file_name] => :environment do |_task, args|
+    original_file_name = args[:original_file_name]
+    updated_file_name = args[:updated_file_name]
+
+    raise "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']" if original_file_name.blank? || updated_file_name.blank?
+
+    s3_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"])
+    original_file_io = s3_service.get_file_io(original_file_name)
+    original_file_io.set_encoding_by_bom
+    original_schemes_csv = CSV.parse(original_file_io, headers: true)
+
+    updated_file_io = s3_service.get_file_io(updated_file_name)
+    updated_file_io.set_encoding_by_bom
+    updated_schemes_csv = CSV.parse(updated_file_io, headers: true)
+
+    updated_schemes_csv.each do |row|
+      original_attributes = {}
+      updated_attributes = {}
+
+      updated_attributes["scheme_code"] = row[0]
+      updated_attributes["service_name"] = row[1]
+      updated_attributes["status"] = row[2]
+      updated_attributes["sensitive"] = row[3]
+      updated_attributes["scheme_type"] = row[4]
+      updated_attributes["registered_under_care_act"] = row[5]
+      updated_attributes["owning_organisation_name"] = row[6]
+      updated_attributes["arrangement_type"] = row[7]
+      updated_attributes["primary_client_group"] = row[8]
+      updated_attributes["has_other_client_group"] = row[9]
+      updated_attributes["secondary_client_group"] = row[10]
+      updated_attributes["support_type"] = row[11]
+      updated_attributes["intended_stay"] = row[12]
+      updated_attributes["created_at"] = row[13]
+      updated_attributes["active_dates"] = row[14]
+
+      original_row = original_schemes_csv.find { |original_schemes_row| original_schemes_row[0] == updated_attributes["scheme_code"] }
+      if original_row.blank? || original_row["scheme_code"].nil?
+        Rails.logger.info("Scheme with id #{updated_attributes['scheme_code']} is not in the original scheme csv")
+        next
+      end
+
+      original_attributes["scheme_code"] = original_row[0]
+      original_attributes["service_name"] = original_row[1]
+      original_attributes["status"] = original_row[2]
+      original_attributes["sensitive"] = original_row[3]
+      original_attributes["scheme_type"] = original_row[4]
+      original_attributes["registered_under_care_act"] = original_row[5]
+      original_attributes["owning_organisation_name"] = original_row[6]
+      original_attributes["arrangement_type"] = original_row[7]
+      original_attributes["primary_client_group"] = original_row[8]
+      original_attributes["has_other_client_group"] = original_row[9]
+      original_attributes["secondary_client_group"] = original_row[10]
+      original_attributes["support_type"] = original_row[11]
+      original_attributes["intended_stay"] = original_row[12]
+      original_attributes["created_at"] = original_row[13]
+      original_attributes["active_dates"] = original_row[14]
+
+      scheme = Scheme.find_by(id: original_attributes["scheme_code"].delete("S"))
+      if scheme.blank?
+        Rails.logger.info("Scheme with id #{original_attributes['scheme_code']} is not in the database")
+        next
+      end
+
+      updated_attributes.each do |key, value|
+        next unless value != original_attributes[key] && value.present?
+
+        case key
+        when "service_name", "sensitive", "scheme_type", "registered_under_care_act", "arrangement_type", "primary_client_group", "has_other_client_group", "secondary_client_group", "support_type", "intended_stay"
+          begin
+            scheme[key] = value
+            Rails.logger.info("Updating scheme #{original_attributes['scheme_code']}, with #{key}: #{value}")
+          rescue ArgumentError => e
+            Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key}: #{value}. #{e.message}")
+          end
+        when "owning_organisation_name"
+          organisation = Organisation.find_by(name: value)
+          if organisation.present?
+            scheme["owning_organisation_id"] = organisation.id
+            Rails.logger.info("Updating scheme #{original_attributes['scheme_code']}, with owning_organisation: #{organisation.name}")
+          else
+            Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key}: #{value}. Organisation with name #{value} is not in the database")
+          end
+        when "scheme_code", "status", "created_at", "active_dates"
+          Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key} as it it not a permitted field")
+        end
+      end
+      scheme.save!
+    end
+  end
+end

spec/fixtures/files/original_schemes.csv

@@ -0,0 +1,6 @@
+scheme_code,scheme_service_name,scheme_status,scheme_sensitive,scheme_type,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_support_services_provided_by,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,scheme_active_dates
+{id1},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+{id2},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+{id3},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+{id4},Incomplete scheme,incomplete,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+SWrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"

spec/fixtures/files/updated_schemes.csv

@@ -0,0 +1,6 @@
+scheme_code,scheme_service_name,scheme_status,scheme_sensitive,scheme_type,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_support_services_provided_by,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,scheme_active_dates
+{id1},Updated test name,incomplete,No,Direct Access Hostel,No,Different organisation,Another registered stock owner,People with drug problems,No,Older people with support needs,Low level,Permanent,2022-04-01T00:00:00+01:00,"Active from 2 April 2020"
+{id2},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+{id3}, ,active,Yse,Direct access Hostel,Yes – registered care home providing nursing care,non existing org,wrong answer,FD,no,lder people with support needs,high,Permanent ,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+Wrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,No,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"
+SWrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,No,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020"

spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb

@@ -0,0 +1,188 @@
+require "rails_helper"
+require "rake"
+
+RSpec.describe "bulk_update" do
+  def replace_entity_ids(scheme_1, scheme_2, scheme_3, _incomplete_scheme, export_template)
+    export_template.sub!(/\{id1\}/, "S#{scheme_1.id}")
+    export_template.sub!(/\{id2\}/, "S#{scheme_2.id}")
+    export_template.sub!(/\{id3\}/, "S#{scheme_3.id}")
+    # export_template.sub!(/\{id4\}/, "S#{incomplete_scheme.id}")
+  end
+
+  before do
+    allow(Storage::S3Service).to receive(:new).and_return(storage_service)
+    allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service)
+    allow(ENV).to receive(:[])
+    allow(ENV).to receive(:[]).with("CSV_DOWNLOAD_PAAS_INSTANCE").and_return(instance_name)
+
+    WebMock.stub_request(:get, /api\.postcodes\.io/)
+      .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {})
+    WebMock.stub_request(:get, /api\.postcodes\.io\/postcodes\/B11BB/)
+      .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {})
+  end
+
+  describe ":update_schemes_from_csv", type: :task do
+    subject(:task) { Rake::Task["bulk_update:update_schemes_from_csv"] }
+
+    let(:instance_name) { "import_instance" }
+    let(:storage_service) { instance_double(Storage::S3Service) }
+    let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) }
+
+    before do
+      Rake.application.rake_require("tasks/update_schemes_and_locations_from_csv")
+      Rake::Task.define_task(:environment)
+      task.reenable
+    end
+
+    context "when the rake task is run" do
+      let(:original_schemes_csv_path) { "original_schemes.csv" }
+      let(:updated_schemes_csv_path) { "updated_schemes.csv" }
+      let(:wrong_file_path) { "/test/no_csv_here.csv" }
+      let!(:different_organisation) { FactoryBot.create(:organisation, name: "Different organisation") }
+      let(:schemes) do
+        create_list(:scheme,
+                    3,
+                    service_name: "Test name",
+                    sensitive: 1,
+                    registered_under_care_act: 4,
+                    support_type: 4,
+                    scheme_type: 7,
+                    arrangement_type: "D",
+                    intended_stay: "M",
+                    primary_client_group: "G",
+                    secondary_client_group: "M",
+                    has_other_client_group: 1,
+                    owning_organisation: FactoryBot.create(:organisation),
+                    confirmed: true,
+                    created_at: Time.zone.local(2021, 4, 1),
+                    total_units: 2)
+      end
+
+      let(:incomplete_scheme) do
+        build(:scheme,
+              service_name: "Incomplete scheme",
+              sensitive: 1,
+              registered_under_care_act: 4,
+              support_type: nil,
+              scheme_type: 7,
+              arrangement_type: "D",
+              intended_stay: nil,
+              primary_client_group: "G",
+              secondary_client_group: nil,
+              has_other_client_group: nil,
+              owning_organisation: FactoryBot.create(:organisation),
+              created_at: Time.zone.local(2021, 4, 1),
+              total_units: 2)
+      end
+
+      before do
+        incomplete_scheme.save!(validate: false)
+        allow(storage_service).to receive(:get_file_io)
+        .with("original_schemes.csv")
+        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/original_schemes.csv").read)))
+
+        allow(storage_service).to receive(:get_file_io)
+        .with("updated_schemes.csv")
+        .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/updated_schemes.csv").read)))
+      end
+
+      it "updates the allowed scheme fields if they have changed and doesn't update other fields" do
+        task.invoke(original_schemes_csv_path, updated_schemes_csv_path)
+        schemes[0].reload
+        expect(schemes[0].service_name).to eq("Updated test name")
+        expect(schemes[0].sensitive).to eq("No")
+        expect(schemes[0].registered_under_care_act).to eq("No")
+        expect(schemes[0].support_type).to eq("Low level")
+        expect(schemes[0].scheme_type).to eq("Direct Access Hostel")
+        expect(schemes[0].arrangement_type).to eq("Another registered stock owner")
+        expect(schemes[0].intended_stay).to eq("Permanent")
+        expect(schemes[0].primary_client_group).to eq("People with drug problems")
+        # expect(schemes[0].secondary_client_group).to eq(nil)
+        expect(schemes[0].has_other_client_group).to eq("No")
+        expect(schemes[0].owning_organisation).to eq(different_organisation)
+        expect(schemes[0].created_at).to eq(Time.zone.local(2021, 4, 1))
+        expect(schemes[0].total_units).to eq(2)
+      end
+
+      it "does not update the scheme if it hasn't changed" do
+        task.invoke(original_schemes_csv_path, updated_schemes_csv_path)
+        schemes[1].reload
+        expect(schemes[1].service_name).to eq("Test name")
+        expect(schemes[1].sensitive).to eq("Yes")
+        expect(schemes[1].registered_under_care_act).to eq("Yes – registered care home providing nursing care")
+        expect(schemes[1].support_type).to eq("High level")
+        expect(schemes[1].scheme_type).to eq("Housing for older people")
+        expect(schemes[1].arrangement_type).to eq("The same organisation that owns the housing stock")
+        expect(schemes[1].intended_stay).to eq("Medium stay")
+        expect(schemes[1].primary_client_group).to eq("People with alcohol problems")
+        expect(schemes[1].secondary_client_group).to eq("Older people with support needs")
+        expect(schemes[1].has_other_client_group).to eq("Yes")
+        expect(schemes[1].owning_organisation).not_to eq(different_organisation)
+        expect(schemes[1].created_at).to eq(Time.zone.local(2021, 4, 1))
+        expect(schemes[1].total_units).to eq(2)
+      end
+
+      it "does not update the scheme with invalid values" do
+        task.invoke(original_schemes_csv_path, updated_schemes_csv_path)
+        schemes[2].reload
+        expect(schemes[2].service_name).to eq("Test name")
+        expect(schemes[2].sensitive).to eq("Yes")
+        expect(schemes[2].registered_under_care_act).to eq("Yes – registered care home providing nursing care")
+        expect(schemes[2].support_type).to eq("High level")
+        expect(schemes[2].scheme_type).to eq("Housing for older people")
+        expect(schemes[2].arrangement_type).to eq("The same organisation that owns the housing stock")
+        expect(schemes[2].intended_stay).to eq("Medium stay")
+        expect(schemes[2].primary_client_group).to eq("People with alcohol problems")
+        expect(schemes[2].secondary_client_group).to eq("Older people with support needs")
+        expect(schemes[2].has_other_client_group).to eq("Yes")
+        expect(schemes[2].owning_organisation).not_to eq(different_organisation)
+        expect(schemes[2].created_at).to eq(Time.zone.local(2021, 4, 1))
+        expect(schemes[2].total_units).to eq(2)
+      end
+
+      it "logs the progress of the update" do
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with service_name: Updated test name")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with sensitive: No")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with scheme_type: Direct Access Hostel")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with arrangement_type: Another registered stock owner")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with primary_client_group: People with drug problems")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with has_other_client_group: No")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with support_type: Low level")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with intended_stay: Permanent")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with registered_under_care_act: No")
+        # expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with secondary_client_group: nil")
+        expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with owning_organisation: Different organisation")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with status as it it not a permitted field")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with created_at as it it not a permitted field")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with active_dates as it it not a permitted field")
+
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with sensitive: Yse. 'Yse' is not a valid sensitive")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with scheme_type: Direct access Hostel. 'Direct access Hostel' is not a valid scheme_type")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with owning_organisation_name: non existing org. Organisation with name non existing org is not in the database")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with arrangement_type: wrong answer. 'wrong answer' is not a valid arrangement_type")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with primary_client_group: FD. 'FD' is not a valid primary_client_group")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with has_other_client_group: no. 'no' is not a valid has_other_client_group")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with secondary_client_group: lder people with support needs. 'lder people with support needs' is not a valid secondary_client_group")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with support_type: high. 'high' is not a valid support_type")
+        expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with intended_stay: Permanent . 'Permanent ' is not a valid intended_stay")
+
+        expect(Rails.logger).to receive(:info).with("Scheme with id Wrong_id is not in the original scheme csv")
+        expect(Rails.logger).to receive(:info).with("Scheme with id SWrong_id is not in the database")
+
+        task.invoke(original_schemes_csv_path, updated_schemes_csv_path)
+      end
+
+      it "raises an error when no paths are given" do
+        expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']")
+      end
+
+      it "raises an error when no original path is given" do
+        expect { task.invoke(nil, updated_schemes_csv_path) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']")
+      end
+
+      it "raises an error when no updated path is given" do
+        expect { task.invoke(original_schemes_csv_path, nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']")
+      end
+    end
+  end
+end