diff --git a/lib/tasks/update_schemes_and_locations_from_csv.rake b/lib/tasks/update_schemes_and_locations_from_csv.rake new file mode 100644 index 000000000..5f2107d91 --- /dev/null +++ b/lib/tasks/update_schemes_and_locations_from_csv.rake @@ -0,0 +1,92 @@ +namespace :bulk_update do + desc "Bulk update scheme data from a csv file" + task :update_schemes_from_csv, %i[original_file_name updated_file_name] => :environment do |_task, args| + original_file_name = args[:original_file_name] + updated_file_name = args[:updated_file_name] + + raise "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']" if original_file_name.blank? || updated_file_name.blank? + + s3_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + original_file_io = s3_service.get_file_io(original_file_name) + original_file_io.set_encoding_by_bom + original_schemes_csv = CSV.parse(original_file_io, headers: true) + + updated_file_io = s3_service.get_file_io(updated_file_name) + updated_file_io.set_encoding_by_bom + updated_schemes_csv = CSV.parse(updated_file_io, headers: true) + + updated_schemes_csv.each do |row| + original_attributes = {} + updated_attributes = {} + + updated_attributes["scheme_code"] = row[0] + updated_attributes["service_name"] = row[1] + updated_attributes["status"] = row[2] + updated_attributes["sensitive"] = row[3] + updated_attributes["scheme_type"] = row[4] + updated_attributes["registered_under_care_act"] = row[5] + updated_attributes["owning_organisation_name"] = row[6] + updated_attributes["arrangement_type"] = row[7] + updated_attributes["primary_client_group"] = row[8] + updated_attributes["has_other_client_group"] = row[9] + updated_attributes["secondary_client_group"] = row[10] + updated_attributes["support_type"] = row[11] + updated_attributes["intended_stay"] = row[12] + updated_attributes["created_at"] = row[13] + updated_attributes["active_dates"] = row[14] + + original_row = original_schemes_csv.find { |original_schemes_row| original_schemes_row[0] == updated_attributes["scheme_code"] } + if original_row.blank? || original_row["scheme_code"].nil? + Rails.logger.info("Scheme with id #{updated_attributes['scheme_code']} is not in the original scheme csv") + next + end + + original_attributes["scheme_code"] = original_row[0] + original_attributes["service_name"] = original_row[1] + original_attributes["status"] = original_row[2] + original_attributes["sensitive"] = original_row[3] + original_attributes["scheme_type"] = original_row[4] + original_attributes["registered_under_care_act"] = original_row[5] + original_attributes["owning_organisation_name"] = original_row[6] + original_attributes["arrangement_type"] = original_row[7] + original_attributes["primary_client_group"] = original_row[8] + original_attributes["has_other_client_group"] = original_row[9] + original_attributes["secondary_client_group"] = original_row[10] + original_attributes["support_type"] = original_row[11] + original_attributes["intended_stay"] = original_row[12] + original_attributes["created_at"] = original_row[13] + original_attributes["active_dates"] = original_row[14] + + scheme = Scheme.find_by(id: original_attributes["scheme_code"].delete("S")) + if scheme.blank? + Rails.logger.info("Scheme with id #{original_attributes['scheme_code']} is not in the database") + next + end + + updated_attributes.each do |key, value| + next unless value != original_attributes[key] && value.present? + + case key + when "service_name", "sensitive", "scheme_type", "registered_under_care_act", "arrangement_type", "primary_client_group", "has_other_client_group", "secondary_client_group", "support_type", "intended_stay" + begin + scheme[key] = value + Rails.logger.info("Updating scheme #{original_attributes['scheme_code']}, with #{key}: #{value}") + rescue ArgumentError => e + Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key}: #{value}. #{e.message}") + end + when "owning_organisation_name" + organisation = Organisation.find_by(name: value) + if organisation.present? + scheme["owning_organisation_id"] = organisation.id + Rails.logger.info("Updating scheme #{original_attributes['scheme_code']}, with owning_organisation: #{organisation.name}") + else + Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key}: #{value}. Organisation with name #{value} is not in the database") + end + when "scheme_code", "status", "created_at", "active_dates" + Rails.logger.info("Cannot update scheme #{original_attributes['scheme_code']} with #{key} as it it not a permitted field") + end + end + scheme.save! + end + end +end diff --git a/spec/fixtures/files/original_schemes.csv b/spec/fixtures/files/original_schemes.csv new file mode 100644 index 000000000..840d7e7a4 --- /dev/null +++ b/spec/fixtures/files/original_schemes.csv @@ -0,0 +1,6 @@ +scheme_code,scheme_service_name,scheme_status,scheme_sensitive,scheme_type,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_support_services_provided_by,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,scheme_active_dates +{id1},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +{id2},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +{id3},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +{id4},Incomplete scheme,incomplete,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +SWrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" diff --git a/spec/fixtures/files/updated_schemes.csv b/spec/fixtures/files/updated_schemes.csv new file mode 100644 index 000000000..e83bd7584 --- /dev/null +++ b/spec/fixtures/files/updated_schemes.csv @@ -0,0 +1,6 @@ +scheme_code,scheme_service_name,scheme_status,scheme_sensitive,scheme_type,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_support_services_provided_by,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,scheme_active_dates +{id1},Updated test name,incomplete,No,Direct Access Hostel,No,Different organisation,Another registered stock owner,People with drug problems,No,Older people with support needs,Low level,Permanent,2022-04-01T00:00:00+01:00,"Active from 2 April 2020" +{id2},Test name,active,Yes,Housing for older people,Yes – registered care home providing nursing care,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,Yes,Older people with support needs,High level,Medium stay,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +{id3}, ,active,Yse,Direct access Hostel,Yes – registered care home providing nursing care,non existing org,wrong answer,FD,no,lder people with support needs,high,Permanent ,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +Wrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,No,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" +SWrong_id,Incomplete scheme,incomplete,Yes,Housing for older people,No,DLUHC,The same organisation that owns the housing stock,People with alcohol problems,,,,,2021-04-01T00:00:00+01:00,"Active from 1 April 2020" diff --git a/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb b/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb new file mode 100644 index 000000000..5e0f00b0c --- /dev/null +++ b/spec/lib/tasks/update_schemes_and_locations_from_csv_spec.rb @@ -0,0 +1,188 @@ +require "rails_helper" +require "rake" + +RSpec.describe "bulk_update" do + def replace_entity_ids(scheme_1, scheme_2, scheme_3, _incomplete_scheme, export_template) + export_template.sub!(/\{id1\}/, "S#{scheme_1.id}") + export_template.sub!(/\{id2\}/, "S#{scheme_2.id}") + export_template.sub!(/\{id3\}/, "S#{scheme_3.id}") + # export_template.sub!(/\{id4\}/, "S#{incomplete_scheme.id}") + end + + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("CSV_DOWNLOAD_PAAS_INSTANCE").and_return(instance_name) + + WebMock.stub_request(:get, /api\.postcodes\.io/) + .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) + WebMock.stub_request(:get, /api\.postcodes\.io\/postcodes\/B11BB/) + .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) + end + + describe ":update_schemes_from_csv", type: :task do + subject(:task) { Rake::Task["bulk_update:update_schemes_from_csv"] } + + let(:instance_name) { "import_instance" } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } + + before do + Rake.application.rake_require("tasks/update_schemes_and_locations_from_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:original_schemes_csv_path) { "original_schemes.csv" } + let(:updated_schemes_csv_path) { "updated_schemes.csv" } + let(:wrong_file_path) { "/test/no_csv_here.csv" } + let!(:different_organisation) { FactoryBot.create(:organisation, name: "Different organisation") } + let(:schemes) do + create_list(:scheme, + 3, + service_name: "Test name", + sensitive: 1, + registered_under_care_act: 4, + support_type: 4, + scheme_type: 7, + arrangement_type: "D", + intended_stay: "M", + primary_client_group: "G", + secondary_client_group: "M", + has_other_client_group: 1, + owning_organisation: FactoryBot.create(:organisation), + confirmed: true, + created_at: Time.zone.local(2021, 4, 1), + total_units: 2) + end + + let(:incomplete_scheme) do + build(:scheme, + service_name: "Incomplete scheme", + sensitive: 1, + registered_under_care_act: 4, + support_type: nil, + scheme_type: 7, + arrangement_type: "D", + intended_stay: nil, + primary_client_group: "G", + secondary_client_group: nil, + has_other_client_group: nil, + owning_organisation: FactoryBot.create(:organisation), + created_at: Time.zone.local(2021, 4, 1), + total_units: 2) + end + + before do + incomplete_scheme.save!(validate: false) + allow(storage_service).to receive(:get_file_io) + .with("original_schemes.csv") + .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/original_schemes.csv").read))) + + allow(storage_service).to receive(:get_file_io) + .with("updated_schemes.csv") + .and_return(StringIO.new(replace_entity_ids(schemes[0], schemes[1], schemes[2], incomplete_scheme, File.open("./spec/fixtures/files/updated_schemes.csv").read))) + end + + it "updates the allowed scheme fields if they have changed and doesn't update other fields" do + task.invoke(original_schemes_csv_path, updated_schemes_csv_path) + schemes[0].reload + expect(schemes[0].service_name).to eq("Updated test name") + expect(schemes[0].sensitive).to eq("No") + expect(schemes[0].registered_under_care_act).to eq("No") + expect(schemes[0].support_type).to eq("Low level") + expect(schemes[0].scheme_type).to eq("Direct Access Hostel") + expect(schemes[0].arrangement_type).to eq("Another registered stock owner") + expect(schemes[0].intended_stay).to eq("Permanent") + expect(schemes[0].primary_client_group).to eq("People with drug problems") + # expect(schemes[0].secondary_client_group).to eq(nil) + expect(schemes[0].has_other_client_group).to eq("No") + expect(schemes[0].owning_organisation).to eq(different_organisation) + expect(schemes[0].created_at).to eq(Time.zone.local(2021, 4, 1)) + expect(schemes[0].total_units).to eq(2) + end + + it "does not update the scheme if it hasn't changed" do + task.invoke(original_schemes_csv_path, updated_schemes_csv_path) + schemes[1].reload + expect(schemes[1].service_name).to eq("Test name") + expect(schemes[1].sensitive).to eq("Yes") + expect(schemes[1].registered_under_care_act).to eq("Yes – registered care home providing nursing care") + expect(schemes[1].support_type).to eq("High level") + expect(schemes[1].scheme_type).to eq("Housing for older people") + expect(schemes[1].arrangement_type).to eq("The same organisation that owns the housing stock") + expect(schemes[1].intended_stay).to eq("Medium stay") + expect(schemes[1].primary_client_group).to eq("People with alcohol problems") + expect(schemes[1].secondary_client_group).to eq("Older people with support needs") + expect(schemes[1].has_other_client_group).to eq("Yes") + expect(schemes[1].owning_organisation).not_to eq(different_organisation) + expect(schemes[1].created_at).to eq(Time.zone.local(2021, 4, 1)) + expect(schemes[1].total_units).to eq(2) + end + + it "does not update the scheme with invalid values" do + task.invoke(original_schemes_csv_path, updated_schemes_csv_path) + schemes[2].reload + expect(schemes[2].service_name).to eq("Test name") + expect(schemes[2].sensitive).to eq("Yes") + expect(schemes[2].registered_under_care_act).to eq("Yes – registered care home providing nursing care") + expect(schemes[2].support_type).to eq("High level") + expect(schemes[2].scheme_type).to eq("Housing for older people") + expect(schemes[2].arrangement_type).to eq("The same organisation that owns the housing stock") + expect(schemes[2].intended_stay).to eq("Medium stay") + expect(schemes[2].primary_client_group).to eq("People with alcohol problems") + expect(schemes[2].secondary_client_group).to eq("Older people with support needs") + expect(schemes[2].has_other_client_group).to eq("Yes") + expect(schemes[2].owning_organisation).not_to eq(different_organisation) + expect(schemes[2].created_at).to eq(Time.zone.local(2021, 4, 1)) + expect(schemes[2].total_units).to eq(2) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with service_name: Updated test name") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with sensitive: No") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with scheme_type: Direct Access Hostel") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with arrangement_type: Another registered stock owner") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with primary_client_group: People with drug problems") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with has_other_client_group: No") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with support_type: Low level") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with intended_stay: Permanent") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with registered_under_care_act: No") + # expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with secondary_client_group: nil") + expect(Rails.logger).to receive(:info).with("Updating scheme S#{schemes[0].id}, with owning_organisation: Different organisation") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with status as it it not a permitted field") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with created_at as it it not a permitted field") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[0].id} with active_dates as it it not a permitted field") + + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with sensitive: Yse. 'Yse' is not a valid sensitive") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with scheme_type: Direct access Hostel. 'Direct access Hostel' is not a valid scheme_type") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with owning_organisation_name: non existing org. Organisation with name non existing org is not in the database") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with arrangement_type: wrong answer. 'wrong answer' is not a valid arrangement_type") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with primary_client_group: FD. 'FD' is not a valid primary_client_group") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with has_other_client_group: no. 'no' is not a valid has_other_client_group") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with secondary_client_group: lder people with support needs. 'lder people with support needs' is not a valid secondary_client_group") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with support_type: high. 'high' is not a valid support_type") + expect(Rails.logger).to receive(:info).with("Cannot update scheme S#{schemes[2].id} with intended_stay: Permanent . 'Permanent ' is not a valid intended_stay") + + expect(Rails.logger).to receive(:info).with("Scheme with id Wrong_id is not in the original scheme csv") + expect(Rails.logger).to receive(:info).with("Scheme with id SWrong_id is not in the database") + + task.invoke(original_schemes_csv_path, updated_schemes_csv_path) + end + + it "raises an error when no paths are given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']") + end + + it "raises an error when no original path is given" do + expect { task.invoke(nil, updated_schemes_csv_path) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']") + end + + it "raises an error when no updated path is given" do + expect { task.invoke(original_schemes_csv_path, nil) }.to raise_error(RuntimeError, "Usage: rake bulk_update:update_schemes_from_csv['original_file_name','updated_file_name']") + end + end + end +end