From c68e6e88ca5d3b3010e59af113ec42b69156c29a Mon Sep 17 00:00:00 2001 From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:47:11 +0100 Subject: [PATCH] CLDC-2810 Send missing addresses template (#1952) * Add missing addresses csv job * Update missing addresses csv service methods * Add rake task * Update the job to send missing town or city templates * Update service to create mising town or city templates * Add send missing town or city csv rake task * Add IDs to the CSVs * Put all log in the same csv * Add issue type column * Write wrong uprn logs to csv * Add mailer methods * Skip uprn issue for specified orgs * Add sales csv rake task * set SKIP_UPRN_ISSUE_ORG_IDS on review apps * test * Update notify template IDs for testing * Initialize service with organisation instead of a hash * Add expiration time to url * Add optional tags and remove LA from csv * Extract log to csv methods * Update casing * Update old IDs in factories * Move constant * Extract some repeating scopes * Pass in organisations to skip instead of using an env var * update template id for sales * Update link expiry time and headers * Lower the threshold for testing * Add issue explanation to the email * Add how to fix * update emails * CLDC-2810 Create all addresses CSV (#1953) * Add rake tasks for creating all addresses CSV * Write headers if logs don't exist, update header names * Rename method * CLDC-2810 Correct addresses from csv (#1957) * Updating importing lettings addresses form csv * Add import_sales_addresses_from_csv rake * Allow correcting addresses from both templates * escape . * Reinfer LA if the postcode hasn't changed * Update labels and email content * Update missing addresses threshold * Remove unused env var --- app/jobs/create_addresses_csv_job.rb | 22 + app/jobs/email_missing_addresses_csv_job.rb | 33 ++ app/mailers/csv_download_mailer.rb | 75 +++ app/models/log.rb | 3 + .../csv/missing_addresses_csv_service.rb | 140 +++++ lib/tasks/import_address_from_csv.rake | 99 +++- lib/tasks/send_missing_addresses_csv.rake | 118 ++++ spec/factories/lettings_log.rb | 3 + spec/factories/sales_log.rb | 3 + spec/fixtures/files/addresses_reimport.csv | 13 +- .../files/addresses_reimport_all_logs.csv | 7 + .../files/missing_lettings_logs_addresses.csv | 2 + ...ing_lettings_logs_addresses_all_issues.csv | 4 + .../missing_lettings_logs_town_or_city.csv | 2 + .../missing_lettings_logs_wrong_uprn.csv | 2 + .../files/missing_sales_logs_addresses.csv | 2 + ...issing_sales_logs_addresses_all_issues.csv | 4 + .../files/missing_sales_logs_town_or_city.csv | 2 + .../files/missing_sales_logs_wrong_uprn.csv | 2 + .../files/sales_addresses_reimport.csv | 7 + .../sales_addresses_reimport_all_logs.csv | 7 + spec/jobs/create_addresses_csv_job_spec.rb | 49 ++ .../email_missing_addresses_csv_job_spec.rb | 66 +++ .../tasks/correct_address_from_csv_spec.rb | 537 ++++++++++++++--- .../tasks/send_missing_addresses_csv_spec.rb | 560 ++++++++++++++++++ spec/mailers/csv_download_mailer_spec.rb | 67 ++- .../csv/missing_addresses_csv_service_spec.rb | 512 ++++++++++++++++ 27 files changed, 2221 insertions(+), 120 deletions(-) create mode 100644 app/jobs/create_addresses_csv_job.rb create mode 100644 app/jobs/email_missing_addresses_csv_job.rb create mode 100644 app/services/csv/missing_addresses_csv_service.rb create mode 100644 lib/tasks/send_missing_addresses_csv.rake create mode 100644 spec/fixtures/files/addresses_reimport_all_logs.csv create mode 100644 spec/fixtures/files/missing_lettings_logs_addresses.csv create mode 100644 spec/fixtures/files/missing_lettings_logs_addresses_all_issues.csv create mode 100644 spec/fixtures/files/missing_lettings_logs_town_or_city.csv create mode 100644 spec/fixtures/files/missing_lettings_logs_wrong_uprn.csv create mode 100644 spec/fixtures/files/missing_sales_logs_addresses.csv create mode 100644 spec/fixtures/files/missing_sales_logs_addresses_all_issues.csv create mode 100644 spec/fixtures/files/missing_sales_logs_town_or_city.csv create mode 100644 spec/fixtures/files/missing_sales_logs_wrong_uprn.csv create mode 100644 spec/fixtures/files/sales_addresses_reimport.csv create mode 100644 spec/fixtures/files/sales_addresses_reimport_all_logs.csv create mode 100644 spec/jobs/create_addresses_csv_job_spec.rb create mode 100644 spec/jobs/email_missing_addresses_csv_job_spec.rb create mode 100644 spec/lib/tasks/send_missing_addresses_csv_spec.rb create mode 100644 spec/services/csv/missing_addresses_csv_service_spec.rb diff --git a/app/jobs/create_addresses_csv_job.rb b/app/jobs/create_addresses_csv_job.rb new file mode 100644 index 000000000..ef83ee746 --- /dev/null +++ b/app/jobs/create_addresses_csv_job.rb @@ -0,0 +1,22 @@ +class CreateAddressesCsvJob < ApplicationJob + queue_as :default + + BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8 + + def perform(organisation, log_type) + csv_service = Csv::MissingAddressesCsvService.new(organisation, []) + case log_type + when "lettings" + csv_string = csv_service.create_lettings_addresses_csv + filename = "#{['lettings-logs-addresses', organisation.name, Time.zone.now].compact.join('-')}.csv" + when "sales" + csv_string = csv_service.create_sales_addresses_csv + filename = "#{['sales-logs-addresses', organisation.name, Time.zone.now].compact.join('-')}.csv" + end + + storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + storage_service.write_file(filename, BYTE_ORDER_MARK + csv_string) + + Rails.logger.info("Created addresses file: #{filename}") + end +end diff --git a/app/jobs/email_missing_addresses_csv_job.rb b/app/jobs/email_missing_addresses_csv_job.rb new file mode 100644 index 000000000..d8b756b20 --- /dev/null +++ b/app/jobs/email_missing_addresses_csv_job.rb @@ -0,0 +1,33 @@ +class EmailMissingAddressesCsvJob < ApplicationJob + queue_as :default + + BYTE_ORDER_MARK = "\uFEFF".freeze # Required to ensure Excel always reads CSV as UTF-8 + EXPIRATION_TIME = 1.week.to_i + MISSING_ADDRESSES_THRESHOLD = 50 + + def perform(user_ids, organisation, log_type, issue_types, skip_uprn_issue_organisations) + csv_service = Csv::MissingAddressesCsvService.new(organisation, skip_uprn_issue_organisations) + case log_type + when "lettings" + csv_string = csv_service.create_missing_lettings_addresses_csv + filename = "#{['missing-lettings-logs-addresses', organisation.name, Time.zone.now].compact.join('-')}.csv" + email_method = :send_missing_lettings_addresses_csv_download_mail + when "sales" + csv_string = csv_service.create_missing_sales_addresses_csv + filename = "#{['missing-sales-logs-addresses', organisation.name, Time.zone.now].compact.join('-')}.csv" + email_method = :send_missing_sales_addresses_csv_download_mail + end + + storage_service = Storage::S3Service.new(Configuration::EnvConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + storage_service.write_file(filename, BYTE_ORDER_MARK + csv_string) + + url = storage_service.get_presigned_url(filename, EXPIRATION_TIME) + + user_ids.each do |id| + user = User.find(id) + next if user.blank? + + CsvDownloadMailer.new.send(email_method, user, url, EXPIRATION_TIME, issue_types) + end + end +end diff --git a/app/mailers/csv_download_mailer.rb b/app/mailers/csv_download_mailer.rb index 619d5e922..b590ebcfc 100644 --- a/app/mailers/csv_download_mailer.rb +++ b/app/mailers/csv_download_mailer.rb @@ -1,5 +1,7 @@ class CsvDownloadMailer < NotifyMailer CSV_DOWNLOAD_TEMPLATE_ID = "7890e3b9-8c0d-4d08-bafe-427fd7cd95bf".freeze + CSV_MISSING_LETTINGS_ADDRESSES_DOWNLOAD_TEMPLATE_ID = "7602b6c2-4f44-4da6-8a68-944e39cd8a05".freeze + CSV_MISSING_SALES_ADDRESSES_DOWNLOAD_TEMPLATE_ID = "1ee6da00-a65e-4a39-b5e5-1846debcb5f8".freeze def send_csv_download_mail(user, link, duration) send_email( @@ -8,4 +10,77 @@ class CsvDownloadMailer < NotifyMailer { name: user.name, link:, duration: ActiveSupport::Duration.build(duration).inspect }, ) end + + def send_missing_lettings_addresses_csv_download_mail(user, link, duration, issue_types) + send_email( + user.email, + CSV_MISSING_LETTINGS_ADDRESSES_DOWNLOAD_TEMPLATE_ID, + { name: user.name, issue_explanation: issue_explanation(issue_types, "lettings"), how_to_fix: how_to_fix(issue_types, link, "lettings"), duration: ActiveSupport::Duration.build(duration).inspect }, + ) + end + + def send_missing_sales_addresses_csv_download_mail(user, link, duration, issue_types) + send_email( + user.email, + CSV_MISSING_SALES_ADDRESSES_DOWNLOAD_TEMPLATE_ID, + { name: user.name, issue_explanation: issue_explanation(issue_types, "sales"), how_to_fix: how_to_fix(issue_types, link, "sales"), duration: ActiveSupport::Duration.build(duration).inspect }, + ) + end + +private + + HELPDESK_URL = "https://dluhcdigital.atlassian.net/servicedesk/customer/portal/6/group/11".freeze + + def issue_explanation(issue_types, log_type) + [ + "We have found #{multiple_issue_types?(issue_types) ? 'these issues' : 'this issue'} in your logs imported to the new version of CORE:\n", + issue_types.include?("missing_address") ? "## Full address required\nThe Unique Property Reference number (UPRN) in some logs is incorrect, so the address data was not imported. Provide the full address and leave the UPRN column blank.\n" : "", + issue_types.include?("missing_town") ? "## Missing town or city\nThe town or city in some logs is missing. This data is required in the new version of CORE.\n" : "", + has_uprn_issues(issue_types) ? uprn_issue_explanation(issue_types, log_type) : "", + ].join("") + end + + def how_to_fix(issue_types, link, log_type) + [ + "You need to:\n\n", + "- download [this spreadsheet for #{log_type} logs](#{link}). This link will expire in one week. To request another link, [contact the CORE helpdesk](#{HELPDESK_URL}).\n", + issue_types.include?("missing_address") || issue_types.include?("missing_town") ? "- fill in the missing address data\n" : "", + uprn_issues_only(issue_types) ? "- check that the address data is correct\n" : "- check that the existing address data is correct\n", + has_uprn_issues(issue_types) ? "- correct any address errors\n" : "", + ].join("") + end + + def uprn_issues_only(issue_types) + issue_types == %w[wrong_uprn] || issue_types == %w[bristol_uprn] || issue_types.count == 2 && issue_types.include?("wrong_uprn") && issue_types.include?("bristol_uprn") + end + + def has_uprn_issues(issue_types) + issue_types.include?("wrong_uprn") || issue_types.include?("bristol_uprn") + end + + def multiple_issue_types?(issue_types) + issue_types.count > 1 && !uprn_issues_only(issue_types) || issue_types.count > 2 + end + + def uprn_issue_explanation(issue_types, log_type) + if issue_types.include?("wrong_uprn") && !issue_types.include?("bristol_uprn") + "## Incorrect UPRN\nThe UPRN in some logs may be incorrect, so the wrong address data may have been imported. + +In some of your logs, the UPRN is the same as the #{log_type == 'lettings' ? 'tenant code or property reference' : 'purchaser code'}, but these are different things. #{log_type == 'lettings' ? 'Property references' : 'Purchaser codes'} are codes that your organisation uses to identify properties. UPRNs are unique numbers assigned by the Ordnance Survey. + +If a log has the correct UPRN, leave the UPRN unchanged. If the UPRN is incorrect, clear the value and provide the full address instead. Alternatively, you can change the UPRN on the CORE system.\n" + elsif issue_types.include?("bristol_uprn") && !issue_types.include?("wrong_uprn") + "## Incorrect UPRN\nThe UPRN in some logs may be incorrect, so the wrong address data may have been imported. In some of your logs, the UPRN links to an address in Bristol, but this is the first time your organisation has submitted logs for properties in Bristol. + +If a log has the correct UPRN, leave the UPRN unchanged. If the UPRN is incorrect, clear the value and provide the full address instead. Alternatively, you can change the UPRN on the CORE system.\n" + else + "## Incorrect UPRN\nThe UPRN in some logs may be incorrect, so the wrong address data may have been imported. We think this is an issue for two reasons. + +In some of your logs, the UPRN links to an address in Bristol, but this is the first time your organisation has submitted logs for properties in Bristol. + +In other logs, the UPRN is the same as the #{log_type == 'lettings' ? 'tenant code or property reference' : 'purchaser code'}, but these are different things. #{log_type == 'lettings' ? 'Property references' : 'Purchaser codes'} are codes that your organisation uses to identify properties. UPRNs are unique numbers assigned by the Ordnance Survey. + +If a log has the correct UPRN, leave the UPRN unchanged. If the UPRN is incorrect, clear the value and provide the full address instead. Alternatively, you can change the UPRN on the CORE system." + end + end end diff --git a/app/models/log.rb b/app/models/log.rb index c1778b299..5ba887c7a 100644 --- a/app/models/log.rb +++ b/app/models/log.rb @@ -46,6 +46,9 @@ class Log < ApplicationRecord scope :created_by, ->(user) { where(created_by: user) } scope :imported, -> { where.not(old_id: nil) } scope :not_imported, -> { where(old_id: nil) } + scope :has_old_form_id, -> { where.not(old_form_id: nil) } + scope :imported_2023_with_old_form_id, -> { imported.filter_by_year(2023).has_old_form_id } + scope :imported_2023, -> { imported.filter_by_year(2023) } attr_accessor :skip_update_status, :skip_update_uprn_confirmed diff --git a/app/services/csv/missing_addresses_csv_service.rb b/app/services/csv/missing_addresses_csv_service.rb new file mode 100644 index 000000000..a10beaa1d --- /dev/null +++ b/app/services/csv/missing_addresses_csv_service.rb @@ -0,0 +1,140 @@ +module Csv + class MissingAddressesCsvService + def initialize(organisation, skip_uprn_issue_organisations) + @organisation = organisation + @skip_uprn_issue_organisations = skip_uprn_issue_organisations + end + + def create_missing_lettings_addresses_csv + logs_with_missing_addresses = @organisation.managed_lettings_logs + .imported_2023_with_old_form_id + .where(needstype: 1, address_line1: nil, town_or_city: nil, uprn_known: [0, nil]) + + logs_with_missing_town_or_city = @organisation.managed_lettings_logs + .imported_2023_with_old_form_id + .where(needstype: 1, town_or_city: nil, uprn_known: [0, nil]) + .where.not(address_line1: nil) + + logs_with_wrong_uprn = if @skip_uprn_issue_organisations.include?(@organisation.id) + [] + else + @organisation.managed_lettings_logs + .imported_2023 + .where(needstype: 1) + .where.not(uprn: nil) + .where("uprn = propcode OR uprn = tenancycode OR town_or_city = 'Bristol'") + end + + return if logs_with_missing_addresses.empty? && logs_with_missing_town_or_city.empty? && logs_with_wrong_uprn.empty? + + CSV.generate(headers: true) do |csv| + csv << ["Issue type", "Log ID", "Tenancy start date", "Tenant code", "Property reference", "Log owner", "Owning organisation", "Managing organisation", "UPRN", "Address Line 1", "Address Line 2 (optional)", "Town or City", "County (optional)", "Property's postcode"] + + logs_with_missing_addresses.each do |log| + csv << ["Full address required"] + lettings_log_to_csv_row(log) + end + + logs_with_missing_town_or_city.each do |log| + csv << ["Missing town or city"] + lettings_log_to_csv_row(log) + end + + logs_with_wrong_uprn.each do |log| + csv << ["Incorrect UPRN"] + lettings_log_to_csv_row(log) + end + end + end + + def create_missing_sales_addresses_csv + logs_with_missing_addresses = @organisation.sales_logs + .imported_2023_with_old_form_id + .where(address_line1: nil, town_or_city: nil, uprn_known: [0, nil]) + + logs_with_missing_town_or_city = @organisation.sales_logs + .imported_2023_with_old_form_id + .where(town_or_city: nil, uprn_known: [0, nil]) + .where.not(address_line1: nil) + + logs_with_wrong_uprn = if @skip_uprn_issue_organisations.include?(@organisation.id) + [] + else + @organisation.sales_logs + .imported_2023 + .where.not(uprn: nil) + .where("uprn = purchid OR town_or_city = 'Bristol'") + end + return if logs_with_missing_addresses.empty? && logs_with_missing_town_or_city.empty? && logs_with_wrong_uprn.empty? + + CSV.generate(headers: true) do |csv| + csv << ["Issue type", "Log ID", "Sale completion date", "Purchaser code", "Log owner", "Owning organisation", "UPRN", "Address Line 1", "Address Line 2 (optional)", "Town or City", "County (optional)", "Property's postcode"] + + logs_with_missing_addresses.each do |log| + csv << ["Full address required"] + sales_log_to_csv_row(log) + end + + logs_with_missing_town_or_city.each do |log| + csv << ["Missing town or city"] + sales_log_to_csv_row(log) + end + + logs_with_wrong_uprn.each do |log| + csv << ["Incorrect UPRN"] + sales_log_to_csv_row(log) + end + end + end + + def create_lettings_addresses_csv + logs = @organisation.managed_lettings_logs.filter_by_year(2023) + + CSV.generate(headers: true) do |csv| + csv << ["Log ID", "Tenancy start date", "Tenant code", "Property reference", "Log owner", "Owning organisation", "Managing organisation", "UPRN", "Address Line 1", "Address Line 2 (optional)", "Town or City", "County (optional)", "Property's postcode"] + + logs.each do |log| + csv << lettings_log_to_csv_row(log) + end + end + end + + def create_sales_addresses_csv + logs = @organisation.sales_logs.filter_by_year(2023) + + CSV.generate(headers: true) do |csv| + csv << ["Log ID", "Sale completion date", "Purchaser code", "Log owner", "Owning organisation", "UPRN", "Address Line 1", "Address Line 2 (optional)", "Town or City", "County (optional)", "Property's postcode"] + + logs.each do |log| + csv << sales_log_to_csv_row(log) + end + end + end + + private + + def sales_log_to_csv_row(log) + [log.id, + log.saledate&.to_date, + log.purchid, + log.created_by&.email, + log.owning_organisation&.name, + log.uprn, + log.address_line1, + log.address_line2, + log.town_or_city, + log.county, + log.postcode_full] + end + + def lettings_log_to_csv_row(log) + [log.id, + log.startdate&.to_date, + log.tenancycode, + log.propcode, + log.created_by&.email, + log.owning_organisation&.name, + log.managing_organisation&.name, + log.uprn, + log.address_line1, + log.address_line2, + log.town_or_city, + log.county, + log.postcode_full] + end + end +end diff --git a/lib/tasks/import_address_from_csv.rake b/lib/tasks/import_address_from_csv.rake index e910f3a77..57496a56c 100644 --- a/lib/tasks/import_address_from_csv.rake +++ b/lib/tasks/import_address_from_csv.rake @@ -1,18 +1,35 @@ namespace :data_import do - desc "Import address data from a csv file" - task :import_address_from_csv, %i[file_name] => :environment do |_task, args| + desc "Import lettings address data from a csv file" + task :import_lettings_addresses_from_csv, %i[file_name] => :environment do |_task, args| file_name = args[:file_name] - raise "Usage: rake data_import:import_address_from_csv['csv_file_name']" if file_name.blank? + raise "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']" if file_name.blank? s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + contains_issue_type = addresses_csv.headers.include?("Issue type") addresses_csv.each do |row| - lettings_log_id = row[0] + lettings_log_id = contains_issue_type ? row[1] : row[0] + uprn = contains_issue_type ? row[8] : row[7] + address_line1 = contains_issue_type ? row[9] : row[8] + address_line2 = contains_issue_type ? row[10] : row[9] + town_or_city = contains_issue_type ? row[11] : row[10] + county = contains_issue_type ? row[12] : row[11] + postcode_full = contains_issue_type ? row[13] : row[12] if lettings_log_id.blank? - Rails.logger.info("Lettings log ID not provided for address: #{[row[1], row[2], row[3], row[4]].join(', ')}") + Rails.logger.info("Lettings log ID not provided for address: #{[address_line1, address_line2, town_or_city, county, postcode_full].join(', ')}") + next + end + + if uprn.present? + Rails.logger.info("Lettings log with ID #{lettings_log_id} contains uprn, skipping log") + next + end + + if address_line1.blank? || town_or_city.blank? || postcode_full.blank? + Rails.logger.info("Lettings log with ID #{lettings_log_id} is missing required address data, skipping log") next end @@ -25,18 +42,78 @@ namespace :data_import do lettings_log.uprn_known = 0 lettings_log.uprn = nil lettings_log.uprn_confirmed = nil - lettings_log.address_line1 = row[1] - lettings_log.address_line2 = row[2] - lettings_log.town_or_city = row[3] - lettings_log.postcode_full = row[4] + lettings_log.address_line1 = address_line1 + lettings_log.address_line2 = address_line2 + lettings_log.town_or_city = town_or_city + lettings_log.county = county + lettings_log.postcode_full = postcode_full lettings_log.postcode_known = lettings_log.postcode_full.present? ? 1 : nil - lettings_log.county = nil lettings_log.is_la_inferred = nil lettings_log.la = nil + lettings_log.send("process_postcode_changes!") lettings_log.values_updated_at = Time.zone.now lettings_log.save! - Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.postcode_full].join(', ')}") + Rails.logger.info("Updated lettings log #{lettings_log_id}, with address: #{[lettings_log.address_line1, lettings_log.address_line2, lettings_log.town_or_city, lettings_log.county, lettings_log.postcode_full].join(', ')}") + end + end + + desc "Import sales address data from a csv file" + task :import_sales_addresses_from_csv, %i[file_name] => :environment do |_task, args| + file_name = args[:file_name] + + raise "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']" if file_name.blank? + + s3_service = Storage::S3Service.new(PlatformHelper.is_paas? ? Configuration::PaasConfigurationService.new : Configuration::EnvConfigurationService.new, ENV["IMPORT_PAAS_INSTANCE"]) + addresses_csv = CSV.parse(s3_service.get_file_io(file_name), headers: true) + contains_issue_type = addresses_csv.headers.include?("Issue type") + + addresses_csv.each do |row| + sales_log_id = contains_issue_type ? row[1] : row[0] + uprn = contains_issue_type ? row[6] : row[5] + address_line1 = contains_issue_type ? row[7] : row[6] + address_line2 = contains_issue_type ? row[8] : row[7] + town_or_city = contains_issue_type ? row[9] : row[8] + county = contains_issue_type ? row[10] : row[9] + postcode_full = contains_issue_type ? row[11] : row[10] + + if sales_log_id.blank? + Rails.logger.info("Sales log ID not provided for address: #{[address_line1, address_line2, town_or_city, county, postcode_full].join(', ')}") + next + end + + if uprn.present? + Rails.logger.info("Sales log with ID #{sales_log_id} contains uprn, skipping log") + next + end + + if address_line1.blank? || town_or_city.blank? || postcode_full.blank? + Rails.logger.info("Sales log with ID #{sales_log_id} is missing required address data, skipping log") + next + end + + sales_log = SalesLog.find_by(id: sales_log_id) + if sales_log.blank? + Rails.logger.info("Could not find a sales log with id #{sales_log_id}") + next + end + + sales_log.uprn_known = 0 + sales_log.uprn = nil + sales_log.uprn_confirmed = nil + sales_log.address_line1 = address_line1 + sales_log.address_line2 = address_line2 + sales_log.town_or_city = town_or_city + sales_log.county = county + sales_log.postcode_full = postcode_full + sales_log.pcodenk = sales_log.postcode_full.present? ? 0 : nil + sales_log.is_la_inferred = nil + sales_log.la = nil + sales_log.send("process_postcode_changes!") + sales_log.values_updated_at = Time.zone.now + + sales_log.save! + Rails.logger.info("Updated sales log #{sales_log_id}, with address: #{[sales_log.address_line1, sales_log.address_line2, sales_log.town_or_city, sales_log.county, sales_log.postcode_full].join(', ')}") end end end diff --git a/lib/tasks/send_missing_addresses_csv.rake b/lib/tasks/send_missing_addresses_csv.rake new file mode 100644 index 000000000..9c8e60b59 --- /dev/null +++ b/lib/tasks/send_missing_addresses_csv.rake @@ -0,0 +1,118 @@ +namespace :correct_addresses do + desc "Send missing lettings addresses csv" + task :send_missing_addresses_lettings_csv, %i[skip_uprn_issue_organisations] => :environment do |_task, args| + skip_uprn_issue_organisations = args[:skip_uprn_issue_organisations]&.split(" ")&.map(&:to_i) || [] + + Organisation.all.each do |organisation| + logs_impacted_by_missing_address = organisation.managed_lettings_logs + .imported_2023_with_old_form_id + .where(needstype: 1, address_line1: nil, town_or_city: nil, uprn_known: [0, nil]).count + + logs_impacted_by_missing_town_or_city = organisation.managed_lettings_logs + .imported_2023_with_old_form_id + .where(needstype: 1, town_or_city: nil, uprn_known: [0, nil]) + .where.not(address_line1: nil).count + + logs_impacted_by_uprn_issue = if skip_uprn_issue_organisations.include?(organisation.id) + [] + else + organisation.managed_lettings_logs + .imported_2023 + .where(needstype: 1) + .where.not(uprn: nil) + .where("uprn = propcode OR uprn = tenancycode") + end + + logs_impacted_by_bristol_uprn_issue = if skip_uprn_issue_organisations.include?(organisation.id) + [] + else + organisation.managed_lettings_logs + .imported_2023 + .where(needstype: 1) + .where.not(uprn: nil) + .where("town_or_city = 'Bristol'") + end + + missing_addresses_threshold = EmailMissingAddressesCsvJob::MISSING_ADDRESSES_THRESHOLD + if logs_impacted_by_missing_address >= missing_addresses_threshold || logs_impacted_by_missing_town_or_city >= missing_addresses_threshold || logs_impacted_by_uprn_issue.any? || logs_impacted_by_bristol_uprn_issue.any? + issue_types = [] + issue_types << "missing_address" if logs_impacted_by_missing_address.positive? + issue_types << "missing_town" if logs_impacted_by_missing_town_or_city.positive? + issue_types << "wrong_uprn" if logs_impacted_by_uprn_issue.any? + issue_types << "bristol_uprn" if logs_impacted_by_bristol_uprn_issue.any? + data_coordinators = organisation.users.where(role: 2).filter_by_active + users_to_contact = data_coordinators.any? ? data_coordinators : organisation.users.filter_by_active + EmailMissingAddressesCsvJob.perform_later(users_to_contact.map(&:id), organisation, "lettings", issue_types, skip_uprn_issue_organisations) + Rails.logger.info("Sending missing lettings addresses CSV for #{organisation.name} to #{users_to_contact.map(&:email).join(', ')}") + else + Rails.logger.info("Missing addresses below threshold for #{organisation.name}") + end + end + end + + desc "Send missing sales addresses csv" + task :send_missing_addresses_sales_csv, %i[skip_uprn_issue_organisations] => :environment do |_task, args| + skip_uprn_issue_organisations = args[:skip_uprn_issue_organisations]&.split(" ")&.map(&:to_i) || [] + + Organisation.all.each do |organisation| + logs_impacted_by_missing_address = organisation.sales_logs + .imported_2023_with_old_form_id + .where(address_line1: nil, town_or_city: nil, uprn_known: [0, nil]).count + + logs_impacted_by_missing_town_or_city = organisation.sales_logs + .imported_2023_with_old_form_id + .where(town_or_city: nil, uprn_known: [0, nil]) + .where.not(address_line1: nil).count + + logs_impacted_by_uprn_issue = if skip_uprn_issue_organisations.include?(organisation.id) + [] + else + organisation.sales_logs + .imported_2023 + .where.not(uprn: nil) + .where("uprn = purchid OR town_or_city = 'Bristol'") + end + missing_addresses_threshold = EmailMissingAddressesCsvJob::MISSING_ADDRESSES_THRESHOLD + if logs_impacted_by_missing_address >= missing_addresses_threshold || logs_impacted_by_missing_town_or_city >= missing_addresses_threshold || logs_impacted_by_uprn_issue.any? + issue_types = [] + issue_types << "missing_address" if logs_impacted_by_missing_address.positive? + issue_types << "missing_town" if logs_impacted_by_missing_town_or_city.positive? + issue_types << "wrong_uprn" if logs_impacted_by_uprn_issue.any? + data_coordinators = organisation.users.where(role: 2).filter_by_active + users_to_contact = data_coordinators.any? ? data_coordinators : organisation.users.filter_by_active + EmailMissingAddressesCsvJob.perform_later(users_to_contact.map(&:id), organisation, "sales", issue_types, skip_uprn_issue_organisations) + Rails.logger.info("Sending missing sales addresses CSV for #{organisation.name} to #{users_to_contact.map(&:email).join(', ')}") + else + Rails.logger.info("Missing addresses below threshold for #{organisation.name}") + end + end + end + + desc "Send all 2023 lettings addresses csv" + task :create_lettings_addresses_csv, %i[organisation_id] => :environment do |_task, args| + organisation_id = args[:organisation_id] + raise "Usage: rake correct_addresses:create_lettings_addresses_csv['organisation_id']" if organisation_id.blank? + + organisation = Organisation.find_by(id: organisation_id) + if organisation.present? + CreateAddressesCsvJob.perform_later(organisation, "lettings") + Rails.logger.info("Creating lettings addresses CSV for #{organisation.name}") + else + Rails.logger.error("Organisation with ID #{organisation_id} not found") + end + end + + desc "Send all 2023 sales addresses csv" + task :create_sales_addresses_csv, %i[organisation_id] => :environment do |_task, args| + organisation_id = args[:organisation_id] + raise "Usage: rake correct_addresses:create_sales_addresses_csv['organisation_id']" if organisation_id.blank? + + organisation = Organisation.find_by(id: organisation_id) + if organisation.present? + CreateAddressesCsvJob.perform_later(organisation, "sales") + Rails.logger.info("Creating sales addresses CSV for #{organisation.name}") + else + Rails.logger.error("Organisation with ID #{organisation_id} not found") + end + end +end diff --git a/spec/factories/lettings_log.rb b/spec/factories/lettings_log.rb index 6cce1dfea..c85b4b00a 100644 --- a/spec/factories/lettings_log.rb +++ b/spec/factories/lettings_log.rb @@ -190,6 +190,9 @@ FactoryBot.define do status { 4 } discarded_at { Time.zone.now } end + trait :imported do + old_id { Random.hex } + end created_at { Time.zone.today } updated_at { Time.zone.today } end diff --git a/spec/factories/sales_log.rb b/spec/factories/sales_log.rb index 177ed600c..e930f3b2a 100644 --- a/spec/factories/sales_log.rb +++ b/spec/factories/sales_log.rb @@ -155,5 +155,8 @@ FactoryBot.define do status { 4 } discarded_at { Time.zone.now } end + trait :imported do + old_id { Random.hex } + end end end diff --git a/spec/fixtures/files/addresses_reimport.csv b/spec/fixtures/files/addresses_reimport.csv index 998d2fbfa..b797ac9b4 100644 --- a/spec/fixtures/files/addresses_reimport.csv +++ b/spec/fixtures/files/addresses_reimport.csv @@ -1,6 +1,7 @@ -lettings_log_id,address_line1,address_line2,town_or_city,postcode_full -{id},address 1,address 2,town,B1 1BB -{id2},address 3,address 4,city,B1 1BB -{id3},,,,C1 1CC -,address 5,address 6,city,D1 1DD -fake_id,address 7,address 8,city,D1 1DD +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 1,address 2,town,county,B1 1BB +Missing town or city,{id2},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 3,,city,,B1 1BB +Incorrect UPRN,{id3},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,123,Some Place,,Bristol,,BS1 1AD +Incorrect UPRN,{id4},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,,,BS1 1AD +Incorrect UPRN,,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD +Incorrect UPRN,fake_id,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/addresses_reimport_all_logs.csv b/spec/fixtures/files/addresses_reimport_all_logs.csv new file mode 100644 index 000000000..0a1368d28 --- /dev/null +++ b/spec/fixtures/files/addresses_reimport_all_logs.csv @@ -0,0 +1,7 @@ +Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 1,address 2,town,county,B1 1BB +{id2},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,address 3,,city,,B1 1BB +{id3},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,123,Some Place,,Bristol,,BS1 1AD +{id4},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,,,BS1 1AD +,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD +fake_id,2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/missing_lettings_logs_addresses.csv b/spec/fixtures/files/missing_lettings_logs_addresses.csv new file mode 100644 index 000000000..12475c026 --- /dev/null +++ b/spec/fixtures/files/missing_lettings_logs_addresses.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,,,,, diff --git a/spec/fixtures/files/missing_lettings_logs_addresses_all_issues.csv b/spec/fixtures/files/missing_lettings_logs_addresses_all_issues.csv new file mode 100644 index 000000000..ad6e1c865 --- /dev/null +++ b/spec/fixtures/files/missing_lettings_logs_addresses_all_issues.csv @@ -0,0 +1,4 @@ +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,,,,, +Missing town or city,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,existing address,,,, +Incorrect UPRN,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,123,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/missing_lettings_logs_town_or_city.csv b/spec/fixtures/files/missing_lettings_logs_town_or_city.csv new file mode 100644 index 000000000..4c8c65ea8 --- /dev/null +++ b/spec/fixtures/files/missing_lettings_logs_town_or_city.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Missing town or city,{id},2023-04-05,tenancycode,propcode,testy@example.com,Address org,Address org,,existing address,,,, diff --git a/spec/fixtures/files/missing_lettings_logs_wrong_uprn.csv b/spec/fixtures/files/missing_lettings_logs_wrong_uprn.csv new file mode 100644 index 000000000..238cc5aef --- /dev/null +++ b/spec/fixtures/files/missing_lettings_logs_wrong_uprn.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Incorrect UPRN,{id},2023-04-05,tenancycode,12,testy@example.com,Address org,Address org,12,Some Place,,Newcastle,,EC1N 2TD diff --git a/spec/fixtures/files/missing_sales_logs_addresses.csv b/spec/fixtures/files/missing_sales_logs_addresses.csv new file mode 100644 index 000000000..80892a2cf --- /dev/null +++ b/spec/fixtures/files/missing_sales_logs_addresses.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,purchaser code,testy@example.com,Address org,,,,,, diff --git a/spec/fixtures/files/missing_sales_logs_addresses_all_issues.csv b/spec/fixtures/files/missing_sales_logs_addresses_all_issues.csv new file mode 100644 index 000000000..1ef3aecb6 --- /dev/null +++ b/spec/fixtures/files/missing_sales_logs_addresses_all_issues.csv @@ -0,0 +1,4 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,purchaser code,testy@example.com,Address org,,,,,, +Missing town or city,{id},2023-04-05,purchaser code,testy@example.com,Address org,,existing address line 1,,,, +Incorrect UPRN,{id},2023-04-05,purchaser code,testy@example.com,Address org,123,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/missing_sales_logs_town_or_city.csv b/spec/fixtures/files/missing_sales_logs_town_or_city.csv new file mode 100644 index 000000000..b1f76611e --- /dev/null +++ b/spec/fixtures/files/missing_sales_logs_town_or_city.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Missing town or city,{id},2023-04-05,purchaser code,testy@example.com,Address org,,existing address line 1,,,, diff --git a/spec/fixtures/files/missing_sales_logs_wrong_uprn.csv b/spec/fixtures/files/missing_sales_logs_wrong_uprn.csv new file mode 100644 index 000000000..1964dd673 --- /dev/null +++ b/spec/fixtures/files/missing_sales_logs_wrong_uprn.csv @@ -0,0 +1,2 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Incorrect UPRN,{id},2023-04-05,12,testy@example.com,Address org,12,Some Place,,Newcastle,,EC1N 2TD diff --git a/spec/fixtures/files/sales_addresses_reimport.csv b/spec/fixtures/files/sales_addresses_reimport.csv new file mode 100644 index 000000000..f9cd0bc13 --- /dev/null +++ b/spec/fixtures/files/sales_addresses_reimport.csv @@ -0,0 +1,7 @@ +Issue type,Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +Full address required,{id},2023-04-05,purchid,testy@example.com,Address org,,address 1,address 2,town,county,B1 1BB +Missing town or city,{id2},2023-04-05,purchid,testy@example.com,Address org,,address 3,,city,,B1 1BB +Incorrect UPRN,{id3},2023-04-05,purchid,testy@example.com,Address org,123,Some Place,,Bristol,,BS1 1AD +Incorrect UPRN,{id4},2023-04-05,purchid,testy@example.com,Address org,,Some Place,,,,BS1 1AD +Incorrect UPRN,,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD +Incorrect UPRN,fake_id,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/fixtures/files/sales_addresses_reimport_all_logs.csv b/spec/fixtures/files/sales_addresses_reimport_all_logs.csv new file mode 100644 index 000000000..e9ebee3cb --- /dev/null +++ b/spec/fixtures/files/sales_addresses_reimport_all_logs.csv @@ -0,0 +1,7 @@ +Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode +{id},2023-04-05,purchid,testy@example.com,Address org,,address 1,address 2,town,county,B1 1BB +{id2},2023-04-05,purchid,testy@example.com,Address org,,address 3,,city,,B1 1BB +{id3},2023-04-05,purchid,testy@example.com,Address org,123,Some Place,,Bristol,,BS1 1AD +{id4},2023-04-05,purchid,testy@example.com,Address org,,Some Place,,,,BS1 1AD +,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD +fake_id,2023-04-05,purchid,testy@example.com,Address org,,Some Place,,Bristol,,BS1 1AD diff --git a/spec/jobs/create_addresses_csv_job_spec.rb b/spec/jobs/create_addresses_csv_job_spec.rb new file mode 100644 index 000000000..3e35feec4 --- /dev/null +++ b/spec/jobs/create_addresses_csv_job_spec.rb @@ -0,0 +1,49 @@ +require "rails_helper" + +describe CreateAddressesCsvJob do + include Helpers + + let(:job) { described_class.new } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:mailer) { instance_double(CsvDownloadMailer) } + let(:missing_addresses_csv_service) { instance_double(Csv::MissingAddressesCsvService) } + let(:organisation) { build(:organisation) } + let(:users) { create_list(:user, 2) } + + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(storage_service).to receive(:write_file) + + allow(Csv::MissingAddressesCsvService).to receive(:new).and_return(missing_addresses_csv_service) + allow(missing_addresses_csv_service).to receive(:create_lettings_addresses_csv).and_return("") + allow(missing_addresses_csv_service).to receive(:create_sales_addresses_csv).and_return("") + end + + context "when sending all lettings logs csv" do + it "uses an appropriate filename in S3" do + expect(storage_service).to receive(:write_file).with(/lettings-logs-addresses-#{organisation.name}-.*\.csv/, anything) + expect(Rails.logger).to receive(:info).with(/Created addresses file: lettings-logs-addresses-#{organisation.name}-.*\.csv/) + job.perform(organisation, "lettings") + end + + it "creates a MissingAddressesCsvService with the correct organisation and calls create all lettings logs adresses csv" do + expect(Csv::MissingAddressesCsvService).to receive(:new).with(organisation, []) + expect(missing_addresses_csv_service).to receive(:create_lettings_addresses_csv) + job.perform(organisation, "lettings") + end + end + + context "when sending all sales logs csv" do + it "uses an appropriate filename in S3" do + expect(storage_service).to receive(:write_file).with(/sales-logs-addresses-#{organisation.name}-.*\.csv/, anything) + expect(Rails.logger).to receive(:info).with(/Created addresses file: sales-logs-addresses-#{organisation.name}-.*\.csv/) + job.perform(organisation, "sales") + end + + it "creates a MissingAddressesCsvService with the correct organisation and calls create all sales logs adresses csv" do + expect(Csv::MissingAddressesCsvService).to receive(:new).with(organisation, []) + expect(missing_addresses_csv_service).to receive(:create_sales_addresses_csv) + job.perform(organisation, "sales") + end + end +end diff --git a/spec/jobs/email_missing_addresses_csv_job_spec.rb b/spec/jobs/email_missing_addresses_csv_job_spec.rb new file mode 100644 index 000000000..69d90cf51 --- /dev/null +++ b/spec/jobs/email_missing_addresses_csv_job_spec.rb @@ -0,0 +1,66 @@ +require "rails_helper" + +describe EmailMissingAddressesCsvJob do + include Helpers + + test_url = :test_url + + let(:job) { described_class.new } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:mailer) { instance_double(CsvDownloadMailer) } + let(:missing_addresses_csv_service) { instance_double(Csv::MissingAddressesCsvService) } + let(:organisation) { build(:organisation) } + let(:users) { create_list(:user, 2) } + + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(storage_service).to receive(:write_file) + allow(storage_service).to receive(:get_presigned_url).and_return(test_url) + + allow(Csv::MissingAddressesCsvService).to receive(:new).and_return(missing_addresses_csv_service) + allow(missing_addresses_csv_service).to receive(:create_missing_lettings_addresses_csv).and_return("") + allow(missing_addresses_csv_service).to receive(:create_missing_sales_addresses_csv).and_return("") + + allow(CsvDownloadMailer).to receive(:new).and_return(mailer) + allow(mailer).to receive(:send_missing_lettings_addresses_csv_download_mail) + allow(mailer).to receive(:send_missing_sales_addresses_csv_download_mail) + end + + context "when sending missing lettings logs csv" do + it "uses an appropriate filename in S3" do + expect(storage_service).to receive(:write_file).with(/missing-lettings-logs-addresses-#{organisation.name}-.*\.csv/, anything) + job.perform(users.map(&:id), organisation, "lettings", %w[missing_address wrong_uprn], [1, 2]) + end + + it "creates a MissingAddressesCsvService with the correct organisation and calls create missing lettings logs adresses csv" do + expect(Csv::MissingAddressesCsvService).to receive(:new).with(organisation, [1, 2]) + expect(missing_addresses_csv_service).to receive(:create_missing_lettings_addresses_csv) + job.perform(users.map(&:id), organisation, "lettings", %w[missing_address wrong_uprn], [1, 2]) + end + + it "sends emails to all the provided users" do + expect(mailer).to receive(:send_missing_lettings_addresses_csv_download_mail).with(users[0], test_url, instance_of(Integer), %w[missing_address wrong_uprn]) + expect(mailer).to receive(:send_missing_lettings_addresses_csv_download_mail).with(users[1], test_url, instance_of(Integer), %w[missing_address wrong_uprn]) + job.perform(users.map(&:id), organisation, "lettings", %w[missing_address wrong_uprn], [1, 2]) + end + end + + context "when sending missing sales logs csv" do + it "uses an appropriate filename in S3" do + expect(storage_service).to receive(:write_file).with(/missing-sales-logs-addresses-#{organisation.name}-.*\.csv/, anything) + job.perform(users.map(&:id), organisation, "sales", %w[missing_town], [2, 3]) + end + + it "creates a MissingAddressesCsvService with the correct organisation and calls create missing sales logs adresses csv" do + expect(Csv::MissingAddressesCsvService).to receive(:new).with(organisation, [2, 3]) + expect(missing_addresses_csv_service).to receive(:create_missing_sales_addresses_csv) + job.perform(users.map(&:id), organisation, "sales", %w[missing_town], [2, 3]) + end + + it "sends emails to all the provided users" do + expect(mailer).to receive(:send_missing_sales_addresses_csv_download_mail).with(users[0], test_url, instance_of(Integer), %w[missing_town]) + expect(mailer).to receive(:send_missing_sales_addresses_csv_download_mail).with(users[1], test_url, instance_of(Integer), %w[missing_town]) + job.perform(users.map(&:id), organisation, "sales", %w[missing_town], [2, 3]) + end + end +end diff --git a/spec/lib/tasks/correct_address_from_csv_spec.rb b/spec/lib/tasks/correct_address_from_csv_spec.rb index 0a3d1c563..294ce535b 100644 --- a/spec/lib/tasks/correct_address_from_csv_spec.rb +++ b/spec/lib/tasks/correct_address_from_csv_spec.rb @@ -2,14 +2,45 @@ require "rails_helper" require "rake" RSpec.describe "data_import" do - def replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, export_template) - export_template.sub!(/\{id\}/, lettings_log.id.to_s) - export_template.sub!(/\{id2\}/, second_lettings_log.id.to_s) - export_template.sub!(/\{id3\}/, third_lettings_log.id.to_s) + def replace_entity_ids(log, second_log, third_log, fourth_log, export_template) + export_template.sub!(/\{id\}/, log.id.to_s) + export_template.sub!(/\{id2\}/, second_log.id.to_s) + export_template.sub!(/\{id3\}/, third_log.id.to_s) + export_template.sub!(/\{id4\}/, fourth_log.id.to_s) end - describe ":import_address_from_csv", type: :task do - subject(:task) { Rake::Task["data_import:import_address_from_csv"] } + before do + allow(Storage::S3Service).to receive(:new).and_return(storage_service) + allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) + allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) + allow(ENV).to receive(:[]) + allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) + allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") + + WebMock.stub_request(:get, /api\.postcodes\.io/) + .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) + WebMock.stub_request(:get, /api\.postcodes\.io\/postcodes\/B11BB/) + .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) + + body = { + results: [ + { + DPA: { + "POSTCODE": "LS16 6FT", + "POST_TOWN": "Westminster", + "PO_BOX_NUMBER": "Wrong Address Line1", + "DOUBLE_DEPENDENT_LOCALITY": "Double Dependent Locality", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key&uprn=1") + .to_return(status: 200, body:, headers: {}) + end + + describe ":import_lettings_addresses_from_csv", type: :task do + subject(:task) { Rake::Task["data_import:import_lettings_addresses_from_csv"] } let(:instance_name) { "paas_import_instance" } let(:storage_service) { instance_double(Storage::S3Service) } @@ -17,25 +48,14 @@ RSpec.describe "data_import" do let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } before do - allow(Storage::S3Service).to receive(:new).and_return(storage_service) - allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) - allow(Configuration::PaasConfigurationService).to receive(:new).and_return(paas_config_service) - allow(ENV).to receive(:[]) - allow(ENV).to receive(:[]).with("IMPORT_PAAS_INSTANCE").and_return(instance_name) - allow(ENV).to receive(:[]).with("VCAP_SERVICES").and_return("dummy") - Rake.application.rake_require("tasks/import_address_from_csv") Rake::Task.define_task(:environment) task.reenable - - WebMock.stub_request(:get, /api.postcodes.io/) - .to_return(status: 200, body: "{\"status\":404,\"error\":\"Postcode not found\"}", headers: {}) - WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/B11BB/) - .to_return(status: 200, body: '{"status":200,"result":{"admin_district":"Westminster","codes":{"admin_district":"E08000035"}}}', headers: {}) end context "when the rake task is run" do let(:addresses_csv_path) { "addresses_reimport_123.csv" } + let(:all_addresses_csv_path) { "all_addresses_reimport_123.csv" } let(:wrong_file_path) { "/test/no_csv_here.csv" } let!(:lettings_log) do create(:lettings_log, @@ -52,102 +72,427 @@ RSpec.describe "data_import" do is_la_inferred: true) end - let!(:second_lettings_log) do - create(:lettings_log, - uprn_known: 1, - uprn: "1", - uprn_confirmed: nil, - address_line1: "wrong address line1", - address_line2: "wrong address 2", - town_or_city: "wrong town", - county: "wrong city", - postcode_known: 1, - postcode_full: "A1 1AA", - la: "E06000064", - is_la_inferred: true) + let!(:lettings_logs) do + create_list(:lettings_log, + 3, + uprn_known: 1, + uprn: "1", + uprn_confirmed: nil, + address_line1: "wrong address line1", + address_line2: "wrong address 2", + town_or_city: "wrong town", + county: "wrong city", + postcode_known: 1, + postcode_full: "A1 1AA", + la: "E06000064", + is_la_inferred: true) end - let!(:third_lettings_log) do - create(:lettings_log, - uprn_known: 1, - uprn: "1", + before do + allow(storage_service).to receive(:get_file_io) + .with("addresses_reimport_123.csv") + .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport.csv").read)) + + allow(storage_service).to receive(:get_file_io) + .with("all_addresses_reimport_123.csv") + .and_return(replace_entity_ids(lettings_log, lettings_logs[0], lettings_logs[1], lettings_logs[2], File.open("./spec/fixtures/files/addresses_reimport_all_logs.csv").read)) + end + + context "when the file contains issue type column" do + it "updates the log address when old address was not given" do + task.invoke(addresses_csv_path) + lettings_log.reload + expect(lettings_log.uprn_known).to eq(0) + expect(lettings_log.uprn).to eq(nil) + expect(lettings_log.uprn_confirmed).to eq(nil) + expect(lettings_log.address_line1).to eq("address 1") + expect(lettings_log.address_line2).to eq("address 2") + expect(lettings_log.town_or_city).to eq("town") + expect(lettings_log.county).to eq("county") + expect(lettings_log.postcode_known).to eq(1) + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(addresses_csv_path) + lettings_logs[0].reload + expect(lettings_logs[0].uprn_known).to eq(0) + expect(lettings_logs[0].uprn).to eq(nil) + expect(lettings_logs[0].uprn_confirmed).to eq(nil) + expect(lettings_logs[0].address_line1).to eq("address 3") + expect(lettings_logs[0].address_line2).to eq(nil) + expect(lettings_logs[0].town_or_city).to eq("city") + expect(lettings_logs[0].county).to eq(nil) + expect(lettings_logs[0].postcode_known).to eq(1) + expect(lettings_logs[0].postcode_full).to eq("B1 1BB") + expect(lettings_logs[0].la).to eq("E08000035") + expect(lettings_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(addresses_csv_path) + lettings_logs[1].reload + expect(lettings_logs[1].uprn_known).to eq(1) + expect(lettings_logs[1].uprn).to eq("1") + expect(lettings_logs[1].uprn_confirmed).to eq(nil) + expect(lettings_logs[1].address_line1).to eq("wrong address line1") + expect(lettings_logs[1].address_line2).to eq("wrong address 2") + expect(lettings_logs[1].town_or_city).to eq("wrong town") + expect(lettings_logs[1].county).to eq("wrong city") + expect(lettings_logs[1].postcode_known).to eq(1) + expect(lettings_logs[1].postcode_full).to eq("A1 1AA") + expect(lettings_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(addresses_csv_path) + lettings_logs[2].reload + expect(lettings_logs[2].uprn_known).to eq(1) + expect(lettings_logs[2].uprn).to eq("1") + expect(lettings_logs[2].uprn_confirmed).to eq(nil) + expect(lettings_logs[2].address_line1).to eq("wrong address line1") + expect(lettings_logs[2].address_line2).to eq("wrong address 2") + expect(lettings_logs[2].town_or_city).to eq("wrong town") + expect(lettings_logs[2].county).to eq("wrong city") + expect(lettings_logs[2].postcode_known).to eq(1) + expect(lettings_logs[2].postcode_full).to eq("A1 1AA") + expect(lettings_logs[2].la).to eq("E06000064") + end + + it "reinfers the LA if the postcode doesn't change" do + lettings_log.update!(postcode_full: "B1 1BB") + task.invoke(addresses_csv_path) + lettings_log.reload + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + + task.invoke(addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']") + end + end + + context "when the file does not contain issue type column" do + it "updates the log address when old address was not given" do + task.invoke(all_addresses_csv_path) + lettings_log.reload + expect(lettings_log.uprn_known).to eq(0) + expect(lettings_log.uprn).to eq(nil) + expect(lettings_log.uprn_confirmed).to eq(nil) + expect(lettings_log.address_line1).to eq("address 1") + expect(lettings_log.address_line2).to eq("address 2") + expect(lettings_log.town_or_city).to eq("town") + expect(lettings_log.county).to eq("county") + expect(lettings_log.postcode_known).to eq(1) + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(all_addresses_csv_path) + lettings_logs[0].reload + expect(lettings_logs[0].uprn_known).to eq(0) + expect(lettings_logs[0].uprn).to eq(nil) + expect(lettings_logs[0].uprn_confirmed).to eq(nil) + expect(lettings_logs[0].address_line1).to eq("address 3") + expect(lettings_logs[0].address_line2).to eq(nil) + expect(lettings_logs[0].town_or_city).to eq("city") + expect(lettings_logs[0].county).to eq(nil) + expect(lettings_logs[0].postcode_known).to eq(1) + expect(lettings_logs[0].postcode_full).to eq("B1 1BB") + expect(lettings_logs[0].la).to eq("E08000035") + expect(lettings_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(all_addresses_csv_path) + lettings_logs[1].reload + expect(lettings_logs[1].uprn_known).to eq(1) + expect(lettings_logs[1].uprn).to eq("1") + expect(lettings_logs[1].uprn_confirmed).to eq(nil) + expect(lettings_logs[1].address_line1).to eq("wrong address line1") + expect(lettings_logs[1].address_line2).to eq("wrong address 2") + expect(lettings_logs[1].town_or_city).to eq("wrong town") + expect(lettings_logs[1].county).to eq("wrong city") + expect(lettings_logs[1].postcode_known).to eq(1) + expect(lettings_logs[1].postcode_full).to eq("A1 1AA") + expect(lettings_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(all_addresses_csv_path) + lettings_logs[2].reload + expect(lettings_logs[2].uprn_known).to eq(1) + expect(lettings_logs[2].uprn).to eq("1") + expect(lettings_logs[2].uprn_confirmed).to eq(nil) + expect(lettings_logs[2].address_line1).to eq("wrong address line1") + expect(lettings_logs[2].address_line2).to eq("wrong address 2") + expect(lettings_logs[2].town_or_city).to eq("wrong town") + expect(lettings_logs[2].county).to eq("wrong city") + expect(lettings_logs[2].postcode_known).to eq(1) + expect(lettings_logs[2].postcode_full).to eq("A1 1AA") + expect(lettings_logs[2].la).to eq("E06000064") + end + + it "reinfers the LA if the postcode hasn't changed" do + lettings_log.update!(postcode_full: "B1 1BB") + task.invoke(all_addresses_csv_path) + lettings_log.reload + expect(lettings_log.postcode_full).to eq("B1 1BB") + expect(lettings_log.la).to eq("E08000035") + expect(lettings_log.is_la_inferred).to eq(true) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log with ID #{lettings_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + + task.invoke(all_addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_lettings_addresses_from_csv['csv_file_name']") + end + end + end + end + + describe ":import_sales_addresses_from_csv", type: :task do + subject(:task) { Rake::Task["data_import:import_sales_addresses_from_csv"] } + + let(:instance_name) { "paas_import_instance" } + let(:storage_service) { instance_double(Storage::S3Service) } + let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } + let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) } + + before do + Rake.application.rake_require("tasks/import_address_from_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:addresses_csv_path) { "addresses_reimport_123.csv" } + let(:all_addresses_csv_path) { "all_addresses_reimport_123.csv" } + let(:wrong_file_path) { "/test/no_csv_here.csv" } + let!(:sales_log) do + create(:sales_log, + :completed, + uprn_known: nil, + uprn: nil, uprn_confirmed: nil, - address_line1: "wrong address line1", - address_line2: "wrong address 2", - town_or_city: "wrong town", - county: "wrong city", - postcode_known: 1, - postcode_full: "A1 1AA", + address_line1: nil, + address_line2: nil, + town_or_city: nil, la: "E06000064", is_la_inferred: true) end + let!(:sales_logs) { create_list(:sales_log, 3, :completed, uprn_known: 1, uprn: "1", la: "E06000064", is_la_inferred: true) } + before do allow(storage_service).to receive(:get_file_io) .with("addresses_reimport_123.csv") - .and_return(replace_entity_ids(lettings_log, second_lettings_log, third_lettings_log, File.open("./spec/fixtures/files/addresses_reimport.csv").read)) - end + .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport.csv").read)) - it "updates the log address when old address was not given" do - task.invoke(addresses_csv_path) - lettings_log.reload - expect(lettings_log.uprn_known).to eq(0) - expect(lettings_log.uprn).to eq(nil) - expect(lettings_log.uprn_confirmed).to eq(nil) - expect(lettings_log.address_line1).to eq("address 1") - expect(lettings_log.address_line2).to eq("address 2") - expect(lettings_log.town_or_city).to eq("town") - expect(lettings_log.county).to eq(nil) - expect(lettings_log.postcode_known).to eq(1) - expect(lettings_log.postcode_full).to eq("B1 1BB") - expect(lettings_log.la).to eq("E08000035") - expect(lettings_log.is_la_inferred).to eq(true) + allow(storage_service).to receive(:get_file_io) + .with("all_addresses_reimport_123.csv") + .and_return(replace_entity_ids(sales_log, sales_logs[0], sales_logs[1], sales_logs[2], File.open("./spec/fixtures/files/sales_addresses_reimport_all_logs.csv").read)) end - it "updates the log address when old address was given" do - task.invoke(addresses_csv_path) - second_lettings_log.reload - expect(second_lettings_log.uprn_known).to eq(0) - expect(second_lettings_log.uprn).to eq(nil) - expect(second_lettings_log.uprn_confirmed).to eq(nil) - expect(second_lettings_log.address_line1).to eq("address 3") - expect(second_lettings_log.address_line2).to eq("address 4") - expect(second_lettings_log.town_or_city).to eq("city") - expect(second_lettings_log.county).to eq(nil) - expect(second_lettings_log.postcode_known).to eq(1) - expect(second_lettings_log.postcode_full).to eq("B1 1BB") - expect(second_lettings_log.la).to eq("E08000035") - expect(second_lettings_log.is_la_inferred).to eq(true) - end + context "when the file contains issue type column" do + it "updates the log address when old address was not given" do + task.invoke(addresses_csv_path) + sales_log.reload + expect(sales_log.uprn_known).to eq(0) + expect(sales_log.uprn).to eq(nil) + expect(sales_log.uprn_confirmed).to eq(nil) + expect(sales_log.address_line1).to eq("address 1") + expect(sales_log.address_line2).to eq("address 2") + expect(sales_log.town_or_city).to eq("town") + expect(sales_log.county).to eq("county") + expect(sales_log.pcodenk).to eq(0) + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end - it "updates the log address when new address is not given" do - task.invoke(addresses_csv_path) - third_lettings_log.reload - expect(third_lettings_log.uprn_known).to eq(0) - expect(third_lettings_log.uprn).to eq(nil) - expect(third_lettings_log.uprn_confirmed).to eq(nil) - expect(third_lettings_log.address_line1).to eq(nil) - expect(third_lettings_log.address_line2).to eq(nil) - expect(third_lettings_log.town_or_city).to eq(nil) - expect(third_lettings_log.county).to eq(nil) - expect(third_lettings_log.postcode_known).to eq(1) - expect(third_lettings_log.postcode_full).to eq("C11CC") - expect(third_lettings_log.la).to eq(nil) - expect(third_lettings_log.is_la_inferred).to eq(false) - end + it "updates the log address when old address was given" do + task.invoke(addresses_csv_path) + sales_logs[0].reload + expect(sales_logs[0].uprn_known).to eq(0) + expect(sales_logs[0].uprn).to eq(nil) + expect(sales_logs[0].uprn_confirmed).to eq(nil) + expect(sales_logs[0].address_line1).to eq("address 3") + expect(sales_logs[0].address_line2).to eq(nil) + expect(sales_logs[0].town_or_city).to eq("city") + expect(sales_logs[0].county).to eq(nil) + expect(sales_logs[0].pcodenk).to eq(0) + expect(sales_logs[0].postcode_full).to eq("B1 1BB") + expect(sales_logs[0].la).to eq("E08000035") + expect(sales_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(addresses_csv_path) + sales_logs[1].reload + expect(sales_logs[1].uprn_known).to eq(1) + expect(sales_logs[1].uprn).to eq("1") + expect(sales_logs[1].uprn_confirmed).to eq(nil) + expect(sales_logs[1].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[1].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[1].town_or_city).to eq("Westminster") + expect(sales_logs[1].county).to eq(nil) + expect(sales_logs[1].pcodenk).to eq(0) + expect(sales_logs[1].postcode_full).to eq("LS16 6FT") + expect(sales_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(addresses_csv_path) + sales_logs[2].reload + expect(sales_logs[2].uprn_known).to eq(1) + expect(sales_logs[2].uprn).to eq("1") + expect(sales_logs[2].uprn_confirmed).to eq(nil) + expect(sales_logs[2].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[2].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[2].town_or_city).to eq("Westminster") + expect(sales_logs[2].county).to eq(nil) + expect(sales_logs[2].pcodenk).to eq(0) + expect(sales_logs[2].postcode_full).to eq("LS16 6FT") + expect(sales_logs[2].la).to eq("E06000064") + end - it "logs the progress of the update" do - expect(Rails.logger).to receive(:info).with("Updated lettings log #{lettings_log.id}, with address: address 1, address 2, town, B1 1BB") - expect(Rails.logger).to receive(:info).with("Updated lettings log #{second_lettings_log.id}, with address: address 3, address 4, city, B1 1BB") - expect(Rails.logger).to receive(:info).with("Updated lettings log #{third_lettings_log.id}, with address: , , , C11CC") - expect(Rails.logger).to receive(:info).with("Lettings log ID not provided for address: address 5, address 6, city, D1 1DD") - expect(Rails.logger).to receive(:info).with("Could not find a lettings log with id fake_id") + it "reinfers the LA if the postcode hasn't changed" do + sales_log.update!(postcode_full: "B1 1BB") + task.invoke(addresses_csv_path) + sales_log.reload + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end - task.invoke(addresses_csv_path) + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a sales log with id fake_id") + + task.invoke(addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']") + end end - it "raises an error when no path is given" do - expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_address_from_csv['csv_file_name']") + context "when the file does not contain issue type column" do + it "updates the log address when old address was not given" do + task.invoke(all_addresses_csv_path) + sales_log.reload + expect(sales_log.uprn_known).to eq(0) + expect(sales_log.uprn).to eq(nil) + expect(sales_log.uprn_confirmed).to eq(nil) + expect(sales_log.address_line1).to eq("address 1") + expect(sales_log.address_line2).to eq("address 2") + expect(sales_log.town_or_city).to eq("town") + expect(sales_log.county).to eq("county") + expect(sales_log.pcodenk).to eq(0) + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end + + it "updates the log address when old address was given" do + task.invoke(all_addresses_csv_path) + sales_logs[0].reload + expect(sales_logs[0].uprn_known).to eq(0) + expect(sales_logs[0].uprn).to eq(nil) + expect(sales_logs[0].uprn_confirmed).to eq(nil) + expect(sales_logs[0].address_line1).to eq("address 3") + expect(sales_logs[0].address_line2).to eq(nil) + expect(sales_logs[0].town_or_city).to eq("city") + expect(sales_logs[0].county).to eq(nil) + expect(sales_logs[0].pcodenk).to eq(0) + expect(sales_logs[0].postcode_full).to eq("B1 1BB") + expect(sales_logs[0].la).to eq("E08000035") + expect(sales_logs[0].is_la_inferred).to eq(true) + end + + it "does not update log address when uprn is given" do + task.invoke(all_addresses_csv_path) + sales_logs[1].reload + expect(sales_logs[1].uprn_known).to eq(1) + expect(sales_logs[1].uprn).to eq("1") + expect(sales_logs[1].uprn_confirmed).to eq(nil) + expect(sales_logs[1].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[1].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[1].town_or_city).to eq("Westminster") + expect(sales_logs[1].county).to eq(nil) + expect(sales_logs[1].pcodenk).to eq(0) + expect(sales_logs[1].postcode_full).to eq("LS16 6FT") + expect(sales_logs[1].la).to eq("E06000064") + end + + it "does not update log address when all required address fields are not present" do + task.invoke(all_addresses_csv_path) + sales_logs[2].reload + expect(sales_logs[2].uprn_known).to eq(1) + expect(sales_logs[2].uprn).to eq("1") + expect(sales_logs[2].uprn_confirmed).to eq(nil) + expect(sales_logs[2].address_line1).to eq("Wrong Address Line1") + expect(sales_logs[2].address_line2).to eq("Double Dependent Locality") + expect(sales_logs[2].town_or_city).to eq("Westminster") + expect(sales_logs[2].county).to eq(nil) + expect(sales_logs[2].pcodenk).to eq(0) + expect(sales_logs[2].postcode_full).to eq("LS16 6FT") + expect(sales_logs[2].la).to eq("E06000064") + end + + it "reinfers the LA if the postcode hasn't changed" do + sales_log.update!(postcode_full: "B1 1BB") + task.invoke(all_addresses_csv_path) + sales_log.reload + expect(sales_log.postcode_full).to eq("B1 1BB") + expect(sales_log.la).to eq("E08000035") + expect(sales_log.is_la_inferred).to eq(true) + end + + it "logs the progress of the update" do + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_log.id}, with address: address 1, address 2, town, county, B1 1BB") + expect(Rails.logger).to receive(:info).with("Updated sales log #{sales_logs[0].id}, with address: address 3, , city, , B1 1BB") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[1].id} contains uprn, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log with ID #{sales_logs[2].id} is missing required address data, skipping log") + expect(Rails.logger).to receive(:info).with("Sales log ID not provided for address: Some Place, , Bristol, , BS1 1AD") + expect(Rails.logger).to receive(:info).with("Could not find a sales log with id fake_id") + + task.invoke(all_addresses_csv_path) + end + + it "raises an error when no path is given" do + expect { task.invoke(nil) }.to raise_error(RuntimeError, "Usage: rake data_import:import_sales_addresses_from_csv['csv_file_name']") + end end end end diff --git a/spec/lib/tasks/send_missing_addresses_csv_spec.rb b/spec/lib/tasks/send_missing_addresses_csv_spec.rb new file mode 100644 index 000000000..dab36f1ad --- /dev/null +++ b/spec/lib/tasks/send_missing_addresses_csv_spec.rb @@ -0,0 +1,560 @@ +require "rails_helper" +require "rake" + +RSpec.describe "correct_addresses" do + describe ":send_missing_addresses_lettings_csv", type: :task do + subject(:task) { Rake::Task["correct_addresses:send_missing_addresses_lettings_csv"] } + + before do + organisation.users.destroy_all + Rake.application.rake_require("tasks/send_missing_addresses_csv") + Rake::Task.define_task(:environment) + task.reenable + + body_1 = { + results: [ + { + DPA: { + "POSTCODE": "BS1 1AD", + "POST_TOWN": "Bristol", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + body_2 = { + results: [ + { + DPA: { + "POSTCODE": "EC1N 2TD", + "POST_TOWN": "Newcastle", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123") + .to_return(status: 200, body: body_1, headers: {}) + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=12") + .to_return(status: 200, body: body_2, headers: {}) + end + + context "when the rake task is run" do + let(:organisation) { create(:organisation, name: "test organisation") } + + before do + stub_const("EmailMissingAddressesCsvJob::MISSING_ADDRESSES_THRESHOLD", 5) + end + + context "when org has more than 5 missing addresses and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:lettings_log, 7, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, needstype: 1, old_form_id: "form_1", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "lettings", %w[missing_address], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 missing addresses and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, needstype: 1, old_form_id: "form_2", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[missing_address], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 missing addresses" do + before do + create_list(:lettings_log, 3, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, needstype: 1, old_form_id: "form_2", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + create_list(:lettings_log, 2, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: nil, needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job with organisations that is missing less addresses than threshold amount" do + expect { task.invoke }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when org has more than 5 missing town_or_city and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:lettings_log, 7, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: "exists", town_or_city: nil, needstype: 1, old_form_id: "form_1", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "lettings", %w[missing_town], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 missing town or city and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: "exists", town_or_city: nil, needstype: 1, old_form_id: "form_2", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[missing_town], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 missing town or city" do + before do + create_list(:lettings_log, 3, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: "address", town_or_city: nil, needstype: 1, old_form_id: "form_2", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + create_list(:lettings_log, 2, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: "address", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job with organisations that is missing less town or city data than threshold amount" do + expect { task.invoke }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when org has more than 5 wrong uprn and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:lettings_log, 7, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "123", town_or_city: "Bristol", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "lettings", %w[bristol_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 wrong uprn and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "12", propcode: "12", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[wrong_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 wrong uprn" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 3, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "123", town_or_city: "Bristol", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + create_list(:lettings_log, 2, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "12", tenancycode: "12", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[wrong_uprn bristol_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing lettings addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org is included in skip_uprn_issue_organisations list" do + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "12", propcode: "12", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job" do + expect { task.invoke(organisation.id.to_s) }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when org is included in skip_uprn_issue_organisations list but faces a different issue" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, needstype: 1, old_form_id: "form_2", owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "12", propcode: "12", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job" do + expect { task.invoke(organisation.id.to_s) }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[missing_address], [organisation.id]) + end + end + + context "when skip_uprn_issue_organisations list is provided" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:lettings_log, 5, :imported, startdate: Time.zone.local(2023, 9, 9), uprn: "12", propcode: "12", needstype: 1, owning_organisation: organisation, managing_organisation: organisation, created_by: organisation.users.first) + end + + it "does enqueues the job with correct skip_uprn_issue_organisations" do + expect { task.invoke("100 400") }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "lettings", %w[wrong_uprn], [100, 400]) + end + end + end + end + + describe ":send_missing_addresses_sales_csv", type: :task do + subject(:task) { Rake::Task["correct_addresses:send_missing_addresses_sales_csv"] } + + before do + organisation.users.destroy_all + Rake.application.rake_require("tasks/send_missing_addresses_csv") + Rake::Task.define_task(:environment) + task.reenable + + body_1 = { + results: [ + { + DPA: { + "POSTCODE": "BS1 1AD", + "POST_TOWN": "Bristol", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + body_2 = { + results: [ + { + DPA: { + "POSTCODE": "EC1N 2TD", + "POST_TOWN": "Newcastle", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123") + .to_return(status: 200, body: body_1, headers: {}) + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=12") + .to_return(status: 200, body: body_2, headers: {}) + end + + context "when the rake task is run" do + let(:organisation) { create(:organisation, name: "test organisation") } + + before do + stub_const("EmailMissingAddressesCsvJob::MISSING_ADDRESSES_THRESHOLD", 5) + end + + context "when org has more than 5 missing addresses and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:sales_log, 7, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, old_form_id: "form_1", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke("70 90") }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "sales", %w[missing_address], [70, 90]) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 missing addresses and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:sales_log, 5, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, old_form_id: "form_2", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "sales", %w[missing_address], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 missing addresses" do + before do + create_list(:sales_log, 3, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: nil, town_or_city: nil, old_form_id: "form_2", owning_organisation: organisation, created_by: organisation.users.first) + create_list(:sales_log, 2, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: nil, owning_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job with organisations that is missing less addresses than threshold amount" do + expect { task.invoke }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when org has more than 5 missing town_or_city and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:sales_log, 7, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: "exists", town_or_city: nil, old_form_id: "form_1", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "sales", %w[missing_town], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 missing town or city and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:sales_log, 5, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: "exists", town_or_city: nil, old_form_id: "form_2", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "sales", %w[missing_town], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 missing town or city" do + before do + create_list(:sales_log, 3, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: "address", town_or_city: nil, old_form_id: "form_2", owning_organisation: organisation, created_by: organisation.users.first) + create_list(:sales_log, 2, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), address_line1: "address", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job with organisations that is missing less town or city data than threshold amount" do + expect { task.invoke }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when org has more than 5 wrong uprn and data coordinators" do + let!(:data_coordinator) { create(:user, :data_coordinator, organisation:, email: "data_coordinator1@example.com") } + let!(:data_coordinator2) { create(:user, :data_coordinator, organisation:, email: "data_coordinator2@example.com") } + + before do + create(:user, :data_provider, organisation:, email: "data_provider1@example.com") + create_list(:sales_log, 7, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "123", town_or_city: "Bristol", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_coordinator.id, data_coordinator2.id), organisation, "sales", %w[wrong_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_coordinator1@example.com, data_coordinator2@example.com") + task.invoke + end + end + + context "when org has 5 wrong uprn and data providers only" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:sales_log, 5, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "12", purchid: "12", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "sales", %w[wrong_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org has less than 5 wrong uprn" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:sales_log, 3, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "123", town_or_city: "Bristol", owning_organisation: organisation, created_by: organisation.users.first) + create_list(:sales_log, 2, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "12", purchid: "12", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "enqueues the job with correct organisations" do + expect { task.invoke }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "sales", %w[wrong_uprn], []) + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Sending missing sales addresses CSV for test organisation to data_provider3@example.com, data_provider4@example.com") + task.invoke + end + end + + context "when org is included in skip_uprn_issue_organisations list" do + before do + create_list(:sales_log, 5, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "12", purchid: "12", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "does not enqueue the job" do + expect { task.invoke("#{organisation.id} 4") }.not_to enqueue_job(EmailMissingAddressesCsvJob) + end + end + + context "when skip_uprn_issue_organisations list is provided" do + let!(:data_provider) { create(:user, :data_provider, organisation:, email: "data_provider3@example.com") } + let!(:data_provider2) { create(:user, :data_provider, organisation:, email: "data_provider4@example.com") } + + before do + create_list(:sales_log, 5, :completed, :imported, saledate: Time.zone.local(2023, 9, 9), uprn_known: 1, uprn: "12", purchid: "12", owning_organisation: organisation, created_by: organisation.users.first) + end + + it "does enqueues the job with correct skip_uprn_issue_organisations" do + expect { task.invoke("100 400") }.to enqueue_job(EmailMissingAddressesCsvJob).with(include(data_provider.id, data_provider2.id), organisation, "sales", %w[wrong_uprn], [100, 400]) + end + end + end + end + + describe ":create_lettings_addresses_csv", type: :task do + subject(:task) { Rake::Task["correct_addresses:create_lettings_addresses_csv"] } + + before do + organisation.users.destroy_all + Rake.application.rake_require("tasks/send_missing_addresses_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:organisation) { create(:organisation, name: "test organisation") } + + context "and organisation ID is provided" do + it "enqueues the job with correct organisation" do + expect { task.invoke(organisation.id) }.to enqueue_job(CreateAddressesCsvJob).with(organisation, "lettings") + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Creating lettings addresses CSV for test organisation") + task.invoke(organisation.id) + end + end + + context "when organisation with given ID cannot be found" do + it "prints out error" do + expect(Rails.logger).to receive(:error).with("Organisation with ID fake not found") + task.invoke("fake") + end + end + + context "when organisation ID is not given" do + it "raises an error" do + expect { task.invoke }.to raise_error(RuntimeError, "Usage: rake correct_addresses:create_lettings_addresses_csv['organisation_id']") + end + end + end + end + + describe ":create_sales_addresses_csv", type: :task do + subject(:task) { Rake::Task["correct_addresses:create_sales_addresses_csv"] } + + before do + organisation.users.destroy_all + Rake.application.rake_require("tasks/send_missing_addresses_csv") + Rake::Task.define_task(:environment) + task.reenable + end + + context "when the rake task is run" do + let(:organisation) { create(:organisation, name: "test organisation") } + + context "and organisation ID is provided" do + it "enqueues the job with correct organisation" do + expect { task.invoke(organisation.id) }.to enqueue_job(CreateAddressesCsvJob).with(organisation, "sales") + end + + it "prints out the jobs enqueued" do + expect(Rails.logger).to receive(:info).with(nil) + expect(Rails.logger).to receive(:info).with("Creating sales addresses CSV for test organisation") + task.invoke(organisation.id) + end + end + + context "when organisation with given ID cannot be found" do + it "prints out error" do + expect(Rails.logger).to receive(:error).with("Organisation with ID fake not found") + task.invoke("fake") + end + end + + context "when organisation ID is not given" do + it "raises an error" do + expect { task.invoke }.to raise_error(RuntimeError, "Usage: rake correct_addresses:create_sales_addresses_csv['organisation_id']") + end + end + end + end +end diff --git a/spec/mailers/csv_download_mailer_spec.rb b/spec/mailers/csv_download_mailer_spec.rb index 6c145c508..704d71e1e 100644 --- a/spec/mailers/csv_download_mailer_spec.rb +++ b/spec/mailers/csv_download_mailer_spec.rb @@ -1,15 +1,15 @@ require "rails_helper" RSpec.describe CsvDownloadMailer do - describe "#send_csv_download_mail" do - let(:notify_client) { instance_double(Notifications::Client) } - let(:user) { FactoryBot.create(:user, email: "user@example.com") } + let(:notify_client) { instance_double(Notifications::Client) } + let(:user) { FactoryBot.create(:user, email: "user@example.com") } - before do - allow(Notifications::Client).to receive(:new).and_return(notify_client) - allow(notify_client).to receive(:send_email).and_return(true) - end + before do + allow(Notifications::Client).to receive(:new).and_return(notify_client) + allow(notify_client).to receive(:send_email).and_return(true) + end + describe "#send_csv_download_mail" do it "sends a CSV download E-mail via notify" do link = :link duration = 20.minutes.to_i @@ -27,4 +27,57 @@ RSpec.describe CsvDownloadMailer do described_class.new.send_csv_download_mail(user, link, duration) end end + + describe "#send_missing_lettings_addresses_csv_download_mail" do + it "sends a CSV download E-mail via notify" do + link = :link + duration = 20.minutes.to_i + + expect(notify_client).to receive(:send_email).with( + email_address: user.email, + template_id: described_class::CSV_MISSING_LETTINGS_ADDRESSES_DOWNLOAD_TEMPLATE_ID, + personalisation: { + name: user.name, + issue_explanation: "We have found this issue in your logs imported to the new version of CORE: +## Missing town or city +The town or city in some logs is missing. This data is required in the new version of CORE.\n", + how_to_fix: "You need to:\n +- download [this spreadsheet for lettings logs](#{link}). This link will expire in one week. To request another link, [contact the CORE helpdesk](https://dluhcdigital.atlassian.net/servicedesk/customer/portal/6/group/11). +- fill in the missing address data +- check that the existing address data is correct\n", + duration: "20 minutes", + }, + ) + + described_class.new.send_missing_lettings_addresses_csv_download_mail(user, link, duration, %w[missing_town]) + end + end + + describe "#send_missing_sales_addresses_csv_download_mail" do + it "sends a CSV download E-mail via notify" do + link = :link + duration = 20.minutes.to_i + + expect(notify_client).to receive(:send_email).with( + email_address: user.email, + template_id: described_class::CSV_MISSING_SALES_ADDRESSES_DOWNLOAD_TEMPLATE_ID, + personalisation: { + name: user.name, + issue_explanation: "We have found this issue in your logs imported to the new version of CORE: +## Incorrect UPRN\nThe UPRN in some logs may be incorrect, so the wrong address data may have been imported. + +In some of your logs, the UPRN is the same as the purchaser code, but these are different things. Purchaser codes are codes that your organisation uses to identify properties. UPRNs are unique numbers assigned by the Ordnance Survey. + +If a log has the correct UPRN, leave the UPRN unchanged. If the UPRN is incorrect, clear the value and provide the full address instead. Alternatively, you can change the UPRN on the CORE system.\n", + how_to_fix: "You need to:\n +- download [this spreadsheet for sales logs](#{link}). This link will expire in one week. To request another link, [contact the CORE helpdesk](https://dluhcdigital.atlassian.net/servicedesk/customer/portal/6/group/11). +- check that the address data is correct +- correct any address errors\n", + duration: "20 minutes", + }, + ) + + described_class.new.send_missing_sales_addresses_csv_download_mail(user, link, duration, %w[wrong_uprn]) + end + end end diff --git a/spec/services/csv/missing_addresses_csv_service_spec.rb b/spec/services/csv/missing_addresses_csv_service_spec.rb new file mode 100644 index 000000000..e8482f4cb --- /dev/null +++ b/spec/services/csv/missing_addresses_csv_service_spec.rb @@ -0,0 +1,512 @@ +require "rails_helper" + +RSpec.describe Csv::MissingAddressesCsvService do + let(:organisation) { create(:organisation, name: "Address org") } + let(:user) { create(:user, organisation:, email: "testy@example.com") } + let(:service) { described_class.new(organisation, skip_uprn_issue_organisations) } + let(:skip_uprn_issue_organisations) { [100, 200] } + + before do + body_1 = { + results: [ + { + DPA: { + "POSTCODE": "BS1 1AD", + "POST_TOWN": "Bristol", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + body_2 = { + results: [ + { + DPA: { + "POSTCODE": "EC1N 2TD", + "POST_TOWN": "Newcastle", + "ORGANISATION_NAME": "Some place", + }, + }, + ], + }.to_json + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=123") + .to_return(status: 200, body: body_1, headers: {}) + + stub_request(:get, "https://api.os.uk/search/places/v1/uprn?key=OS_DATA_KEY&uprn=12") + .to_return(status: 200, body: body_2, headers: {}) + end + + def replace_entity_ids(lettings_log, export_template) + export_template.sub!(/\{id\}/, lettings_log.id.to_s) + end + + describe "#create_missing_lettings_addresses_csv" do + let!(:lettings_log) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + address_line1: nil, + town_or_city: nil, + old_id: "old_id", + old_form_id: "old_form_id", + needstype: 1, + uprn_known: 0) + end + + let!(:lettings_log_missing_town) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + address_line1: "existing address", + town_or_city: nil, + old_id: "older_id", + old_form_id: "old_form_id", + needstype: 1, + uprn_known: 0) + end + + let!(:lettings_log_wrong_uprn) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + uprn: "123", + uprn_known: 1, + old_id: "oldest_id", + needstype: 1) + end + + context "when the organisation has logs with missing addresses, missing town or city and wrong uprn" do + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(lettings_log, File.open("spec/fixtures/files/missing_lettings_logs_addresses_all_issues.csv").read) + expected_content = replace_entity_ids(lettings_log_missing_town, expected_content) + expected_content = replace_entity_ids(lettings_log_wrong_uprn, expected_content) + csv = service.create_missing_lettings_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with missing addresses only" do + before do + lettings_log_missing_town.update!(town_or_city: "towncity") + lettings_log_wrong_uprn.update!(uprn: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(lettings_log, File.open("spec/fixtures/files/missing_lettings_logs_addresses.csv").read) + csv = service.create_missing_lettings_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with missing town or city only" do + before do + lettings_log.update!(address_line1: "existing address", town_or_city: "towncity") + lettings_log_wrong_uprn.update!(uprn: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(lettings_log_missing_town, File.open("spec/fixtures/files/missing_lettings_logs_town_or_city.csv").read) + csv = service.create_missing_lettings_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with wrong uprn only" do + before do + lettings_log.update!(address_line1: "existing address", town_or_city: "towncity") + lettings_log_missing_town.update!(town_or_city: "towncity") + lettings_log_wrong_uprn.update!(uprn: "12", propcode: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(lettings_log_wrong_uprn, File.open("spec/fixtures/files/missing_lettings_logs_wrong_uprn.csv").read) + csv = service.create_missing_lettings_addresses_csv + expect(csv).to eq(expected_content) + end + + context "and the organisation is marked as an organisation to skip" do + let(:skip_uprn_issue_organisations) { [organisation.id] } + + it "returns nil" do + expect(service.create_missing_lettings_addresses_csv).to be_nil + end + end + end + + context "when the organisation only has supported housing logs with missing addresses or town or city" do + before do + lettings_log.update!(needstype: 2) + lettings_log_missing_town.update!(needstype: 2) + lettings_log_wrong_uprn.update!(needstype: 2) + end + + it "returns nil" do + expect(service.create_missing_lettings_addresses_csv).to be_nil + end + end + + context "when the organisation only has logs with missing addresses or town or city from 2022" do + before do + lettings_log.update!(startdate: Time.zone.local(2022, 4, 5)) + lettings_log_missing_town.update!(startdate: Time.zone.local(2022, 4, 5)) + lettings_log_wrong_uprn.update!(startdate: Time.zone.local(2022, 4, 5)) + end + + it "returns nil" do + expect(service.create_missing_lettings_addresses_csv).to be_nil + end + end + + context "when the organisation has any address and town or city fields filled in or correct uprn" do + before do + lettings_log.update!(address_line1: "address_line1", town_or_city: "towncity") + lettings_log_missing_town.update!(address_line1: "address_line1", town_or_city: "towncity") + lettings_log_wrong_uprn.update!(uprn: "12") + end + + it "returns nil" do + expect(service.create_missing_lettings_addresses_csv).to be_nil + end + end + end + + describe "#create_missing_sales_addresses_csv" do + let!(:sales_log) do + create(:sales_log, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + address_line1: nil, + town_or_city: nil, + old_id: "old_id", + old_form_id: "old_form_id", + uprn_known: 0) + end + + let!(:sales_log_missing_town) do + create(:sales_log, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + address_line1: "existing address line 1", + town_or_city: nil, + old_id: "older_id", + old_form_id: "old_form_id", + uprn_known: 0) + end + + let!(:sales_log_wrong_uprn) do + create(:sales_log, + :completed, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + uprn: "123", + town_or_city: "Bristol", + old_id: "oldest_id", + uprn_known: 1, + uprn_confirmed: 1, + la: nil) + end + + context "when the organisation has logs with missing addresses, town or city and wrong uprn" do + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(sales_log, File.open("spec/fixtures/files/missing_sales_logs_addresses_all_issues.csv").read) + expected_content = replace_entity_ids(sales_log_missing_town, expected_content) + expected_content = replace_entity_ids(sales_log_wrong_uprn, expected_content) + csv = service.create_missing_sales_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with missing addresses" do + before do + sales_log_missing_town.update!(town_or_city: "towncity") + sales_log_wrong_uprn.update!(uprn: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(sales_log, File.open("spec/fixtures/files/missing_sales_logs_addresses.csv").read) + csv = service.create_missing_sales_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with missing town_or_city only" do + before do + sales_log.update!(address_line1: "address", town_or_city: "towncity") + sales_log_wrong_uprn.update!(uprn: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(sales_log_missing_town, File.open("spec/fixtures/files/missing_sales_logs_town_or_city.csv").read) + csv = service.create_missing_sales_addresses_csv + expect(csv).to eq(expected_content) + end + end + + context "when the organisation has logs with wrong uprn only" do + before do + sales_log.update!(address_line1: "address", town_or_city: "towncity") + sales_log_missing_town.update!(town_or_city: "towncity") + sales_log_wrong_uprn.update!(uprn: "12", purchid: "12") + end + + it "returns a csv with relevant logs" do + expected_content = replace_entity_ids(sales_log_wrong_uprn, File.open("spec/fixtures/files/missing_sales_logs_wrong_uprn.csv").read) + csv = service.create_missing_sales_addresses_csv + expect(csv).to eq(expected_content) + end + + context "and the organisation is marked as an organisation to skip" do + let(:skip_uprn_issue_organisations) { [organisation.id] } + + it "returns nil" do + expect(service.create_missing_sales_addresses_csv).to be_nil + end + end + end + + context "when the organisation only has logs with missing addresses from 2022" do + before do + sales_log.update!(saledate: Time.zone.local(2022, 4, 5)) + sales_log_missing_town.update!(saledate: Time.zone.local(2022, 4, 5)) + sales_log_wrong_uprn.update!(saledate: Time.zone.local(2022, 4, 5)) + end + + it "returns nil" do + expect(service.create_missing_sales_addresses_csv).to be_nil + end + end + + context "when the organisation has address fields filled in" do + before do + sales_log.update!(town_or_city: "town", address_line1: "line1") + sales_log_missing_town.update!(town_or_city: "town") + sales_log_wrong_uprn.update!(uprn: "12") + end + + it "returns nil" do + expect(service.create_missing_sales_addresses_csv).to be_nil + end + end + end + + describe "#create_lettings_addresses_csv" do + context "when the organisation has lettings logs" do + let!(:lettings_log) do + create(:lettings_log, + tenancycode: "tenancycode1", + propcode: "propcode1", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + address_line1: "address", + town_or_city: "town", + old_id: "old_id_1", + old_form_id: "old_form_id_1", + needstype: 1, + uprn_known: 0) + end + + let!(:lettings_log_missing_address) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + address_line1: nil, + town_or_city: nil, + old_id: "old_id", + old_form_id: "old_form_id", + needstype: 1, + uprn_known: 0) + end + + let!(:lettings_log_missing_town) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + address_line1: "existing address", + town_or_city: nil, + old_id: "older_id", + old_form_id: "old_form_id", + needstype: 1, + uprn_known: 0) + end + + let!(:lettings_log_wrong_uprn) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + uprn: "123", + uprn_known: 1, + old_id: "oldest_id", + needstype: 1) + end + + let!(:lettings_log_not_imported) do + create(:lettings_log, + tenancycode: "tenancycode", + propcode: "propcode", + startdate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + managing_organisation: organisation, + uprn: "123", + uprn_known: 1, + needstype: 1) + end + + before do + create(:lettings_log, managing_organisation: organisation, old_id: "exists", startdate: Time.zone.local(2022, 4, 5)) + end + + it "returns a csv with relevant logs" do + csv = CSV.parse(service.create_lettings_addresses_csv) + expect(csv.count).to eq(6) + expect(csv).to include([lettings_log.id.to_s, "2023-04-05", "tenancycode1", "propcode1", "testy@example.com", "Address org", "Address org", nil, "address", nil, "town", nil, nil]) + expect(csv).to include([lettings_log_missing_address.id.to_s, "2023-04-05", "tenancycode", "propcode", "testy@example.com", "Address org", "Address org", nil, nil, nil, nil, nil, nil]) + expect(csv).to include([lettings_log_missing_town.id.to_s, "2023-04-05", "tenancycode", "propcode", "testy@example.com", "Address org", "Address org", nil, "existing address", nil, nil, nil, nil]) + expect(csv).to include([lettings_log_wrong_uprn.id.to_s, "2023-04-05", "tenancycode", "propcode", "testy@example.com", "Address org", "Address org", "123", "Some Place", nil, "Bristol", nil, "BS1 1AD"]) + expect(csv).to include([lettings_log_not_imported.id.to_s, "2023-04-05", "tenancycode", "propcode", "testy@example.com", "Address org", "Address org", "123", "Some Place", nil, "Bristol", nil, "BS1 1AD"]) + end + end + + context "when the organisation does not have relevant lettings logs" do + before do + create(:lettings_log, managing_organisation: organisation, startdate: Time.zone.local(2022, 4, 5)) + end + + it "returns only headers" do + csv = service.create_lettings_addresses_csv + expect(csv).to eq "Log ID,Tenancy start date,Tenant code,Property reference,Log owner,Owning organisation,Managing organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode\n" + end + end + end + + describe "#create_sales_addresses_csv" do + context "when the organisation has sales" do + let!(:sales_log) do + create(:sales_log, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + address_line1: "address", + town_or_city: "city", + old_id: "old_id_1", + old_form_id: "old_form_id_1", + uprn_known: 0) + end + + let!(:sales_log_missing_address) do + create(:sales_log, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + address_line1: nil, + town_or_city: nil, + old_id: "old_id", + old_form_id: "old_form_id", + uprn_known: 0) + end + + let!(:sales_log_missing_town) do + create(:sales_log, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + address_line1: "existing address line 1", + town_or_city: nil, + old_id: "older_id", + old_form_id: "old_form_id", + uprn_known: 0) + end + + let!(:sales_log_wrong_uprn) do + create(:sales_log, + :completed, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + uprn: "123", + town_or_city: "Bristol", + old_id: "oldest_id", + uprn_known: 1, + uprn_confirmed: 1, + la: nil) + end + + let!(:sales_log_not_imported) do + create(:sales_log, + :completed, + purchid: "purchaser code", + saledate: Time.zone.local(2023, 4, 5), + created_by: user, + owning_organisation: organisation, + uprn: "123", + town_or_city: "Bristol", + uprn_known: 1, + uprn_confirmed: 1, + la: nil) + end + + before do + create(:sales_log, :completed, saledate: Time.zone.local(2022, 4, 5)) + end + + it "returns a csv with relevant logs" do + csv = CSV.parse(service.create_sales_addresses_csv) + expect(csv.count).to eq(6) + expect(csv).to include([sales_log.id.to_s, "2023-04-05", "purchaser code", "testy@example.com", "Address org", nil, "address", nil, "city", nil, nil]) + expect(csv).to include([sales_log_missing_address.id.to_s, "2023-04-05", "purchaser code", "testy@example.com", "Address org", nil, nil, nil, nil, nil, nil]) + expect(csv).to include([sales_log_missing_town.id.to_s, "2023-04-05", "purchaser code", "testy@example.com", "Address org", nil, "existing address line 1", nil, nil, nil, nil]) + expect(csv).to include([sales_log_wrong_uprn.id.to_s, "2023-04-05", "purchaser code", "testy@example.com", "Address org", "123", "Some Place", nil, "Bristol", nil, "BS1 1AD"]) + expect(csv).to include([sales_log_not_imported.id.to_s, "2023-04-05", "purchaser code", "testy@example.com", "Address org", "123", "Some Place", nil, "Bristol", nil, "BS1 1AD"]) + end + end + + context "when the organisation does not have relevant sales logs" do + before do + create(:sales_log, :completed, saledate: Time.zone.local(2022, 4, 5)) + end + + it "returns only headers" do + csv = service.create_sales_addresses_csv + expect(csv).to eq("Log ID,Sale completion date,Purchaser code,Log owner,Owning organisation,UPRN,Address Line 1,Address Line 2 (optional),Town or City,County (optional),Property's postcode\n") + end + end + end +end