From 197e2ae4cb1368354e529120ba810b1004e8086e Mon Sep 17 00:00:00 2001 From: Phil Lee Date: Thu, 12 Jan 2023 13:37:44 +0000 Subject: [PATCH] [1778] Bulk upload errors (#1091) * lockdown bulk upload routes * able to view lettings bulk upload errors * add error count to bulk upload results * coverage for bulk upload filename on results * group bulk upload errors by row on results * able to view bulk upload sales results * scope lettings and sales bulk upload results * fix linting * call service correctly in test * add bulk upload sales questions mapping * appease linter * bulk upload error shows correct question - depending on log type it will show relevant question for the field concerned * use local disk for bulk upload for dev env - this saves the need to connect to S3 to play with bulk upload in dev environment * improve namespacing of classes * add job to process bulk uploads * use local disk storage for dev file upload * fix test * use inline active job queue_adapter for dev * use test active job queue adapter for test env * remove rubocop violation * delete bulk upload from disk after processing * populate errors with cell, row + metadata * update error message with something meaningful * shim in sales validator * able to parse sales bulk uploads * change migration to add purchase_code to errors * bulk upload error component renders purchaser code - when a sales log * populate purchaser_code for bulk upload errors - when log type is sales * remove superfluous private method --- .../bulk_upload_error_row_component.html.erb | 33 +++ .../bulk_upload_error_row_component.rb | 48 ++++ ...bulk_upload_lettings_results_controller.rb | 9 + .../bulk_upload_sales_results_controller.rb | 9 + app/jobs/process_bulk_upload_job.rb | 7 + app/models/bulk_upload.rb | 5 + app/models/bulk_upload_error.rb | 3 + .../bulk_upload_lettings/upload_your_file.rb | 15 +- .../bulk_upload_sales/upload_your_file.rb | 19 +- app/models/user.rb | 1 + app/services/bulk_upload/downloader.rb | 18 +- .../bulk_upload/lettings/row_parser.rb | 185 +++++++++++++ .../bulk_upload/lettings/validator.rb | 250 ++++++++++++++++++ app/services/bulk_upload/processor.rb | 42 +++ app/services/bulk_upload/sales/row_parser.rb | 176 ++++++++++++ app/services/bulk_upload/sales/validator.rb | 228 ++++++++++++++++ app/services/storage/local_disk_service.rb | 26 ++ .../show.html.erb | 20 ++ .../bulk_upload_sales_results/show.html.erb | 20 ++ config/environments/development.rb | 2 + config/environments/test.rb | 2 + config/routes.rb | 8 +- ...0221209161927_create_bulk_upload_errors.rb | 19 ++ .../20230103100531_rename_purchaser_code.rb | 5 + db/schema.rb | 14 + .../bulk_upload_error_row_component_spec.rb | 75 ++++++ spec/factories/bulk_upload.rb | 9 + spec/factories/bulk_upload_error.rb | 14 + .../files/2021_22_lettings_bulk_upload.csv | 20 ++ .../files/2022_23_sales_bulk_upload.csv | 119 +++++++++ .../upload_your_file_spec.rb | 6 + .../upload_your_file_spec.rb | 6 + ...upload_lettings_results_controller_spec.rb | 57 ++++ ...lk_upload_sales_results_controller_spec.rb | 70 +++++ spec/services/bulk_upload/downloader_spec.rb | 23 +- .../bulk_upload/lettings/row_parser_spec.rb | 81 ++++++ .../bulk_upload/lettings/validator_spec.rb | 39 +++ spec/services/bulk_upload/processor_spec.rb | 34 +++ .../bulk_upload/sales/row_parser_spec.rb | 21 ++ .../bulk_upload/sales/validator_spec.rb | 47 ++++ 40 files changed, 1768 insertions(+), 17 deletions(-) create mode 100644 app/components/bulk_upload_error_row_component.html.erb create mode 100644 app/components/bulk_upload_error_row_component.rb create mode 100644 app/controllers/bulk_upload_lettings_results_controller.rb create mode 100644 app/controllers/bulk_upload_sales_results_controller.rb create mode 100644 app/jobs/process_bulk_upload_job.rb create mode 100644 app/models/bulk_upload_error.rb create mode 100644 app/services/bulk_upload/lettings/row_parser.rb create mode 100644 app/services/bulk_upload/lettings/validator.rb create mode 100644 app/services/bulk_upload/processor.rb create mode 100644 app/services/bulk_upload/sales/row_parser.rb create mode 100644 app/services/bulk_upload/sales/validator.rb create mode 100644 app/services/storage/local_disk_service.rb create mode 100644 app/views/bulk_upload_lettings_results/show.html.erb create mode 100644 app/views/bulk_upload_sales_results/show.html.erb create mode 100644 db/migrate/20221209161927_create_bulk_upload_errors.rb create mode 100644 db/migrate/20230103100531_rename_purchaser_code.rb create mode 100644 spec/components/bulk_upload_error_row_component_spec.rb create mode 100644 spec/factories/bulk_upload_error.rb create mode 100644 spec/fixtures/files/2021_22_lettings_bulk_upload.csv create mode 100644 spec/fixtures/files/2022_23_sales_bulk_upload.csv create mode 100644 spec/requests/bulk_upload_lettings_results_controller_spec.rb create mode 100644 spec/requests/bulk_upload_sales_results_controller_spec.rb create mode 100644 spec/services/bulk_upload/lettings/row_parser_spec.rb create mode 100644 spec/services/bulk_upload/lettings/validator_spec.rb create mode 100644 spec/services/bulk_upload/processor_spec.rb create mode 100644 spec/services/bulk_upload/sales/row_parser_spec.rb create mode 100644 spec/services/bulk_upload/sales/validator_spec.rb diff --git a/app/components/bulk_upload_error_row_component.html.erb b/app/components/bulk_upload_error_row_component.html.erb new file mode 100644 index 000000000..6f8de6919 --- /dev/null +++ b/app/components/bulk_upload_error_row_component.html.erb @@ -0,0 +1,33 @@ +
+
+ <% if lettings? %> +

Row <%= row %> Tenant code: <%= tenant_code %> Property reference: <%= property_ref %>

+ <% else %> +

Row <%= row %> Purchaser code: <%= purchaser_code %>

+ <% end %> +
+ +
+ <%= govuk_table do |table| %> + <% table.head do |head| %> + <% head.row do |row| %> + <% row.cell(header: true, text: "Cell") %> + <% row.cell(header: true, text: "Question") %> + <% row.cell(header: true, text: "Error") %> + <% row.cell(header: true, text: "Specification") %> + <% end %> + + <% table.body do |body| %> + <% bulk_upload_errors.each do |error| %> + <% body.row do |row| %> + <% row.cell(header: true, text: error.cell) %> + <% row.cell(text: question_for_field(error.field)) %> + <% row.cell(text: error.error) %> + <% row.cell(text: error.field.humanize) %> + <% end %> + <% end %> + <% end %> + <% end %> + <% end %> +
+
diff --git a/app/components/bulk_upload_error_row_component.rb b/app/components/bulk_upload_error_row_component.rb new file mode 100644 index 000000000..de33fbb3a --- /dev/null +++ b/app/components/bulk_upload_error_row_component.rb @@ -0,0 +1,48 @@ +class BulkUploadErrorRowComponent < ViewComponent::Base + attr_reader :bulk_upload_errors + + def initialize(bulk_upload_errors:) + @bulk_upload_errors = bulk_upload_errors + + super + end + + def row + bulk_upload_errors.first.row + end + + def tenant_code + bulk_upload_errors.first.tenant_code + end + + def purchaser_code + bulk_upload_errors.first.purchaser_code + end + + def property_ref + bulk_upload_errors.first.property_ref + end + + def question_for_field(field) + case bulk_upload.log_type + when "lettings" + BulkUpload::Lettings::Validator.question_for_field(field.to_sym) + when "sales" + BulkUpload::Sales::Validator.question_for_field(field.to_sym) + else + "Unknown question" + end + end + + def bulk_upload + bulk_upload_errors.first.bulk_upload + end + + def lettings? + bulk_upload.log_type == "lettings" + end + + def sales? + bulk_upload.log_type == "sales" + end +end diff --git a/app/controllers/bulk_upload_lettings_results_controller.rb b/app/controllers/bulk_upload_lettings_results_controller.rb new file mode 100644 index 000000000..a0a962b3e --- /dev/null +++ b/app/controllers/bulk_upload_lettings_results_controller.rb @@ -0,0 +1,9 @@ +class BulkUploadLettingsResultsController < ApplicationController + before_action :authenticate_user! + + rescue_from ActiveRecord::RecordNotFound, with: :render_not_found + + def show + @bulk_upload = current_user.bulk_uploads.lettings.find(params[:id]) + end +end diff --git a/app/controllers/bulk_upload_sales_results_controller.rb b/app/controllers/bulk_upload_sales_results_controller.rb new file mode 100644 index 000000000..6af8cb659 --- /dev/null +++ b/app/controllers/bulk_upload_sales_results_controller.rb @@ -0,0 +1,9 @@ +class BulkUploadSalesResultsController < ApplicationController + before_action :authenticate_user! + + rescue_from ActiveRecord::RecordNotFound, with: :render_not_found + + def show + @bulk_upload = current_user.bulk_uploads.sales.find(params[:id]) + end +end diff --git a/app/jobs/process_bulk_upload_job.rb b/app/jobs/process_bulk_upload_job.rb new file mode 100644 index 000000000..59be3ec9e --- /dev/null +++ b/app/jobs/process_bulk_upload_job.rb @@ -0,0 +1,7 @@ +class ProcessBulkUploadJob < ApplicationJob + queue_as :default + + def perform(bulk_upload:) + BulkUpload::Processor.new(bulk_upload:).call + end +end diff --git a/app/models/bulk_upload.rb b/app/models/bulk_upload.rb index 8b007f450..3cab1fe23 100644 --- a/app/models/bulk_upload.rb +++ b/app/models/bulk_upload.rb @@ -2,9 +2,14 @@ class BulkUpload < ApplicationRecord enum log_type: { lettings: "lettings", sales: "sales" } belongs_to :user + has_many :bulk_upload_errors after_initialize :generate_identifier, unless: :identifier + def year_combo + "#{year}/#{year - 2000 + 1}" + end + private def generate_identifier diff --git a/app/models/bulk_upload_error.rb b/app/models/bulk_upload_error.rb new file mode 100644 index 000000000..df584b63e --- /dev/null +++ b/app/models/bulk_upload_error.rb @@ -0,0 +1,3 @@ +class BulkUploadError < ApplicationRecord + belongs_to :bulk_upload +end diff --git a/app/models/forms/bulk_upload_lettings/upload_your_file.rb b/app/models/forms/bulk_upload_lettings/upload_your_file.rb index da2947f53..57ac017a3 100644 --- a/app/models/forms/bulk_upload_lettings/upload_your_file.rb +++ b/app/models/forms/bulk_upload_lettings/upload_your_file.rb @@ -41,9 +41,9 @@ module Forms filename: file.original_filename, ) - if upload_enabled? - storage_service.write_file(bulk_upload.identifier, File.read(file.path)) - end + storage_service.write_file(bulk_upload.identifier, File.read(file.path)) + + ProcessBulkUploadJob.perform_later(bulk_upload:) true end @@ -55,7 +55,14 @@ module Forms end def storage_service - @storage_service ||= Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + @storage_service ||= if upload_enabled? + Storage::S3Service.new( + Configuration::PaasConfigurationService.new, + ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], + ) + else + Storage::LocalDiskService.new + end end def validate_file_is_csv diff --git a/app/models/forms/bulk_upload_sales/upload_your_file.rb b/app/models/forms/bulk_upload_sales/upload_your_file.rb index b8454000b..117e612b1 100644 --- a/app/models/forms/bulk_upload_sales/upload_your_file.rb +++ b/app/models/forms/bulk_upload_sales/upload_your_file.rb @@ -38,21 +38,24 @@ module Forms filename: file.original_filename, ) - if upload_enabled? - storage_service.write_file(bulk_upload.identifier, File.read(file.path)) - end + storage_service.write_file(bulk_upload.identifier, File.read(file.path)) + + ProcessBulkUploadJob.perform_later(bulk_upload:) true end private - def upload_enabled? - !Rails.env.development? - end - def storage_service - @storage_service ||= Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"]) + @storage_service ||= if FeatureToggle.upload_enabled? + Storage::S3Service.new( + Configuration::PaasConfigurationService.new, + ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], + ) + else + Storage::LocalDiskService.new + end end def validate_file_is_csv diff --git a/app/models/user.rb b/app/models/user.rb index 6126e7004..4014abc29 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -12,6 +12,7 @@ class User < ApplicationRecord has_many :owned_sales_logs, through: :organisation has_many :managed_sales_logs, through: :organisation has_many :legacy_users + has_many :bulk_uploads validates :name, presence: true validates :email, presence: true diff --git a/app/services/bulk_upload/downloader.rb b/app/services/bulk_upload/downloader.rb index 9d40ef5b9..8fcd9ccca 100644 --- a/app/services/bulk_upload/downloader.rb +++ b/app/services/bulk_upload/downloader.rb @@ -11,6 +11,10 @@ class BulkUpload::Downloader download end + def delete_local_file! + file.unlink + end + private def download @@ -25,9 +29,21 @@ private end def storage_service - @storage_service ||= Storage::S3Service.new( + @storage_service ||= if FeatureToggle.upload_enabled? + s3_storage_service + else + local_disk_storage_service + end + end + + def s3_storage_service + Storage::S3Service.new( Configuration::PaasConfigurationService.new, ENV["CSV_DOWNLOAD_PAAS_INSTANCE"], ) end + + def local_disk_storage_service + Storage::LocalDiskService.new + end end diff --git a/app/services/bulk_upload/lettings/row_parser.rb b/app/services/bulk_upload/lettings/row_parser.rb new file mode 100644 index 000000000..23d037416 --- /dev/null +++ b/app/services/bulk_upload/lettings/row_parser.rb @@ -0,0 +1,185 @@ +class BulkUpload::Lettings::RowParser + include ActiveModel::Model + include ActiveModel::Attributes + + attribute :field_1, :integer + attribute :field_2 + attribute :field_3 + attribute :field_4, :integer + attribute :field_5, :integer + attribute :field_6 + attribute :field_7, :string + attribute :field_8, :integer + attribute :field_9, :integer + attribute :field_10, :string + attribute :field_11, :integer + attribute :field_12, :string + attribute :field_13, :string + attribute :field_14, :string + attribute :field_15, :string + attribute :field_16, :string + attribute :field_17, :string + attribute :field_18, :string + attribute :field_19, :string + attribute :field_20, :string + attribute :field_21, :string + attribute :field_22, :string + attribute :field_23, :string + attribute :field_24, :string + attribute :field_25, :string + attribute :field_26, :string + attribute :field_27, :string + attribute :field_28, :string + attribute :field_29, :string + attribute :field_30, :string + attribute :field_31, :string + attribute :field_32, :string + attribute :field_33, :string + attribute :field_34, :string + attribute :field_35, :integer + attribute :field_36, :integer + attribute :field_37, :integer + attribute :field_38, :integer + attribute :field_39, :integer + attribute :field_40, :integer + attribute :field_41, :integer + attribute :field_42, :integer + attribute :field_43, :integer + attribute :field_44, :integer + attribute :field_45, :integer + attribute :field_46, :integer + attribute :field_47, :integer + attribute :field_48, :integer + attribute :field_49, :integer + attribute :field_50, :integer + attribute :field_51, :integer + attribute :field_52, :integer + attribute :field_53, :string + attribute :field_54 + attribute :field_55, :integer + attribute :field_56, :integer + attribute :field_57, :integer + attribute :field_58, :integer + attribute :field_59, :integer + attribute :field_60, :integer + attribute :field_61, :integer + attribute :field_62, :string + attribute :field_63, :string + attribute :field_64, :string + attribute :field_65, :integer + attribute :field_66, :integer + attribute :field_67, :integer + attribute :field_68, :integer + attribute :field_69, :integer + attribute :field_70, :integer + attribute :field_71, :integer + attribute :field_72, :integer + attribute :field_73, :integer + attribute :field_74, :integer + attribute :field_75, :integer + attribute :field_76, :integer + attribute :field_77, :integer + attribute :field_78, :integer + attribute :field_79, :integer + attribute :field_80, :decimal + attribute :field_81, :decimal + attribute :field_82, :decimal + attribute :field_83, :decimal + attribute :field_84, :decimal + attribute :field_85, :decimal + attribute :field_86, :integer + attribute :field_87, :integer + attribute :field_88, :decimal + attribute :field_89, :integer + attribute :field_90, :integer + attribute :field_91, :integer + attribute :field_92, :integer + attribute :field_93, :integer + attribute :field_94, :integer + attribute :field_95 + attribute :field_96, :integer + attribute :field_97, :integer + attribute :field_98, :integer + attribute :field_99, :integer + attribute :field_100, :string + attribute :field_101, :integer + attribute :field_102, :integer + attribute :field_103, :integer + attribute :field_104, :integer + attribute :field_105, :integer + attribute :field_106, :integer + attribute :field_107, :string + attribute :field_108, :string + attribute :field_109, :string + attribute :field_110 + attribute :field_111, :integer + attribute :field_112, :string + attribute :field_113, :integer + attribute :field_114, :integer + attribute :field_115 + attribute :field_116, :integer + attribute :field_117, :integer + attribute :field_118, :integer + attribute :field_119, :integer + attribute :field_120, :integer + attribute :field_121, :integer + attribute :field_122, :integer + attribute :field_123, :integer + attribute :field_124, :integer + attribute :field_125, :integer + attribute :field_126, :integer + attribute :field_127, :integer + attribute :field_128, :integer + attribute :field_129, :integer + attribute :field_130, :integer + attribute :field_131, :string + attribute :field_132, :integer + attribute :field_133, :integer + attribute :field_134, :integer + + validates :field_1, presence: true, numericality: { in: (1..12) } + validates :field_4, numericality: { in: (1..999), allow_blank: true } + validates :field_4, presence: true, if: :field_4_presence_check + + validate :validate_possible_answers + +# delegate :valid?, to: :native_object +# delegate :errors, to: :native_object + +private + + def native_object + @native_object ||= LettingsLog.new(attributes_for_log) + end + + def field_mapping + { + field_134: :renewal, + } + end + + def validate_possible_answers + field_mapping.each do |field, attribute| + possible_answers = FormHandler.instance.current_lettings_form.questions.find { |q| q.id == attribute.to_s }.answer_options.keys + + unless possible_answers.include?(public_send(field)) + errors.add(field, "Value supplied is not one of the permitted values") + end + end + end + + def attributes_for_log + hash = field_mapping.invert + attributes = {} + + hash.map do |k, v| + attributes[k] = public_send(v) + end + + attributes + end + + def field_4_presence_check + [1, 3, 5, 7, 9, 11].include?(field_1) + end +end diff --git a/app/services/bulk_upload/lettings/validator.rb b/app/services/bulk_upload/lettings/validator.rb new file mode 100644 index 000000000..d03d59bd9 --- /dev/null +++ b/app/services/bulk_upload/lettings/validator.rb @@ -0,0 +1,250 @@ +require "csv" + +class BulkUpload::Lettings::Validator + include ActiveModel::Validations + + QUESTIONS = { + field_1: "What is the letting type?", + field_2: "This question has been removed", + field_3: "This question has been removed", + field_4: "Management group code", + field_5: "Scheme code", + field_6: "This question has been removed", + field_7: "What is the tenant code?", + field_8: "Is this a starter tenancy?", + field_9: "What is the tenancy type?", + field_10: "If 'Other', what is the tenancy type?", + field_11: "What is the length of the fixed-term tenancy to the nearest year?", + field_12: "Age of Person 1", + field_13: "Age of Person 2", + field_14: "Age of Person 3", + field_15: "Age of Person 4", + field_16: "Age of Person 5", + field_17: "Age of Person 6", + field_18: "Age of Person 7", + field_19: "Age of Person 8", + field_20: "Gender identity of Person 1", + field_21: "Gender identity of Person 2", + field_22: "Gender identity of Person 3", + field_23: "Gender identity of Person 4", + field_24: "Gender identity of Person 5", + field_25: "Gender identity of Person 6", + field_26: "Gender identity of Person 7", + field_27: "Gender identity of Person 8", + field_28: "Relationship to Person 1 for Person 2", + field_29: "Relationship to Person 1 for Person 3", + field_30: "Relationship to Person 1 for Person 4", + field_31: "Relationship to Person 1 for Person 5", + field_32: "Relationship to Person 1 for Person 6", + field_33: "Relationship to Person 1 for Person 7", + field_34: "Relationship to Person 1 for Person 8", + field_35: "Working situation of Person 1", + field_36: "Working situation of Person 2", + field_37: "Working situation of Person 3", + field_38: "Working situation of Person 4", + field_39: "Working situation of Person 5", + field_40: "Working situation of Person 6", + field_41: "Working situation of Person 7", + field_42: "Working situation of Person 8", + field_43: "What is the lead tenant's ethnic group?", + field_44: "What is the lead tenant's nationality?", + field_45: "Does anybody in the household have links to the UK armed forces?", + field_46: "Was the person seriously injured or ill as a result of serving in the UK armed forces?", + field_47: "Is anybody in the household pregnant?", + field_48: "Is the tenant likely to be receiving benefits related to housing?", + field_49: "How much of the household's income is from Universal Credit, state pensions or benefits?", + field_50: "How much income does the household have in total?", + field_51: "Do you know the household's income?", + field_52: "What is the tenant's main reason for the household leaving their last settled home?", + field_53: "If 'Other', what was the main reason for leaving their last settled home?", + field_54: "This question has been removed", + field_55: "Does anybody in the household have any disabled access needs?", + field_56: "Does anybody in the household have any disabled access needs?", + field_57: "Does anybody in the household have any disabled access needs?", + field_58: "Does anybody in the household have any disabled access needs?", + field_59: "Does anybody in the household have any disabled access needs?", + field_60: "Does anybody in the household have any disabled access needs?", + field_61: "Where was the household immediately before this letting?", + field_62: "What is the local authority of the household's last settled home?", + field_63: "Part 1 of postcode of last settled home", + field_64: "Part 2 of postcode of last settled home", + field_65: "Do you know the postcode of last settled home?", + field_66: "How long has the household continuously lived in the local authority area of the new letting?", + field_67: "How long has the household been on the waiting list for the new letting?", + field_68: "Was the tenant homeless directly before this tenancy?", + field_69: "Was the household given 'reasonable preference' by the local authority?", + field_70: "Reasonable preference. They were homeless or about to lose their home (within 56 days)", + field_71: "Reasonable preference. They were living in insanitary, overcrowded or unsatisfactory housing", + field_72: "Reasonable preference. They needed to move on medical and welfare grounds (including a disability)", + field_73: "Reasonable preference. They needed to move to avoid hardship to themselves or others", + field_74: "Reasonable preference. Don't know", + field_75: "Was the letting made under any of the following allocations systems?", + field_76: "Was the letting made under any of the following allocations systems?", + field_77: "Was the letting made under any of the following allocations systems?", + field_78: "What was the source of referral for this letting?", + field_79: "How often does the household pay rent and other charges?", + field_80: "What is the basic rent?", + field_81: "What is the service charge?", + field_82: "What is the personal service charge?", + field_83: "What is the support charge?", + field_84: "Total Charge", + field_85: "If this is a care home, how much does the household pay every [time period]?", + field_86: "Does the household pay rent or other charges for the accommodation?", + field_87: "After the household has received any housing-related benefits, will they still need to pay basic rent and other charges?", + field_88: "What do you expect the outstanding amount to be?", + field_89: "What is the void or renewal date?", + field_90: "What is the void or renewal date?", + field_91: "What is the void or renewal date?", + field_92: "What date were major repairs completed on?", + field_93: "What date were major repairs completed on?", + field_94: "What date were major repairs completed on?", + field_95: "This question has been removed", + field_96: "What date did the tenancy start?", + field_97: "What date did the tenancy start?", + field_98: "What date did the tenancy start?", + field_99: "Since becoming available, how many times has the property been previously offered?", + field_100: "What is the property reference?", + field_101: "How many bedrooms does the property have?", + field_102: "What type of unit is the property?", + field_103: "Which type of building is the property?", + field_104: "Is the property built or adapted to wheelchair-user standards?", + field_105: "What type was the property most recently let as?", + field_106: "What is the reason for the property being vacant?", + field_107: "What is the local authority of the property?", + field_108: "Part 1 of postcode of the property", + field_109: "Part 2 of postcode of the property", + field_110: "This question has been removed", + field_111: "Which organisation owns this property?", + field_112: "Username field", + field_113: "Which organisation manages this property?", + field_114: "Is the person still serving in the UK armed forces?", + field_115: "This question has been removed", + field_116: "How often does the household receive income?", + field_117: "Is this letting sheltered accommodation?", + field_118: "Does anybody in the household have a physical or mental health condition (or other illness) expected to last for 12 months or more?", + field_119: "Vision, for example blindness or partial sight", + field_120: "Hearing, for example deafness or partial hearing", + field_121: "Mobility, for example walking short distances or climbing stairs", + field_122: "Dexterity, for example lifting and carrying objects, using a keyboard", + field_123: "Learning or understanding or concentrating", + field_124: "Memory", + field_125: "Mental health", + field_126: "Stamina or breathing or fatigue", + field_127: "Socially or behaviourally, for example associated with autism spectral disorder (ASD) which includes Aspergers' or attention deficit hyperactivity disorder (ADHD)", + field_128: "Other", + field_129: "Is this letting a London Affordable Rent letting?", + field_130: "Which type of Intermediate Rent is this letting?", + field_131: "Which 'Other' type of Intermediate Rent is this letting?", + field_132: "Data Protection", + field_133: "Is this a joint tenancy?", + field_134: "Is this letting a renewal?", + }.freeze + + attr_reader :bulk_upload, :path + + validate :validate_file_not_empty + validate :validate_max_columns + + def initialize(bulk_upload:, path:) + @bulk_upload = bulk_upload + @path = path + end + + def call + row_offset = 6 + col_offset = 0 + + row_parsers.each_with_index do |row_parser, index| + row_parser.valid? + + row = index + row_offset + + row_parser.errors.each do |error| + bulk_upload.bulk_upload_errors.create!( + field: error.attribute, + error: error.type, + tenant_code: row_parser.field_7, + property_ref: row_parser.field_100, + row:, + cell: "#{cols[field_number_for_attribute(error.attribute) + col_offset]}#{row}", + ) + end + end + end + + def self.question_for_field(field) + QUESTIONS[field] + end + +private + + def field_number_for_attribute(attribute) + attribute.to_s.split("_").last.to_i + end + + def cols + @cols ||= ("A".."EE").to_a + end + + def row_parsers + @row_parsers ||= body_rows.map do |row| + stripped_row = row[1..] + headers = ("field_1".."field_134").to_a + hash = Hash[headers.zip(stripped_row)] + + BulkUpload::Lettings::RowParser.new(hash) + end + end + + # determine the row seperator from CSV + # Windows will use \r\n + def row_sep + contents = "" + + File.open(path, "r") do |f| + f.seek(9900) + contents = f.read + end + + rn_count = contents.scan("\r\n").count + n_count = contents.scan(/[^\r]\n/).count + + if rn_count > n_count + "\r\n" + else + "\n" + end + end + + def rows + @rows ||= CSV.read(path, row_sep:) + end + + def body_rows + rows[6..] + end + + def validate_file_not_empty + if File.size(path).zero? + errors.add(:file, :blank) + + halt_validations! + end + end + + def validate_max_columns + return if halt_validations? + + max_row_size = rows.map(&:size).max + + errors.add(:file, :max_row_size) if max_row_size > 136 + end + + def halt_validations! + @halt_validations = true + end + + def halt_validations? + @halt_validations ||= false + end +end diff --git a/app/services/bulk_upload/processor.rb b/app/services/bulk_upload/processor.rb new file mode 100644 index 000000000..93d3fd3e7 --- /dev/null +++ b/app/services/bulk_upload/processor.rb @@ -0,0 +1,42 @@ +class BulkUpload::Processor + attr_reader :bulk_upload + + def initialize(bulk_upload:) + @bulk_upload = bulk_upload + end + + def call + download + validator.call + ensure + downloader.delete_local_file! + end + +private + + def downloader + @downloader ||= BulkUpload::Downloader.new(bulk_upload:) + end + + def download + downloader.call + end + + def validator + @validator ||= validator_class.new( + bulk_upload:, + path: downloader.path, + ) + end + + def validator_class + case bulk_upload.log_type + when "lettings" + BulkUpload::Lettings::Validator + when "sales" + BulkUpload::Sales::Validator + else + raise "Validator not found for #{bulk_upload.log_type}" + end + end +end diff --git a/app/services/bulk_upload/sales/row_parser.rb b/app/services/bulk_upload/sales/row_parser.rb new file mode 100644 index 000000000..f7b74dfe1 --- /dev/null +++ b/app/services/bulk_upload/sales/row_parser.rb @@ -0,0 +1,176 @@ +class BulkUpload::Sales::RowParser + include ActiveModel::Model + include ActiveModel::Attributes + + attribute :field_1, :string + attribute :field_2, :integer + attribute :field_3, :integer + attribute :field_4, :integer + attribute :field_5 + attribute :field_6, :integer + attribute :field_7, :integer + attribute :field_8, :integer + attribute :field_9, :integer + attribute :field_10, :integer + attribute :field_11, :integer + attribute :field_12, :integer + attribute :field_13, :string + attribute :field_14, :string + attribute :field_15, :string + attribute :field_16, :string + attribute :field_17, :string + attribute :field_18, :string + attribute :field_19, :string + attribute :field_20, :integer + attribute :field_21, :integer + attribute :field_22, :integer + attribute :field_23, :integer + attribute :field_24, :integer + attribute :field_25, :integer + attribute :field_26, :integer + attribute :field_27, :integer + attribute :field_28, :integer + attribute :field_29, :integer + attribute :field_30, :integer + attribute :field_31, :integer + attribute :field_32, :integer + attribute :field_33, :integer + attribute :field_34, :integer + attribute :field_35, :integer + attribute :field_36, :integer + attribute :field_37, :integer + attribute :field_38 + attribute :field_39, :integer + attribute :field_40, :string + attribute :field_41, :string + attribute :field_42, :string + attribute :field_43, :integer + attribute :field_44, :integer + attribute :field_45, :integer + attribute :field_46, :integer + attribute :field_47, :integer + attribute :field_48, :integer + attribute :field_49, :integer + attribute :field_50, :integer + attribute :field_51, :integer + attribute :field_52, :integer + attribute :field_53, :string + attribute :field_54, :string + attribute :field_55, :string + attribute :field_56, :integer + attribute :field_57, :integer + attribute :field_58, :integer + attribute :field_59, :integer + attribute :field_60, :integer + attribute :field_61, :integer + attribute :field_62, :integer + attribute :field_63, :integer + attribute :field_64, :integer + attribute :field_65, :integer + attribute :field_66, :integer + attribute :field_67, :integer + attribute :field_68, :integer + attribute :field_69, :integer + attribute :field_70, :integer + attribute :field_71, :integer + attribute :field_72, :integer + attribute :field_73, :integer + attribute :field_74, :decimal + attribute :field_75, :decimal + attribute :field_76, :integer + attribute :field_77, :integer + attribute :field_78, :integer + attribute :field_79, :integer + attribute :field_80, :integer + attribute :field_81, :integer + attribute :field_82, :integer + attribute :field_83, :integer + attribute :field_84, :integer + attribute :field_85, :string + attribute :field_86 + attribute :field_87, :integer + attribute :field_88, :integer + attribute :field_89, :integer + attribute :field_90, :integer + attribute :field_91, :integer + attribute :field_92, :integer + attribute :field_93, :string + attribute :field_94 + attribute :field_95, :integer + attribute :field_96 + attribute :field_97, :integer + attribute :field_98, :integer + attribute :field_99, :string + attribute :field_100, :integer + attribute :field_101, :string + attribute :field_102, :integer + attribute :field_103, :string + attribute :field_104, :integer + attribute :field_105, :integer + attribute :field_106, :integer + attribute :field_107, :integer + attribute :field_108, :integer + attribute :field_109, :integer + attribute :field_110, :integer + attribute :field_111, :integer + attribute :field_112, :integer + attribute :field_113, :integer + attribute :field_114, :integer + attribute :field_115, :integer + attribute :field_116, :integer + attribute :field_117, :integer + attribute :field_118, :integer + attribute :field_119, :integer + attribute :field_120, :integer + attribute :field_121, :integer + attribute :field_122, :integer + attribute :field_123, :integer + attribute :field_124, :integer + attribute :field_125, :integer + + # validates :field_1, presence: true, numericality: { in: (1..12) } + # validates :field_4, numericality: { in: (1..999), allow_blank: true } + # validates :field_4, presence: true, if: :field_4_presence_check + + validate :validate_possible_answers + +# delegate :valid?, to: :native_object +# delegate :errors, to: :native_object + +private + + def native_object + @native_object ||= SalesLog.new(attributes_for_log) + end + + def field_mapping + { + field_117: :buy1livein, + } + end + + def validate_possible_answers + field_mapping.each do |field, attribute| + possible_answers = FormHandler.instance.current_sales_form.questions.find { |q| q.id == attribute.to_s }.answer_options.keys + + unless possible_answers.include?(public_send(field)) + errors.add(field, "Value supplied is not one of the permitted values") + end + end + end + + def attributes_for_log + hash = field_mapping.invert + attributes = {} + + hash.map do |k, v| + attributes[k] = public_send(v) + end + + attributes + end + + # def field_4_presence_check + # [1, 3, 5, 7, 9, 11].include?(field_1) + # end +end diff --git a/app/services/bulk_upload/sales/validator.rb b/app/services/bulk_upload/sales/validator.rb new file mode 100644 index 000000000..43b8c001d --- /dev/null +++ b/app/services/bulk_upload/sales/validator.rb @@ -0,0 +1,228 @@ +class BulkUpload::Sales::Validator + include ActiveModel::Validations + + QUESTIONS = { + field_1: "What is the purchaser code?", + field_2: "What is the day of the sale completion date? - DD", + field_3: "What is the month of the sale completion date? - MM", + field_4: "What is the year of the sale completion date? - YY", + field_5: "This question has been removed", + field_6: "Was the buyer interviewed for any of the answers you will provide on this log?", + field_7: "Age of Buyer 1", + field_8: "Age of Person 2", + field_9: "Age of Person 3", + field_10: "Age of Person 4", + field_11: "Age of Person 5", + field_12: "Age of Person 6", + field_13: "Gender identity of Buyer 1", + field_14: "Gender identity of Person 2", + field_15: "Gender identity of Person 3", + field_16: "Gender identity of Person 4", + field_17: "Gender identity of Person 5", + field_18: "Gender identity of Person 6", + field_19: "Relationship to Buyer 1 for Person 2", + field_20: "Relationship to Buyer 1 for Person 3", + field_21: "Relationship to Buyer 1 for Person 4", + field_22: "Relationship to Buyer 1 for Person 5", + field_23: "Relationship to Buyer 1 for Person 6", + field_24: "Working situation of Buyer 1", + field_25: "Working situation of Person 2", + field_26: "Working situation of Person 3", + field_27: "Working situation of Person 4", + field_28: "Working situation of Person 5", + field_29: "Working situation of Person 6", + field_30: "What is buyer 1's ethnic group?", + field_31: "What is buyer 1's nationality?", + field_32: "What is buyer 1's gross annual income?", + field_33: "What is buyer 2's gross annual income?", + field_34: "Was buyer 1's income used for a mortgage application?", + field_35: "Was buyer 2's income used for a mortgage application?", + field_36: "What is the total amount the buyers had in savings before they paid any deposit for the property?", + field_37: "Have any of the purchasers previously owned a property?", + field_38: "This question has been removed", + field_39: "What was buyer 1's previous tenure?", + field_40: "What is the local authority of buyer 1's last settled home?", + field_41: "Part 1 of postcode of buyer 1's last settled home", + field_42: "Part 2 of postcode of buyer 1's last settled home", + field_43: "Do you know the postcode of buyer 1's last settled home?", + field_44: "Was the buyer registered with their PRP (HA)?", + field_45: "Was the buyer registered with the local authority?", + field_46: "Was the buyer registered with a Help to Buy agent?", + field_47: "Was the buyer registered with another PRP (HA)?", + field_48: "Does anyone in the household consider themselves to have a disability?", + field_49: "Does anyone in the household use a wheelchair?", + field_50: "How many bedrooms does the property have?", + field_51: "What type of unit is the property?", + field_52: "Which type of bulding is the property?", + field_53: "What is the local authority of the property?", + field_54: "Part 1 of postcode of property", + field_55: "Part 2 of postcode of property", + field_56: "Is the property built or adapted to wheelchair user standards?", + field_57: "What is the type of shared ownership sale?", + field_58: "Is this a resale?", + field_59: "What is the day of the practical completion or handover date?", + field_60: "What is the month of the practical completion or handover date?", + field_61: "What is the day of the exchange of contracts date?", + field_62: "What is the day of the practical completion or handover date?", + field_63: "What is the month of the practical completion or handover date?", + field_64: "What is the year of the practical completion or handover date?", + field_65: "Was the household re-housed under a local authority nominations agreement?", + field_66: "How many bedrooms did the buyer's previous property have?", + field_67: "What was the type of the buyer's previous property?", + field_68: "What was the full purchase price?", + field_69: "What was the initial percentage equity stake purchased?", + field_70: "What is the mortgage amount?", + field_71: "Does this include any extra borrowing?", + field_72: "How much was the cash deposit paid on the property?", + field_73: "How much cash discount was given through Social Homebuy?", + field_74: "What is the basic monthly rent?", + field_75: "What are the total monthly leasehold charges for the property?", + field_76: "What is the type of discounted ownership sale?", + field_77: "What was the full purchase price?", + field_78: "What was the amount of any loan, grant, discount or subsidy given?", + field_79: "What was the percentage discount?", + field_80: "What is the mortgage amount?", + field_81: "Does this include any extra borrowing?", + field_82: "How much was the cash deposit paid on the property?", + field_83: "What are the total monthly leasehold charges for the property?", + field_84: "What is the type of outright sale?", + field_85: "If 'other', what is the 'other' type?", + field_86: "This question has been removed", + field_87: "What is the full purchase price?", + field_88: "What is the mortgage amount?", + field_89: "Does this include any extra borrowing?", + field_90: "How much was the cash deposit paid on the property?", + field_91: "What are the total monthly leasehold charges for the property?", + field_92: "Which organisation owned this property before the sale?", + field_93: "Username", + field_94: "This question has been removed", + field_95: "Has the buyer ever served in the UK Armed Forces and for how long?", + field_96: "This question has been removed", + field_97: "Are any of the buyers a spouse or civil partner of a UK Armed Forces regular who died in service within the last 2 years?", + field_98: "What is the name of the mortgage lender? - Shared ownership", + field_99: "If 'other', what is the name of the mortgage lender?", + field_100: "What is the name of the mortgage lender? - Discounted ownership", + field_101: "If 'other', what is the name of the mortgage lender?", + field_102: "What is the name of the mortgage lender? - Outright sale", + field_103: "If 'other', what is the name of the mortgage lender?", + field_104: "Were the buyers receiving any of these housing-related benefits immediately before buying this property?", + field_105: "What is the length of the mortgage in years? - Shared ownership", + field_106: "What is the length of the mortgage in years? - Discounted ownership", + field_107: "What is the length of the mortgage in years? - Outright sale", + field_108: "How long have the buyers been living in the property before the purchase? - Discounted ownership", + field_109: "Are there more than two joint purchasers of this property?", + field_110: "How long have the buyers been living in the property before the purchase? - Shared ownership", + field_111: "Is this a staircasing transaction?", + field_112: "Data Protection question", + field_113: "Was this purchase made through an ownership scheme?", + field_114: "Is the buyer a company?", + field_115: "Will the buyers live in the property?", + field_116: "Is this a joint purchase?", + field_117: "Will buyer 1 live in the property?", + field_118: "Will buyer 2 live in the property?", + field_119: "Besides the buyers, how many people will live in the property?", + field_120: "What percentage of the property has been bought in this staircasing transaction?", + field_121: "What percentage of the property does the buyer now own in total?", + field_122: "What was the rent type of the buyer's previous property?", + field_123: "Was a mortgage used for the purchase of this property? - Shared ownership", + field_124: "Was a mortgage used for the purchase of this property? - Discounted ownership", + field_125: "Was a mortgage used for the purchase of this property? - Outright sale", + }.freeze + + def self.question_for_field(field) + QUESTIONS[field] + end + + attr_reader :bulk_upload, :path + + validate :validate_file_not_empty + validate :validate_max_columns + + def initialize(bulk_upload:, path:) + @bulk_upload = bulk_upload + @path = path + end + + def call + row_parsers.each_with_index do |row_parser, index| + row_parser.valid? + + row = index + row_offset + 1 + + row_parser.errors.each do |error| + bulk_upload.bulk_upload_errors.create!( + field: error.attribute, + error: error.type, + purchaser_code: row_parser.field_1, + row:, + cell: "#{cols[field_number_for_attribute(error.attribute) + col_offset - 1]}#{row}", + ) + end + end + end + +private + + def field_number_for_attribute(attribute) + attribute.to_s.split("_").last.to_i + end + + def rows + @rows ||= CSV.read(path, row_sep:) + end + + def body_rows + rows[row_offset..] + end + + def row_offset + 5 + end + + def col_offset + 1 + end + + def cols + @cols ||= ("A".."DV").to_a + end + + def row_parsers + @row_parsers ||= body_rows.map do |row| + stripped_row = row[col_offset..] + headers = ("field_1".."field_125").to_a + hash = Hash[headers.zip(stripped_row)] + + BulkUpload::Sales::RowParser.new(hash) + end + end + + def row_sep + "\r\n" + # "\n" + end + + def validate_file_not_empty + if File.size(path).zero? + errors.add(:file, :blank) + + halt_validations! + end + end + + def validate_max_columns + return if halt_validations? + + max_row_size = rows.map(&:size).max + + errors.add(:file, :max_row_size) if max_row_size > 126 + end + + def halt_validations! + @halt_validations = true + end + + def halt_validations? + @halt_validations ||= false + end +end diff --git a/app/services/storage/local_disk_service.rb b/app/services/storage/local_disk_service.rb new file mode 100644 index 000000000..f0cc358d1 --- /dev/null +++ b/app/services/storage/local_disk_service.rb @@ -0,0 +1,26 @@ +require "fileutils" + +module Storage + class LocalDiskService < StorageService + def list_files(folder = "/") + path = Rails.root.join("tmp/storage", folder) + Dir.entries(path) + end + + def get_file_io(filename) + path = Rails.root.join("tmp/storage", filename) + + File.open(path, "r") + end + + def write_file(filename, data) + path = Rails.root.join("tmp/storage", filename) + + FileUtils.mkdir_p(path.dirname) + + File.open(path, "w") do |f| + f.write data + end + end + end +end diff --git a/app/views/bulk_upload_lettings_results/show.html.erb b/app/views/bulk_upload_lettings_results/show.html.erb new file mode 100644 index 000000000..452926787 --- /dev/null +++ b/app/views/bulk_upload_lettings_results/show.html.erb @@ -0,0 +1,20 @@ +
+
+ Bulk Upload for lettings (<%= @bulk_upload.year_combo %>) +

We found <%= pluralize(@bulk_upload.bulk_upload_errors.count, "error") %> in your file

+ +
+ Here’s a list of everything that you need to fix your spreadsheet. You can download the specification to help you fix the cells in your CSV file. +
+ +

<%= @bulk_upload.filename %>

+
+
+ +
+
+ <% @bulk_upload.bulk_upload_errors.group_by(&:row).each do |_row, errors_for_row| %> + <%= render BulkUploadErrorRowComponent.new(bulk_upload_errors: errors_for_row) %> + <% end %> +
+
diff --git a/app/views/bulk_upload_sales_results/show.html.erb b/app/views/bulk_upload_sales_results/show.html.erb new file mode 100644 index 000000000..aacc80f9c --- /dev/null +++ b/app/views/bulk_upload_sales_results/show.html.erb @@ -0,0 +1,20 @@ +
+
+ Bulk Upload for sales (<%= @bulk_upload.year_combo %>) +

We found <%= pluralize(@bulk_upload.bulk_upload_errors.count, "error") %> in your file

+ +
+ Here’s a list of everything that you need to fix your spreadsheet. You can download the specification to help you fix the cells in your CSV file. +
+ +

<%= @bulk_upload.filename %>

+
+
+ +
+
+ <% @bulk_upload.bulk_upload_errors.group_by(&:row).each do |_row, errors_for_row| %> + <%= render BulkUploadErrorRowComponent.new(bulk_upload_errors: errors_for_row) %> + <% end %> +
+
diff --git a/config/environments/development.rb b/config/environments/development.rb index f39f4d5be..24155edfa 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -83,4 +83,6 @@ Rails.application.configure do # see https://discuss.rubyonrails.org/t/cve-2022-32224-possible-rce-escalation-bug-with-serialized-columns-in-active-record/81017 config.active_record.yaml_column_permitted_classes = [Time] + + config.active_job.queue_adapter = :inline end diff --git a/config/environments/test.rb b/config/environments/test.rb index 2af4affe7..dca19f3d1 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -65,4 +65,6 @@ Rails.application.configure do # see https://discuss.rubyonrails.org/t/cve-2022-32224-possible-rce-escalation-bug-with-serialized-columns-in-active-record/81017 config.active_record.yaml_column_permitted_classes = [Time] + + config.active_job.queue_adapter = :test end diff --git a/config/routes.rb b/config/routes.rb index eef759e92..4e5b224c1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -123,15 +123,19 @@ Rails.application.routes.draw do collection do post "bulk-upload", to: "bulk_upload#bulk_upload" get "bulk-upload", to: "bulk_upload#show" + get "csv-download", to: "lettings_logs#download_csv" post "email-csv", to: "lettings_logs#email_csv" get "csv-confirmation", to: "lettings_logs#csv_confirmation" - resources :bulk_upload_lettings_logs, path: "bulk-upload-logs" do + resources :bulk_upload_lettings_logs, path: "bulk-upload-logs", only: %i[show update] do collection do get :start end end + + resources :bulk_upload_lettings_results, path: "bulk-upload-results", only: [:show] + get "update-logs", to: "lettings_logs#update_logs" end @@ -158,6 +162,8 @@ Rails.application.routes.draw do get :start end end + + resources :bulk_upload_sales_results, path: "bulk-upload-results", only: [:show] end FormHandler.instance.sales_forms.each do |_key, form| diff --git a/db/migrate/20221209161927_create_bulk_upload_errors.rb b/db/migrate/20221209161927_create_bulk_upload_errors.rb new file mode 100644 index 000000000..ae72298e9 --- /dev/null +++ b/db/migrate/20221209161927_create_bulk_upload_errors.rb @@ -0,0 +1,19 @@ +class CreateBulkUploadErrors < ActiveRecord::Migration[7.0] + def change + create_table :bulk_upload_errors do |t| + t.references :bulk_upload + + t.text :cell + t.text :row + + t.text :tenant_code + t.text :property_ref + t.text :purchase_code + + t.text :field + t.text :error + + t.timestamps + end + end +end diff --git a/db/migrate/20230103100531_rename_purchaser_code.rb b/db/migrate/20230103100531_rename_purchaser_code.rb new file mode 100644 index 000000000..d6da4b67c --- /dev/null +++ b/db/migrate/20230103100531_rename_purchaser_code.rb @@ -0,0 +1,5 @@ +class RenamePurchaserCode < ActiveRecord::Migration[7.0] + def change + rename_column :bulk_upload_errors, :purchase_code, :purchaser_code + end +end diff --git a/db/schema.rb b/db/schema.rb index b0276b3fb..9d821379a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -14,6 +14,20 @@ ActiveRecord::Schema[7.0].define(version: 2023_01_09_160738) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "bulk_upload_errors", force: :cascade do |t| + t.bigint "bulk_upload_id" + t.text "cell" + t.text "row" + t.text "tenant_code" + t.text "property_ref" + t.text "purchaser_code" + t.text "field" + t.text "error" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["bulk_upload_id"], name: "index_bulk_upload_errors_on_bulk_upload_id" + end + create_table "bulk_uploads", force: :cascade do |t| t.bigint "user_id" t.text "log_type", null: false diff --git a/spec/components/bulk_upload_error_row_component_spec.rb b/spec/components/bulk_upload_error_row_component_spec.rb new file mode 100644 index 000000000..86222acf4 --- /dev/null +++ b/spec/components/bulk_upload_error_row_component_spec.rb @@ -0,0 +1,75 @@ +require "rails_helper" + +RSpec.describe BulkUploadErrorRowComponent, type: :component do + context "when a single error" do + let(:row) { rand(9_999) } + let(:tenant_code) { SecureRandom.hex(4) } + let(:property_ref) { SecureRandom.hex(4) } + let(:field) { :field_134 } + let(:error) { "some error" } + let(:bulk_upload) { create(:bulk_upload, :lettings) } + let(:bulk_upload_errors) do + [ + FactoryBot.build( + :bulk_upload_error, + bulk_upload:, + row:, + tenant_code:, + property_ref:, + field:, + error:, + ), + ] + end + + it "renders the row number" do + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content("Row #{row}") + end + + it "renders the tenant_code" do + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content("Tenant code: #{tenant_code}") + end + + it "renders the property_ref" do + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content("Property reference: #{property_ref}") + end + + it "renders the cell of error" do + expected = bulk_upload_errors.first.cell + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content(expected) + end + + it "renders the question for lettings" do + expected = "Is this letting a renewal?" + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content(expected) + end + + context "when a sales bulk upload" do + let(:bulk_upload) { create(:bulk_upload, :sales) } + let(:field) { :field_87 } + + it "renders the question for sales" do + expected = "What is the full purchase price?" + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content(expected) + end + end + + it "renders the error" do + expected = error + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content(expected) + end + + it "renders the field number" do + expected = bulk_upload_errors.first.field.humanize + result = render_inline(described_class.new(bulk_upload_errors:)) + expect(result).to have_content(expected) + end + end +end diff --git a/spec/factories/bulk_upload.rb b/spec/factories/bulk_upload.rb index 26b80c172..437f977d9 100644 --- a/spec/factories/bulk_upload.rb +++ b/spec/factories/bulk_upload.rb @@ -6,5 +6,14 @@ FactoryBot.define do log_type { BulkUpload.log_types.values.sample } year { 2022 } identifier { SecureRandom.uuid } + sequence(:filename) { |n| "bulk-upload-#{n}.csv" } + + trait(:sales) do + log_type { BulkUpload.log_types[:sales] } + end + + trait(:lettings) do + log_type { BulkUpload.log_types[:lettings] } + end end end diff --git a/spec/factories/bulk_upload_error.rb b/spec/factories/bulk_upload_error.rb new file mode 100644 index 000000000..1f42f763d --- /dev/null +++ b/spec/factories/bulk_upload_error.rb @@ -0,0 +1,14 @@ +require "securerandom" + +FactoryBot.define do + factory :bulk_upload_error do + bulk_upload + row { rand(9_999) } + cell { "#{('A'..'Z').to_a.sample}#{row}" } + tenant_code { SecureRandom.hex(4) } + property_ref { SecureRandom.hex(4) } + purchaser_code { SecureRandom.hex(4) } + field { "field_#{rand(134)}" } + error { "some error" } + end +end diff --git a/spec/fixtures/files/2021_22_lettings_bulk_upload.csv b/spec/fixtures/files/2021_22_lettings_bulk_upload.csv new file mode 100644 index 000000000..f1f947596 --- /dev/null +++ b/spec/fixtures/files/2021_22_lettings_bulk_upload.csv @@ -0,0 +1,20 @@ +Question,Type of letting,Who is the landlord?,Registration no/LA CORE code,Management group,Scheme code,First letting?,Tenant code,"Starter/ +Introductory Tenancy?",Type of tenancy,"If you responded 'Other' in Q2b, please state tenancy type",Tenancy Duration,Age of Person 1,Age of Person 2,Age of Person 3,Age of Person 4,Age of Person 5,Age of Person 6,Age of Person 7,Age of Person 8,Gender of Person 1,Gender of Person 2,Gender of Person 3,Gender of Person 4,Gender of Person 5,Gender of Person 6,Gender of Person 7,Gender of Person 8,Relationship (of Person 2) to Person 1,Relationship (of Person 3) to Person 1,Relationship (of Person 4) to Person 1,Relationship (of Person 5) to Person 1,Relationship (of Person 6) to Person 1,Relationship (of Person 7) to Person 1,Relationship (of Person 8) to Person 1,Economic Status of Person 1,Economic Status of Person 2,Economic Status of Person 3,Economic Status of Person 4,Economic Status of Person 5,Economic Status of Person 6,Economic Status of Person 7,Economic Status of Person 8,Ethnic group of person 1 as defined by applicant,Nationality of person 1 as defined by applicant,Is anyone in the household…?,Has anyone in the household been seriously injured or ill as a direct result of their time and activities serving as a regular or a reserve?,Does the household contain a pregnant person?,Is the tenant in receipt of or likely to be in receipt of the following:,"How much of your income comes from universal credit, state pensions or benefits (excluding child and housing benefit, UC housing element, and council tax support or tax credit)?","Tenant’s or tenant and partner’s net income (after tax deductions). For those receiving Universal Credit, enter net weekly income from employment, pensions and Universal Credit. Exclude child benefit, housing element of universal credit and council tax support. For those not receiving Universal Credit, please enter net weekly income from employment, pensions and other benefits. Exclude housing benefit, child benefit and council tax support.",Income refused,In the tenant's view what was the main reason the household left their last settled home?,"If you responded 'Other' in 9a, please state main reason",Was the reason for leaving a direct result of the removal of the spare room subsidy or benefit cap introduced from 2013?,a) Fully wheelchair accessible,b) Wheelchair access to essential rooms,c) Requires level access housing,f) Other disability,g) No disability,H) Don't know,The housing situation for this household immediately before this letting,Enter LA in which household lived immediately before this letting,Part 1 of postcode of previous accommodation,Part 2 of postcode of previous accommodation,If postcode of previous accommodation is unknown or if the previous accommodation was temporary,How long has the household continuously lived in the local authority area where the new letting is located?,How long has the household been on the waiting list of the local authority district where the new letting is located?,"Immediately prior to this letting, was this household...?",Was the household given Reasonable Preference (i.e. priority) for housing by the local authority?,Homeless or about to lose their home (within 56 days),"Living in insanitary, overcrowded or unsatisfactory housing",A need to move on medical and welfare grounds (including a disability),A need to move to avoid hardship to themselves or others,Don't know,Was the letting made under any of the following allocations systems? (CBL),Was the letting made under any of the following allocations systems? (CHR),Was the letting made under any of the following allocations systems? (CAP),Source of referral for this letting?,Rent and other charges period,Basic rent,Service charge,Personal Service Charge,Support charge,Total charge,total charge for care homes,Please tick if there is neither rent nor charge to the occupant for the accommodation,"After housing benefit and/or housing element of Universal Credit payment is received, will there be an outstanding amount for basic rent (18i) and/or benefit eligible charges (18ii)?","After housing benefit and/or other housing support payments are received, will there be an outstanding amount for basic rent and/or benefit eligible charges?",void date,void date,void date,Major repairs completion date,Major repairs completion date,Major repairs completion date,if the unit is in a supported scheme for stays of one month or less….,tenancy start date,tenancy start date,tenancy start date,"How many times has the unit been previously offered since becoming available for relet since the last tenancy ended or as a first let? For an Affordable Rent or Intermediate Rent Letting, only include number of offers as that type. (For a property let at the firsty attempt enter '0').",RP property reference (if applicable),Number of bedrooms,Type of unit,Type of building,Is property built or adapted to wheelchair users standards,"If this is a relet, was the property most recently let on",Reason for vacancy,Enter LA of property,Part 1 of postcode of property,Part 2 of postcode of property,If previous postcode and new postcode are the same,Managed (owning) organisation CORE ID,Username Field,Managing (Data providing) Organisation CORE ID ,"If they've ever served as a regular, did they leave…",Enter the Unique Property Reference Number if known,Is the income…?,Is this letting sheltered accommodation?,Does anyone in the household have any physical or mental health conditions or illness lasting or expected to last for 12 months or more?,Vision (e.g. blindness or partial sight),Hearing (e.g. deatness or partial hearing),Mobility (e.g. walking short distances or climbing stairs),"Dexterity (e.g. lifting and carrying objects, using a keyboard)",Learning or understanding or concentrating,Memory,Mental health,Stamina or breathing or fatigue,"Socially or behaviourally (e.g. associated with autism spectral disorder (ASD) which includes Aspergers', or attention deficit hyperactivity disorder (ADHD))",Other,Is this letting a London Affordable Rent letting?,Is this letting…?,"If you responded 'Other' in 0bi, please state product",Data Protection question,,, +Values,1 - 12,1 - 2,6 or 7 digits,1 - 999,,1 - 2,max 13 digits,1 - 2,1 - 5,Text ,1 - 99,15 - 120 or R,,,,,,,,"F, M, X or R",,,,,,,,"P,C,X or R",,,,,,,0 - 10,,,,,,,,1 - 19,1 - 17,1 - 5,1 - 3,,"1, 3 or 6 - 9",1 - 4,0 - 9999,1 or Null,"1 - 2, 4, 7 - 14, 16 - 20, 28 - 31 or 34 - 46",Text,2 - 6,1 or null ,,,,,,"3- 4, 6 - 10, 13, 14, 18, 19, 21 or 23 - 33",ONS CODE - 4 or 9 Digits,XXX(X),XXX,1 or null,"1, 2 or 5 - 10",,"1, 7 or 11",1 - 3,1 or null,,,,,1 or 2,,,"1 - 16 (not 5, 6 or 11)",1 - 10,xxxx.xx,,,,,,1 or null,1 - 3,0 - 9999,1 - 31,1 - 12,10 - 20,1 - 31,1 - 12,13 - 20,1 or null,1 - 31,1 - 12,19 or 20,0+,12 Digits,1 - 7,"1, 2, 4 or 6 - 10",1 or 2,,1 - 4,"5, 6 or 8 - 19",ONS CODE 9 digits,,,1 or null,up to 7,username (name for signing into CORE) of person this log should be assigned to.,up to 7,3 - 6,"Numeric, up to 12 digits",1 - 3,1 - 4,1 - 3,1 or null,,,,,,,,,,1 - 3,,Text,1,,, +Can Be Null?,No,"only if 1 = 2, 4, 6, 8, 10 or 12","only if field 1 = 2, 4, 6, 8, 10 or 12 or field 2 = 1","only if field 1 = 1, 3, 5, 7, 9 or 11",,"only if 1 = 1, 3, 5, 7, 9 or 11",No,,,"Yes, if 9 is not 3","only if 9 = 1, 2, 3 or 5",No,"Yes, if field 21, 28 and 36 are null","Yes, if 22, 29 and 37 are null","Yes, if 23, 30 and 38 are null","Yes, if 24, 31 and 39 are null","Yes, if 25, 32 and 40 are null","Yes, if 26, 33 and 41 are null","Yes, if 27, 34 and 42 are null", No,"Yes, if 13, 28 and 36 are null","Yes, if 14, 29 and 37 are null","Yes, if 15, 30 and 38 are null","Yes, if 16, 31 and 39 are null","Yes, if 17, 32 and 40 are null","Yes, if 18, 33 and 41 are null","Yes, if 19, 34 and 42 are null","Yes, if 13, 21 and 36 are null","Yes, if 14, 22 and 37 are null","Yes, if 15, 23 and 38 are null","Yes, if 16, 24 and 39 are null","Yes, if 17, 25 and 40 are null","Yes, if 18, 26 and 41 are null","Yes, if 19, 27 and 42 are null",No,"Yes, if 13, 21 and 28 are null","Yes, if 14, 22 and 29 are null","Yes, if 15, 23 and 30 are null","Yes, if 16, 24 and 31 are null","Yes, if 17, 25 and 32 are null","Yes, if 18, 26 and 33 are null","Yes, if 19, 27 and 34 are null",No,,,"Yes, must be null if 45 = 2 or 3; +no, if 45 = 1, 4 or 5",No,,,If 51 = 1,Yes,No,"Yes, if 52 is not 20",No,"Selections are ((A or B or C)) - ((A, or B or C and F)) - ((F)) - ((G)) - ((H))",,,,,,No,,"Yes, if 65 = 1",,"Yes, if 63 and 64 contain full and valid entries",No,,,," If 69 = 1, select at least one of the 5 categories; +If 69 = 2 or 3, then Null.",,,,,No,,,,,if 85 or 86 = 1,Yes,,,if 85 or 86 = 1,Only if fields 80 - 84 and 86 are not null,Only if fields 80 - 85 are not null,"Only if field 48 = 3, 6, 7 or 8 ","If 87 = 2 or 3; +if 87 = 1, then a value must be entered",No,,,Yes,,,,No,,,,"Only if 1 = 2, 4, 6, 8, 10 or 12",,,,No,"Only if 1 = 2, 4, 6, 8, 10 or 12; +or 106 = 15 - 17",No,"Only if 1 = 2, 4, 6, 8, 10 or 12",,,Yes,No,,Yes: DCLG Admin only,"Only if 45 = 2, 3 or 4",Yes,"Yes, if 51 = 1","Only if 1 = 1, 3, 5, 7, 9 or 11",No,Yes,,,,,,,,,,Only if 1 = 1 - 4 or 9 - 12.,Only if 1 = 1 - 8.,,No,,, +eCORE Format and Duplicate check,Incorrect GN/SH combination check fields,,,,,Question Removed from 2020/21,,,,,,Duplicate check field,,,,,,,,Duplicate check field,,,,,,,,,,,,,,,Duplicate check field,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Duplicate check field,,,,,,,,,,,,Duplicate check fields,,,,Duplicate check field,General Needs lettings only,,,All lettings,General Needs lettings only,All lettings,General Needs lettings only,,,Question Removed from 2020/21,Duplicate check field, “Username does not exist”. ,,,Question removed from 21/22 onwards,,Supported Housing lettings only.,,,,,,,,,,,,Affordable Rent Lettings only,Intermediate Rent Lettings only,,,,, +Question Number,1a,1b,,1c,,1d,1b,2a,2b,2ba,2c,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,4ai,4b,5,6,7,8,,9a,9aa,9b,10,,,,,,11,12a,12b,,,12c,12d,13,14a,14b,,,,,15,,,16,17, 18ai, 18aii, 18aiii, 18aiv, 18av, 18b, 18c,18d,,19,,,,,,,1,,,20,21a,22,23,24,25,26,27,28,,,28,,,,4aii,21b,8a,1e,10ia,10ib,10ib,10ib,10ib,10ib,10ib,10ib,10ib,10ib,10ib,0a,0bi,0bii,,,, +Field Number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,,, +,2,,,1,1,,4626,2,5,,,47,,,,,,,,M,,,,,,,,,,,,,,,4,,,,,,,,13,1,2,,3,1,4,,1,34,,3,,,,,,1,33,E09000028,,,1,6,6,7,2,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,10,8,20,,,,,17,9,21,0,439JOE005A,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE005A,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4173,2,5,,,R,,,,,,,,R,,,,,,,,,,,,,,,0,,,,,,,,17,13,3,,3,3,4,,1,28,,3,,,,,,1,25,E09000030,,,1,6,6,1,3,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,2,3,20,,,,,22,4,21,0,439JOE001D,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE001D,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4196,2,5,,,46,,,,,,,,F,,,,,,,,,,,,,,,6,,,,,,,,7,1,2,,3,3,4,,1,28,,3,,,,,,1,28,E09000028,,,1,6,6,7,3,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,14,4,20,,,,,29,4,21,0,439JOE006F,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE006F,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4198,2,5,,,41,,,,,,,,M,,,,,,,,,,,,,,,8,,,,,,,,1,1,3,,3,3,4,,1,28,,3,,,,,,1,28,E09000028,,,1,6,6,7,3,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,26,3,20,,,,,30,4,21,0,439JOE001A,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE001A,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4220,2,5,,,R,,,,,,,,R,,,,,,,,,,,,,,,0,,,,,,,,17,13,3,,3,3,4,,1,28,,3,,,,,,1,25,E09000030,,,1,6,6,1,3,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,13,4,20,,,,,8,5,21,0,439JOE008A,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE008A,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4285,2,5,,,R,,,,,,,,R,,,,,,,,,,,,,,,0,,,,,,,,17,13,3,,3,3,4,,1,28,,3,,,,,,1,25,E09000030,,,1,6,6,1,3,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,7,5,20,,,,,1,6,21,0,439JOE006B,,,,2,,8,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE006B,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4189,2,5,,,27,,,,,,,,F,,,,,,,,,,,,,,,4,,,,,,,,12,1,2,,3,1,4,,1,18,,3,,,,,,1,18,E09000028,,,1,6,6,7,2,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,17,4,20,,,,,26,4,21,0,439JOE008F,,,,2,,11,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE008F,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4533,2,5,,,30,,,,,,,,F,,,,,,,,,,,,,,,4,,,,,,,,1,1,3,,3,1,4,,1,7,,3,,,,,,1,7,E09000028,,,1,6,6,7,2,,,,,,2,2,2,4,1,90,141,5,43,279,,,3,,22,7,20,,,,,14,8,21,0,439KOE007F,,,,2,,6,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439KOE007F,1,1,1,,,,,,,,,,,,,,,,, +,2,,,1,1,,4366,2,5,,,34,,,,,,,,M,,,,,,,,,,,,,,,4,,,,,,,,3,7,2,,3,1,4,,1,18,,3,,,,,,1,18,E09000028,,,1,6,6,7,1,,,,,1,2,2,2,4,1,90,141,5,43,279,,,3,,30,5,20,,,,,22,6,21,0,439JOE001C,,,,2,,12,,,,,107242,mt.data.gary.meyler@communities.gov.uk,107242,,439JOE001C,1,1,1,,,,,,,,,,,,,,,,, diff --git a/spec/fixtures/files/2022_23_sales_bulk_upload.csv b/spec/fixtures/files/2022_23_sales_bulk_upload.csv new file mode 100644 index 000000000..b7ae570f7 --- /dev/null +++ b/spec/fixtures/files/2022_23_sales_bulk_upload.csv @@ -0,0 +1,119 @@ +Question,What is the purchaser code?,What is the day of the sale completion date? - DD,What is the month of the sale completion date? - MM,What is the year of the sale completion date? - YY,[BLANK],Was the buyer interviewed for any of the answers you will provide on this log?,Age of Buyer 1,Age of Buyer 2 or Person 2,Age of Person 3,Age of Person 4,Age of Person 5,Age of Person 6,Gender identity of Buyer 1,Gender identity of Buyer 2 or Person 2,Gender identity of Person 3,Gender identity of Person 4,Gender identity of Person 5,Gender identity of Person 6,Person 2's relationship to lead tenant,Person 3's relationship to lead tenant,Person 4's relationship to lead tenant,Person 5's relationship to lead tenant,Person 6's relationship to lead tenant,Working situation of Buyer 1,Working situation of Buyer 2 or Person 2,Working situation of Person 3,Working situation of Person 4,Working situation of Person 5,Working situation of Person 6,What is the buyer 1's ethnic group?,What is buyer 1's nationality?,What is buyer 1's gross annual income?,What is buyer 2's gross annual income?,Was buyer 1's income used for a mortgage application?,Was buyer 2's income used for a mortgage application?,"What is the total amount the buyers had in savings before they paid any deposit for the property? + +To the nearest £10",Have any of the buyers previously owned a property?,[BLANK],What was buyer 1's previous tenure?,What is the local authority of buyer 1's last settled home,Part 1 of postcode of buyer 1's last settled home,Part 2 of postcode of buyer 1's last settled home,Do you know the postcode of buyer 1's last settled home?,Was the buyer registered with their PRP (HA)?,Was the buyer registered with the local authority?,Was the buyer registered with a Help to Buy agent?,Was the buyer registered with another PRP (HA)?,Does anyone in the household consider themselves to have a disability?,Does anyone in the household use a wheelchair?,How many bedrooms does the property have?,What type of unit is the property?,Which type of building is the property?,What is the local authority of the property?,Part 1 of postcode of property,Part 2 of postcode of property,Is the property built or adapted to wheelchair-user standards?,What is the type of shared ownership sale?,"Is this a resale? + +Shared ownership","What is the day of the practical completion or handover date? - DD + +Shared ownership","What is the month of the practical completion or handover date? - MM + +Shared ownership","What is the year of the practical completion or handover date? - YY + +Shared ownership","What is the day of the exchange of contracts date? - DD + +Shared ownership","What is the month of the exchange of contracts date? - MM + +Shared ownership","What is the year of the exchange of contracts date? - YY + +Shared ownership","Was the household re-housed under a local authority nominations agreement? + +Shared ownership","How many bedrooms did the buyer's previous property have? + +Shared ownership","What was the type of the buyer's previous property? + +Shared ownership","What was the full purchase price? + +Shared ownership","What was the initial percentage equity stake purchased? + +Shared ownership","What is the mortgage amount? + +Shared ownership","Does this include any extra borrowing? + +Shared ownership","How much was the cash deposit paid on the property? + +Shared ownership","How much cash discount was given through Social Homebuy? + +Shared ownership","What is the basic monthly rent? + +Shared ownership","What are the total monthly leasehold charges for the property? + +Shared ownership",What is the type of discounted ownership sale?,"What was the full purchase price? + +Discounted ownership","What was the amount of any loan, grant, discount or subsidy given? + +Discounted ownership","What was the percentage discount? + +Discounted ownership","What is the mortgage amount? + +Discounted ownership","Does this include any extra borrowing? + +Discounted ownership","How much was the cash deposit paid on the property? + +Discounted ownership","What are the total monthly leasehold charges for the property? + +Discounted ownership",What is the type of outright sale?,"What is the 'other' type of outright sale? + +Outright sale",[BLANK],"What is the full purchase price? + +Outright sale","What is the mortgage amount? + +Outright sale","Does this include any extra borrowing? + +Outright sale","How much was the cash deposit paid on the property? + +Outright sale","What are the total monthly leasehold charges for the property? + +Outright sale","Which organisation owned this property before the sale? + +Organisation's CORE ID",Username,BLANK,Has the buyer ever served in the UK Armed Forces and for how long?,[BLANK],Are any of the buyers a spouse or civil partner of a UK Armed Forces regular who died in service within the last 2 years?,"What is the name of the mortgage lender? + +Shared ownership","What is the name of the 'other' mortgage lender? + +Shared ownership","What is the name of the mortgage lender? + +Discounted ownership","What is the name of the 'other' mortgage lender? + +Discounted ownership","What is the name of the mortgage lender? + +Outright sale","What is the name of the 'other' mortgage lender? + +Outright sale",Were the buyers receiving any of these housing-related benefits immediately before buying this property?,"What is the length of the mortgage in years? + +Shared ownership","What is the length of the mortgage in years? + +Discounted ownership","What is the length of the mortgage in years? + +Outright sale","How long have the buyers been living in the property before the purchase? + +Discounted ownership",Are there more than two joint purchasers of this property?,"How long have the buyers been living in the property before the purchase? + +Shared ownership",Is this a staircasing transaction?,Data Protection question,Was this purchase made through an ownership scheme?,"Is the buyer a company? + +Outright sale",Will the buyers live in the property?,Is this a joint purchase?,Will buyer 1 live in the property?,Will buyer 2 live in the property?,"Besides the buyers, how many people live in the property?","What percentage of the property has been bought in this staircasing transaction? + +Shared ownership","What percentage of the property does the buyer now own in total? + +Shared ownership","What was the rent type of the buyer's previous property? + +Shared ownership","Was a mortgage used for the purchase of this property? + +Shared ownership","Was a mortgage used for the purchase of this property? + +Discounted ownership","Was a mortgage used for the purchase of this property? + +Outright sale" +Values,Max 9 digits,1 - 31,1 - 12,19 - 23,,1 or null,"15 - 110 +or R",1 - 110 or R,,,,,"M, F, X or R",,,,,,"P, C, X or R",,,,,0 - 10,,,,,,1 - 19,"12 -13, 17 -19",0 - 99999,,1 or 2,1 or 2,0 - 999990,1 - 3,,1 - 7 or 9,ONS CODE - E + 9 digits,XXX(X),XXX,1 or null,,,,,1 - 3,1 - 3,1 - 9,1 - 4 or 9,1 or 2,ONS CODE E + 9 digits,XXX(X),XXX,1 - 3,"2, 16, 18, 24, 28 or 30-31",1 or 2,1 - 31,1 - 12,19 - 23,1 - 31,1 - 12,19 - 23,1 - 3,1 - 9,1 - 4 or 9,0 - 999999,0 - 100,0 - 999999,1 - 3,0 - 999999,,0 - 999.99,,"8, 9, 14, 21, 22, 27 or 29",0 - 999999,,0 - 100,0 - 999999,1 - 3,0 - 999999,0 - 999.99,10 or 12,,,0 - 999999,,1-3,0 - 999999,0-999.99,Up to 7 digits,Username of CORE account this sales log should be assigned to,,3 - 8,,4 - 7,1 - 40,,1 - 40,,1 - 40,,1 - 4, Integer <=60, Integer <=60, Integer <=60, Integer <=80,1 - 3, Integer <=80,1 - 3,1,1 - 3,1 - 2,1 - 2,1 - 2,1 - 2,1 - 2,0 - 5,1 - 100,1 - 100,1-3 or 9-10,1 - 2,1 - 2,1 - 2 +Can be Null?,No,,,,,No,No,"If fields 14, 19 and 25 are all also null","If fields 15, 20 and 26 are all also null","If fields 16, 21 and 27 are all also null","If fields 17, 22 and 28 are all also null","If fields 18, 23 and 29 are all also null",No,"If fields 8, 19 and 25 are all also null","If fields 9, 20 and 26 are also null","If fields 10, 21 and 27 are all also null","If fields 11, 22 and 28 are all also null","If fields 12, 23 and 29 are all also null","If fields 8, 14 and 25 are all also null","If fields 9, 15 and 26 are all also null","If fields 10, 16 and 27 are all also null","If fields 11, 17 and 28 are all also null","If fields 12, 18 and 29 are all also null",If field 6 = 1,"If fields 8, 14 and 19 are all also null","If fields 9, 15 and 20 are all also null","If fields 10, 16 and 21 are all also null","If fields 11, 17 and 22 are all also null","If fields 12, 18 and 23 are all also null",If field 6 = 1,,,If field 116 = 2,If field 32 is null,If field 116 = 2,If field 6 = 1,,,If field 6 = 1,No,If field 43 = 1,,If fields 41 and 42 BOTH have valid entries,Yes,,,,If field 6 = 1,,No,,,,,,,If field 113 = 2 or 3,,,,,,,,,"If field 113 = 2 or 3 +OR +field 39 = 3 - 7 or 9",,If field 113 = 2 or 3,,,,,"If field 57 is null, 2, 16, 24 or 28",If field 113 = 2 or 3,,If field 113 = 1 or 3,If field 76 is null,"If field 76 is null, 9 or 14","If field 76 is null, 8, 21 or 22",If field 113 = 1 or 3,,,,If field 113 = 1 or 2,If field 84 is null or 10,,If field 113 = 1 or 2,,,,,No,Yes,,No,,No,If field 113 = 2 or 3,"If field 113 = 2 or 3 +OR +If field 98 is not 40",If field 113 = 1 or 3,"If field 113 = 1 or 3 +OR +If field 100 is not 40",If field 113 = 1 or 2,"If field 113 = 1 or 2 +OR +If field 102 is not 40",No,If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2,If field 113 = 1 or 3,If field 116 = 2,If field 113 = 2 or 3,If field 113 = 2 or 3,No,No,If field 113 = 1 or 2,If field 113 = 1 or 2,No,No,If field 116 = 2,No,If field 113 = 2 or 3,If field 113 = 2 or 3,"If field 113 = 1 or 2 +OR +If field 39 = 3 - 9",If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2 +Bulk upload format and duplicate check,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +Field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125 +,1,1,1,23,,1,30,,,,,,M,,,,,,,,,,,1,,,,,,1,18,20000,,1,,10000,2,,3,,EC1N,2TD,2,2,2,2,2,2,2,2,1,1,,,,2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb b/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb index 49f7510ef..a71d446fe 100644 --- a/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb +++ b/spec/models/forms/bulk_upload_lettings/upload_your_file_spec.rb @@ -51,5 +51,11 @@ RSpec.describe Forms::BulkUploadLettings::UploadYourFile do expect(Storage::S3Service).to have_received(:new) expect(mock_storage_service).to have_received(:write_file).with(bulk_upload.identifier, actual_file.read) end + + it "enqueues job to process bulk upload" do + expect { + form.save! + }.to have_enqueued_job(ProcessBulkUploadJob) + end end end diff --git a/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb b/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb index 8e7497633..512e85aa2 100644 --- a/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb +++ b/spec/models/forms/bulk_upload_sales/upload_your_file_spec.rb @@ -49,5 +49,11 @@ RSpec.describe Forms::BulkUploadSales::UploadYourFile do expect(Storage::S3Service).to have_received(:new) expect(mock_storage_service).to have_received(:write_file).with(bulk_upload.identifier, actual_file.read) end + + it "enqueues job to process bulk upload" do + expect { + form.save! + }.to have_enqueued_job(ProcessBulkUploadJob) + end end end diff --git a/spec/requests/bulk_upload_lettings_results_controller_spec.rb b/spec/requests/bulk_upload_lettings_results_controller_spec.rb new file mode 100644 index 000000000..15ba0b7bb --- /dev/null +++ b/spec/requests/bulk_upload_lettings_results_controller_spec.rb @@ -0,0 +1,57 @@ +require "rails_helper" + +RSpec.describe BulkUploadLettingsResultsController, type: :request do + let(:user) { create(:user) } + let(:bulk_upload) { create(:bulk_upload, :lettings, user:, bulk_upload_errors:) } + let(:bulk_upload_errors) { create_list(:bulk_upload_error, 2) } + + before do + sign_in user + end + + describe "GET /lettings-logs/bulk-upload-results/:ID" do + it "renders correct year" do + get "/lettings-logs/bulk-upload-results/#{bulk_upload.id}" + + expect(response).to be_successful + expect(response.body).to include("Bulk Upload for lettings (2022/23)") + end + + it "renders correct number of errors" do + get "/lettings-logs/bulk-upload-results/#{bulk_upload.id}" + + expect(response).to be_successful + expect(response.body).to include("We found 2 errors in your file") + end + + it "renders filename of the upload" do + get "/lettings-logs/bulk-upload-results/#{bulk_upload.id}" + + expect(response).to be_successful + expect(response.body).to include(bulk_upload.filename) + end + + context "when there are errors for more than 1 row" do + let(:bulk_upload_errors) { [bulk_upload_error_1, bulk_upload_error_2] } + let(:bulk_upload_error_1) { create(:bulk_upload_error, row: 1) } + let(:bulk_upload_error_2) { create(:bulk_upload_error, row: 2) } + + it "renders no. of tables equal to no. of rows with errors" do + get "/lettings-logs/bulk-upload-results/#{bulk_upload.id}" + + expect(response.body).to include("