Browse Source

able to view lettings bulk upload errors

remotes/origin/bulk-upload-errors-integration
Phil Lee 4 years ago
parent
commit
cfb3fddaaf
  1. 29
      app/components/bulk_upload_error_row_component.html.erb
  2. 25
      app/components/bulk_upload_error_row_component.rb
  3. 7
      app/controllers/bulk_upload_lettings_results_controller.rb
  4. 5
      app/models/bulk_upload.rb
  5. 3
      app/models/bulk_upload_error.rb
  6. 1
      app/models/user.rb
  7. 185
      app/services/bulk_upload/lettings/row_parser.rb
  8. 230
      app/services/bulk_upload/lettings_validator.rb
  9. 18
      app/views/bulk_upload_lettings_results/show.html.erb
  10. 3
      config/routes.rb
  11. 18
      db/migrate/20221209161927_create_bulk_upload_errors.rb
  12. 13
      db/schema.rb
  13. 62
      spec/components/bulk_upload_error_row_component_spec.rb
  14. 13
      spec/factories/bulk_upload_error.rb
  15. 20
      spec/fixtures/files/2021_22_lettings_bulk_upload.csv
  16. 81
      spec/services/bulk_upload/lettings/row_parser_spec.rb
  17. 38
      spec/services/bulk_upload/lettings_validator_spec.rb

29
app/components/bulk_upload_error_row_component.html.erb

@ -0,0 +1,29 @@
<div class="x-govuk-summary-card">
<div class="x-govuk-summary-card__header">
<h3 class="x-govuk-summary-card__title"><strong>Row <%= row %></strong> <span class="govuk-!-margin-left-3">Tenant code: <%= tenant_code %></span> <span class="govuk-!-margin-left-3">Property reference: <%= property_ref %></span></h3>
</div>
<div class="x-govuk-summary-card__body">
<%= govuk_table do |table| %>
<% table.head do |head| %>
<% head.row do |row| %>
<% row.cell(header: true, text: "Cell") %>
<% row.cell(header: true, text: "Question") %>
<% row.cell(header: true, text: "Error") %>
<% row.cell(header: true, text: "Specification") %>
<% end %>
<% table.body do |body| %>
<% bulk_upload_errors.each do |error| %>
<% body.row do |row| %>
<% row.cell(header: true, text: error.cell) %>
<% row.cell(text: question_for_field(error.field)) %>
<% row.cell(text: error.error) %>
<% row.cell(text: error.field.humanize) %>
<% end %>
<% end %>
<% end %>
<% end %>
<% end %>
</div>
</div>

25
app/components/bulk_upload_error_row_component.rb

@ -0,0 +1,25 @@
class BulkUploadErrorRowComponent < ViewComponent::Base
attr_reader :bulk_upload_errors
def initialize(bulk_upload_errors:)
@bulk_upload_errors = bulk_upload_errors
super
end
def row
bulk_upload_errors.first.row
end
def tenant_code
bulk_upload_errors.first.tenant_code
end
def property_ref
bulk_upload_errors.first.property_ref
end
def question_for_field(field)
BulkUpload::LettingsValidator.question_for_field(field.to_sym)
end
end

7
app/controllers/bulk_upload_lettings_results_controller.rb

@ -0,0 +1,7 @@
class BulkUploadLettingsResultsController < ApplicationController
before_action :authenticate_user!
def show
@bulk_upload = current_user.bulk_uploads.find(params[:id])
end
end

5
app/models/bulk_upload.rb

@ -2,9 +2,14 @@ class BulkUpload < ApplicationRecord
enum log_type: { lettings: "lettings", sales: "sales" }
belongs_to :user
has_many :bulk_upload_errors
after_initialize :generate_identifier, unless: :identifier
def year_combo
"#{year}/#{year - 2000 + 1}"
end
private
def generate_identifier

3
app/models/bulk_upload_error.rb

@ -0,0 +1,3 @@
class BulkUploadError < ApplicationRecord
belongs_to :bulk_upload
end

1
app/models/user.rb

@ -12,6 +12,7 @@ class User < ApplicationRecord
has_many :owned_sales_logs, through: :organisation
has_many :managed_sales_logs, through: :organisation
has_many :legacy_users
has_many :bulk_uploads
validates :name, presence: true
validates :email, presence: true

185
app/services/bulk_upload/lettings/row_parser.rb

@ -0,0 +1,185 @@
class BulkUpload::Lettings::RowParser
include ActiveModel::Model
include ActiveModel::Attributes
attribute :field_1, :integer
attribute :field_2
attribute :field_3
attribute :field_4, :integer
attribute :field_5, :integer
attribute :field_6
attribute :field_7, :string
attribute :field_8, :integer
attribute :field_9, :integer
attribute :field_10, :string
attribute :field_11, :integer
attribute :field_12, :string
attribute :field_13, :string
attribute :field_14, :string
attribute :field_15, :string
attribute :field_16, :string
attribute :field_17, :string
attribute :field_18, :string
attribute :field_19, :string
attribute :field_20, :string
attribute :field_21, :string
attribute :field_22, :string
attribute :field_23, :string
attribute :field_24, :string
attribute :field_25, :string
attribute :field_26, :string
attribute :field_27, :string
attribute :field_28, :string
attribute :field_29, :string
attribute :field_30, :string
attribute :field_31, :string
attribute :field_32, :string
attribute :field_33, :string
attribute :field_34, :string
attribute :field_35, :integer
attribute :field_36, :integer
attribute :field_37, :integer
attribute :field_38, :integer
attribute :field_39, :integer
attribute :field_40, :integer
attribute :field_41, :integer
attribute :field_42, :integer
attribute :field_43, :integer
attribute :field_44, :integer
attribute :field_45, :integer
attribute :field_46, :integer
attribute :field_47, :integer
attribute :field_48, :integer
attribute :field_49, :integer
attribute :field_50, :integer
attribute :field_51, :integer
attribute :field_52, :integer
attribute :field_53, :string
attribute :field_54
attribute :field_55, :integer
attribute :field_56, :integer
attribute :field_57, :integer
attribute :field_58, :integer
attribute :field_59, :integer
attribute :field_60, :integer
attribute :field_61, :integer
attribute :field_62, :string
attribute :field_63, :string
attribute :field_64, :string
attribute :field_65, :integer
attribute :field_66, :integer
attribute :field_67, :integer
attribute :field_68, :integer
attribute :field_69, :integer
attribute :field_70, :integer
attribute :field_71, :integer
attribute :field_72, :integer
attribute :field_73, :integer
attribute :field_74, :integer
attribute :field_75, :integer
attribute :field_76, :integer
attribute :field_77, :integer
attribute :field_78, :integer
attribute :field_79, :integer
attribute :field_80, :decimal
attribute :field_81, :decimal
attribute :field_82, :decimal
attribute :field_83, :decimal
attribute :field_84, :decimal
attribute :field_85, :decimal
attribute :field_86, :integer
attribute :field_87, :integer
attribute :field_88, :decimal
attribute :field_89, :integer
attribute :field_90, :integer
attribute :field_91, :integer
attribute :field_92, :integer
attribute :field_93, :integer
attribute :field_94, :integer
attribute :field_95
attribute :field_96, :integer
attribute :field_97, :integer
attribute :field_98, :integer
attribute :field_99, :integer
attribute :field_100, :string
attribute :field_101, :integer
attribute :field_102, :integer
attribute :field_103, :integer
attribute :field_104, :integer
attribute :field_105, :integer
attribute :field_106, :integer
attribute :field_107, :string
attribute :field_108, :string
attribute :field_109, :string
attribute :field_110
attribute :field_111, :integer
attribute :field_112, :string
attribute :field_113, :integer
attribute :field_114, :integer
attribute :field_115
attribute :field_116, :integer
attribute :field_117, :integer
attribute :field_118, :integer
attribute :field_119, :integer
attribute :field_120, :integer
attribute :field_121, :integer
attribute :field_122, :integer
attribute :field_123, :integer
attribute :field_124, :integer
attribute :field_125, :integer
attribute :field_126, :integer
attribute :field_127, :integer
attribute :field_128, :integer
attribute :field_129, :integer
attribute :field_130, :integer
attribute :field_131, :string
attribute :field_132, :integer
attribute :field_133, :integer
attribute :field_134, :integer
validates :field_1, presence: true, numericality: { in: (1..12) }
validates :field_4, numericality: { in: (1..999), allow_blank: true }
validates :field_4, presence: true, if: :field_4_presence_check
validate :validate_possible_answers
# delegate :valid?, to: :native_object
# delegate :errors, to: :native_object
private
def native_object
@native_object ||= LettingsLog.new(attributes_for_log)
end
def field_mapping
{
field_134: :renewal,
}
end
def validate_possible_answers
field_mapping.each do |field, attribute|
possible_answers = FormHandler.instance.current_lettings_form.questions.find { |q| q.id == attribute.to_s }.answer_options.keys
unless possible_answers.include?(public_send(field))
errors.add(field, "foo")
end
end
end
def attributes_for_log
hash = field_mapping.invert
attributes = {}
hash.map do |k, v|
attributes[k] = public_send(v)
end
attributes
end
def field_4_presence_check
[1, 3, 5, 7, 9, 11].include?(field_1)
end
end

230
app/services/bulk_upload/lettings_validator.rb

@ -0,0 +1,230 @@
require "csv"
class BulkUpload::LettingsValidator
include ActiveModel::Validations
QUESTIONS = {
field_1: "What is the letting type?",
field_2: "This question has been removed",
field_3: "This question has been removed",
field_4: "Management group code",
field_5: "Scheme code",
field_6: "This question has been removed",
field_7: "What is the tenant code?",
field_8: "Is this a starter tenancy?",
field_9: "What is the tenancy type?",
field_10: "If 'Other', what is the tenancy type?",
field_11: "What is the length of the fixed-term tenancy to the nearest year?",
field_12: "Age of Person 1",
field_13: "Age of Person 2",
field_14: "Age of Person 3",
field_15: "Age of Person 4",
field_16: "Age of Person 5",
field_17: "Age of Person 6",
field_18: "Age of Person 7",
field_19: "Age of Person 8",
field_20: "Gender identity of Person 1",
field_21: "Gender identity of Person 2",
field_22: "Gender identity of Person 3",
field_23: "Gender identity of Person 4",
field_24: "Gender identity of Person 5",
field_25: "Gender identity of Person 6",
field_26: "Gender identity of Person 7",
field_27: "Gender identity of Person 8",
field_28: "Relationship to Person 1 for Person 2",
field_29: "Relationship to Person 1 for Person 3",
field_30: "Relationship to Person 1 for Person 4",
field_31: "Relationship to Person 1 for Person 5",
field_32: "Relationship to Person 1 for Person 6",
field_33: "Relationship to Person 1 for Person 7",
field_34: "Relationship to Person 1 for Person 8",
field_35: "Working situation of Person 1",
field_36: "Working situation of Person 2",
field_37: "Working situation of Person 3",
field_38: "Working situation of Person 4",
field_39: "Working situation of Person 5",
field_40: "Working situation of Person 6",
field_41: "Working situation of Person 7",
field_42: "Working situation of Person 8",
field_43: "What is the lead tenant's ethnic group?",
field_44: "What is the lead tenant's nationality?",
field_45: "Does anybody in the household have links to the UK armed forces?",
field_46: "Was the person seriously injured or ill as a result of serving in the UK armed forces?",
field_47: "Is anybody in the household pregnant?",
field_48: "Is the tenant likely to be receiving benefits related to housing?",
field_49: "How much of the household's income is from Universal Credit, state pensions or benefits?",
field_50: "How much income does the household have in total?",
field_51: "Do you know the household's income?",
field_52: "What is the tenant's main reason for the household leaving their last settled home?",
field_53: "If 'Other', what was the main reason for leaving their last settled home?",
field_54: "This question has been removed",
field_55: "Does anybody in the household have any disabled access needs?",
field_56: "Does anybody in the household have any disabled access needs?",
field_57: "Does anybody in the household have any disabled access needs?",
field_58: "Does anybody in the household have any disabled access needs?",
field_59: "Does anybody in the household have any disabled access needs?",
field_60: "Does anybody in the household have any disabled access needs?",
field_61: "Where was the household immediately before this letting?",
field_62: "What is the local authority of the household's last settled home?",
field_63: "Part 1 of postcode of last settled home",
field_64: "Part 2 of postcode of last settled home",
field_65: "Do you know the postcode of last settled home?",
field_66: "How long has the household continuously lived in the local authority area of the new letting?",
field_67: "How long has the household been on the waiting list for the new letting?",
field_68: "Was the tenant homeless directly before this tenancy?",
field_69: "Was the household given 'reasonable preference' by the local authority?",
field_70: "Reasonable preference. They were homeless or about to lose their home (within 56 days)",
field_71: "Reasonable preference. They were living in insanitary, overcrowded or unsatisfactory housing",
field_72: "Reasonable preference. They needed to move on medical and welfare grounds (including a disability)",
field_73: "Reasonable preference. They needed to move to avoid hardship to themselves or others",
field_74: "Reasonable preference. Don't know",
field_75: "Was the letting made under any of the following allocations systems?",
field_76: "Was the letting made under any of the following allocations systems?",
field_77: "Was the letting made under any of the following allocations systems?",
field_78: "What was the source of referral for this letting?",
field_79: "How often does the household pay rent and other charges?",
field_80: "What is the basic rent?",
field_81: "What is the service charge?",
field_82: "What is the personal service charge?",
field_83: "What is the support charge?",
field_84: "Total Charge",
field_85: "If this is a care home, how much does the household pay every [time period]?",
field_86: "Does the household pay rent or other charges for the accommodation?",
field_87: "After the household has received any housing-related benefits, will they still need to pay basic rent and other charges?",
field_88: "What do you expect the outstanding amount to be?",
field_89: "What is the void or renewal date?",
field_90: "What is the void or renewal date?",
field_91: "What is the void or renewal date?",
field_92: "What date were major repairs completed on?",
field_93: "What date were major repairs completed on?",
field_94: "What date were major repairs completed on?",
field_95: "This question has been removed",
field_96: "What date did the tenancy start?",
field_97: "What date did the tenancy start?",
field_98: "What date did the tenancy start?",
field_99: "Since becoming available, how many times has the property been previously offered?",
field_100: "What is the property reference?",
field_101: "How many bedrooms does the property have?",
field_102: "What type of unit is the property?",
field_103: "Which type of building is the property?",
field_104: "Is the property built or adapted to wheelchair-user standards?",
field_105: "What type was the property most recently let as?",
field_106: "What is the reason for the property being vacant?",
field_107: "What is the local authority of the property?",
field_108: "Part 1 of postcode of the property",
field_109: "Part 2 of postcode of the property",
field_110: "This question has been removed",
field_111: "Which organisation owns this property?",
field_112: "Username field",
field_113: "Which organisation manages this property?",
field_114: "Is the person still serving in the UK armed forces?",
field_115: "This question has been removed",
field_116: "How often does the household receive income?",
field_117: "Is this letting sheltered accommodation?",
field_118: "Does anybody in the household have a physical or mental health condition (or other illness) expected to last for 12 months or more?",
field_119: "Vision, for example blindness or partial sight",
field_120: "Hearing, for example deafness or partial hearing",
field_121: "Mobility, for example walking short distances or climbing stairs",
field_122: "Dexterity, for example lifting and carrying objects, using a keyboard",
field_123: "Learning or understanding or concentrating",
field_124: "Memory",
field_125: "Mental health",
field_126: "Stamina or breathing or fatigue",
field_127: "Socially or behaviourally, for example associated with autism spectral disorder (ASD) which includes Aspergers' or attention deficit hyperactivity disorder (ADHD)",
field_128: "Other",
field_129: "Is this letting a London Affordable Rent letting?",
field_130: "Which type of Intermediate Rent is this letting?",
field_131: "Which 'Other' type of Intermediate Rent is this letting?",
field_132: "Data Protection",
field_133: "Is this a joint tenancy?",
field_134: "Is this letting a renewal?",
}.freeze
attr_reader :bulk_upload, :path
validate :validate_file_not_empty
validate :validate_max_columns
def initialize(bulk_upload:, path:)
@bulk_upload = bulk_upload
@path = path
end
def call
row_parsers.each do |row_parser|
row_parser.valid?
row_parser.errors.each do |error|
bulk_upload.bulk_upload_errors.create!(field: error.attribute, error: error.type)
end
end
end
def self.question_for_field(field)
QUESTIONS[field]
end
private
def row_parsers
@row_parsers ||= body_rows.map do |row|
stripped_row = row[1..]
headers = ("field_1".."field_134").to_a
hash = Hash[headers.zip(stripped_row)]
BulkUpload::Lettings::RowParser.new(hash)
end
end
# determine the row seperator from CSV
# Windows will use \r\n
def row_sep
contents = ""
File.open(path, "r") do |f|
f.seek(9900)
contents = f.read
end
rn_count = contents.scan("\r\n").count
n_count = contents.scan(/[^\r]\n/).count
if rn_count > n_count
"\r\n"
else
"\n"
end
end
def rows
@rows ||= CSV.read(path, row_sep:)
end
def body_rows
rows[6..]
end
def validate_file_not_empty
if File.size(path).zero?
errors.add(:file, :blank)
halt_validations!
end
end
def validate_max_columns
return if halt_validations?
max_row_size = rows.map(&:size).max
errors.add(:file, :max_row_size) if max_row_size > 136
end
def halt_validations!
@halt_validations = true
end
def halt_validations?
@halt_validations ||= false
end
end

18
app/views/bulk_upload_lettings_results/show.html.erb

@ -0,0 +1,18 @@
<div class="govuk-grid-row">
<div class="govuk-grid-column-two-thirds">
<span class="govuk-caption-l">Bulk Upload for lettings (<%= @bulk_upload.year_combo %>)</span>
<h1 class="govuk-heading-l">We found X errors in your file</h1>
<div class="govuk-body">
Here’s a list of everything that you need to fix your spreadsheet. You can download the specification to help you fix the cells in your CSV file.
</div>
<h2 class="govuk-heading-m"><%= @bulk_upload.filename %></h2>
</div>
</div>
<div class="govuk-grid-row">
<div class="govuk-grid-column-full">
<%= render BulkUploadErrorRowComponent.new(bulk_upload_errors: @bulk_upload.bulk_upload_errors) %>
</div>
</div>

3
config/routes.rb

@ -123,6 +123,9 @@ Rails.application.routes.draw do
collection do
post "bulk-upload", to: "bulk_upload#bulk_upload"
get "bulk-upload", to: "bulk_upload#show"
resources :bulk_upload_lettings_results, path: "bulk-upload-results", only: [:show]
get "csv-download", to: "lettings_logs#download_csv"
post "email-csv", to: "lettings_logs#email_csv"
get "csv-confirmation", to: "lettings_logs#csv_confirmation"

18
db/migrate/20221209161927_create_bulk_upload_errors.rb

@ -0,0 +1,18 @@
class CreateBulkUploadErrors < ActiveRecord::Migration[7.0]
def change
create_table :bulk_upload_errors do |t|
t.references :bulk_upload
t.text :cell
t.text :row
t.text :tenant_code
t.text :property_ref
t.text :field
t.text :error
t.timestamps
end
end
end

13
db/schema.rb

@ -14,6 +14,19 @@ ActiveRecord::Schema[7.0].define(version: 2023_01_04_164318) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
create_table "bulk_upload_errors", force: :cascade do |t|
t.bigint "bulk_upload_id"
t.text "cell"
t.text "row"
t.text "tenant_code"
t.text "property_ref"
t.text "field"
t.text "error"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["bulk_upload_id"], name: "index_bulk_upload_errors_on_bulk_upload_id"
end
create_table "bulk_uploads", force: :cascade do |t|
t.bigint "user_id"
t.text "log_type", null: false

62
spec/components/bulk_upload_error_row_component_spec.rb

@ -0,0 +1,62 @@
require "rails_helper"
RSpec.describe BulkUploadErrorRowComponent, type: :component do
context "when a single error" do
let(:row) { rand(9_999) }
let(:tenant_code) { SecureRandom.hex(4) }
let(:property_ref) { SecureRandom.hex(4) }
let(:field) { :field_134 }
let(:error) { "some error" }
let(:bulk_upload_errors) do
[
FactoryBot.build(
:bulk_upload_error,
row:,
tenant_code:,
property_ref:,
field:,
error:,
),
]
end
it "renders the row number" do
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content("Row #{row}")
end
it "renders the tenant_code" do
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content("Tenant code: #{tenant_code}")
end
it "renders the property_ref" do
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content("Property reference: #{property_ref}")
end
it "renders the cell of error" do
expected = bulk_upload_errors.first.cell
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content(expected)
end
it "renders the question" do
expected = "Is this letting a renewal?"
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content(expected)
end
it "renders the error" do
expected = error
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content(expected)
end
it "renders the field number" do
expected = bulk_upload_errors.first.field.humanize
result = render_inline(described_class.new(bulk_upload_errors:))
expect(result).to have_content(expected)
end
end
end

13
spec/factories/bulk_upload_error.rb

@ -0,0 +1,13 @@
require "securerandom"
FactoryBot.define do
factory :bulk_upload_error do
bulk_upload
row { rand(9_999) }
cell { "#{('A'..'Z').to_a.sample}#{row}" }
tenant_code { SecureRandom.hex(4) }
property_ref { SecureRandom.hex(4) }
field { "field_#{rand(134)}" }
error { "some error" }
end
end

20
spec/fixtures/files/2021_22_lettings_bulk_upload.csv vendored

File diff suppressed because one or more lines are too long

81
spec/services/bulk_upload/lettings/row_parser_spec.rb

@ -0,0 +1,81 @@
require "rails_helper"
RSpec.describe BulkUpload::Lettings::RowParser do
subject(:parser) { described_class.new(attributes) }
describe "validations" do
before do
parser.valid?
end
describe "field_1" do
context "when null" do
let(:attributes) { { field_1: nil } }
it "returns an error" do
expect(parser.errors).to include(:field_1)
end
end
context "when outside permited range" do
let(:attributes) { { field_1: "13" } }
it "returns an error" do
expect(parser.errors).to include(:field_1)
end
end
context "when valid" do
let(:attributes) { { field_1: 1 } }
it "is valid" do
expect(parser.errors).not_to include(:field_1)
end
end
end
describe "field_4" do
context "when text" do
let(:attributes) { { field_4: "R" } }
it "is not valid" do
expect(parser.errors).to include(:field_4)
end
end
context "when valid" do
let(:attributes) { { field_4: "3" } }
it "is valid" do
expect(parser.errors).not_to include(:field_4)
end
end
context "when allowed to be null" do
let(:attributes) { { field_1: "2", field_4: "" } }
it "is valid" do
expect(parser.errors).not_to include(:field_4)
end
end
context "when not allowed to be null" do
let(:attributes) { { field_1: "3", field_4: "" } }
it "is not valid" do
expect(parser.errors).to include(:field_4)
end
end
end
describe "#field_134" do
context "when not a possible value" do
let(:attributes) { { field_134: "3" } }
it "is not valid" do
expect(parser.errors).to include(:field_134)
end
end
end
end
end

38
spec/services/bulk_upload/lettings_validator_spec.rb

@ -0,0 +1,38 @@
require "rails_helper"
RSpec.describe BulkUpload::LettingsValidator do
subject(:validator) { described_class.new(path:) }
let(:path) { file.path }
let(:file) { Tempfile.new }
describe "validations" do
context "when file is empty" do
it "is not valid" do
expect(validator).not_to be_valid
end
end
context "when file has too many columns" do
before do
file.write("a," * 136)
file.write("\n")
file.rewind
end
it "is not valid" do
expect(validator).not_to be_valid
end
end
context "incorrect headers"
end
context do
let(:path) { file_fixture("2021_22_lettings_bulk_upload.csv") }
it do
validator.call
end
end
end
Loading…
Cancel
Save