Compare commits

...

6 Commits

Author SHA1 Message Date
samyou-softwire 09af58c785 CLDC-4270: Fix tests post merge 5 months ago
samyou-softwire f44ace7d7f Merge branch 'main' into CLDC-4270-fix-text-being-coerced-to-0-on-integer-fields 5 months ago
samyou-softwire 4fe2e5b4bc fixup! CLDC-4270: Port changes to other years' parsers 5 months ago
samyou-softwire 35cbe23973 fixup! CLDC-4270: Add an error for fields with the wrong type for 2026 lettings 5 months ago
Samuel Young 61eb52516d
CLDC-4204: Update bulk upload fields sales (#3223) 5 months ago
Nat Dean-Lewis c95e33bf35
CLDC-4295: hhtype export updates (#3232) 5 months ago
  1. 1
      app/helpers/bulk_upload/lettings_log_to_csv.rb
  2. 76
      app/helpers/bulk_upload/sales_log_to_csv.rb
  3. 21
      app/services/bulk_upload/lettings/year2025/csv_parser.rb
  4. 25
      app/services/bulk_upload/lettings/year2026/csv_parser.rb
  5. 21
      app/services/bulk_upload/sales/year2025/csv_parser.rb
  6. 2
      app/services/bulk_upload/sales/year2025/row_parser.rb
  7. 25
      app/services/bulk_upload/sales/year2026/csv_parser.rb
  8. 1058
      app/services/bulk_upload/sales/year2026/row_parser.rb
  9. 32
      spec/fixtures/files/2026_27_sales_bulk_upload.csv
  10. 2
      spec/fixtures/files/sales_logs_csv_export_codes_26.csv
  11. 2
      spec/fixtures/files/sales_logs_csv_export_labels_26.csv
  12. 1
      spec/fixtures/variable_definitions/sales_download_26_27.csv
  13. 2
      spec/lib/tasks/log_variable_definitions_spec.rb
  14. 2
      spec/services/bulk_upload/sales/validator_spec.rb
  15. 8
      spec/services/bulk_upload/sales/year2026/csv_parser_spec.rb
  16. 512
      spec/services/bulk_upload/sales/year2026/row_parser_spec.rb

1
app/helpers/bulk_upload/lettings_log_to_csv.rb

@ -94,7 +94,6 @@ class BulkUpload::LettingsLogToCsv
end
def to_2026_row
# TODO: CLDC-4162: Implement when 2026 format is known
[
overrides[:organisation_id] || log.owning_organisation&.old_visible_id, # 1
overrides[:managing_organisation_id] || log.managing_organisation&.old_visible_id,

76
app/helpers/bulk_upload/sales_log_to_csv.rb

@ -533,7 +533,6 @@ class BulkUpload::SalesLogToCsv
end
def to_2026_row
# TODO: CLDC-4162: Implement when 2026 template is available
[
log.saledate&.day,
log.saledate&.month,
@ -552,53 +551,66 @@ class BulkUpload::SalesLogToCsv
log.privacynotice,
log.uprn,
log.address_line1&.tr(",", " "), # 20
log.address_line1&.tr(",", " "),
log.address_line2&.tr(",", " "),
log.town_or_city&.tr(",", " "),
log.county&.tr(",", " "),
log.county&.tr(",", " "), # 20
((log.postcode_full || "").split(" ") || [""]).first,
((log.postcode_full || "").split(" ") || [""]).last,
log.la,
log.proptype,
log.buildheightclass,
log.beds,
log.builtype,
log.wchair,
log.age1,
log.sexrab1,
log.ethnic, # 30
log.sexrab1, # 30
log.gender_same_as_sex1,
log.gender_description1,
log.ethnic,
log.nationality_all_group,
log.ecstat1,
log.buy1livein,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat2],
log.age2,
log.sexrab2,
log.gender_same_as_sex2, # 40
log.gender_description2,
log.ethnic_group2,
log.nationality_all_buyer2_group,
log.ecstat2,
log.buy2livein, # 40
log.buy2livein,
log.hholdcount,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat3],
log.age3,
log.sexrab3,
log.gender_same_as_sex3, # 50
log.gender_description3,
log.ecstat3,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat4],
log.age4,
log.sexrab4,
log.gender_same_as_sex4,
log.gender_description4,
log.ecstat4,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat5], # 50
log.age5,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat5],
log.age5, # 60
log.sexrab5,
log.gender_same_as_sex5,
log.gender_description5,
log.ecstat5,
{ "P" => 1, "X" => 2, "R" => 3 }[log.relat6],
log.age6,
log.sexrab6,
log.ecstat6,
log.gender_same_as_sex6,
log.gender_description6,
log.ecstat6, # 70
log.prevten,
log.ppcodenk,
((log.ppostcode_full || "").split(" ") || [""]).first, # 60
((log.ppostcode_full || "").split(" ") || [""]).first,
((log.ppostcode_full || "").split(" ") || [""]).last,
log.prevloc,
log.buy2living,
@ -606,32 +618,32 @@ class BulkUpload::SalesLogToCsv
log.hhregres,
log.hhregresstill,
log.armedforcesspouse,
log.armedforcesspouse, # 80
log.disabled,
log.wheel,
log.income1, # 70
log.income1,
log.inc1mort,
log.income2,
log.inc2mort,
log.hb,
log.savings.present? || "R",
log.prevown,
log.prevshared,
log.prevshared, # 90
log.resale,
log.proplen,
log.hodate&.day, # 80
log.hodate&.day,
log.hodate&.month,
log.hodate&.strftime("%y"),
log.frombeds,
log.fromprop,
log.socprevten,
log.value,
log.equity,
log.equity, # 100
log.mortgageused,
log.mortgage,
log.mortlen, # 90
log.mortlen,
log.deposit,
log.cashdis,
log.mrent,
@ -639,48 +651,34 @@ class BulkUpload::SalesLogToCsv
log.management_fee,
log.stairbought,
log.stairowned,
log.stairowned, # 110
log.staircasesale,
log.firststair,
log.initialpurchase&.day, # 100
log.initialpurchase&.day,
log.initialpurchase&.month,
log.initialpurchase&.strftime("%y"),
log.numstair,
log.lasttransaction&.day,
log.lasttransaction&.month,
log.lasttransaction&.strftime("%y"),
log.value,
log.value, # 120
log.equity,
log.mortgageused,
log.mrentprestaircasing, # 110
log.mrentprestaircasing,
log.mrent,
log.hasservicechargeschanged,
log.newservicecharges,
log.proplen,
log.value,
log.grant,
log.discount,
log.discount, # 130
log.mortgageused,
log.mortgage,
log.mortlen,
log.extrabor,
log.deposit, # 120
log.mscharge,
log.buildheightclass,
log.gender_same_as_sex1,
log.gender_description1,
log.gender_same_as_sex2,
log.gender_description2,
log.gender_same_as_sex3,
log.gender_description3,
log.gender_same_as_sex4,
log.gender_description4, # 130
log.gender_same_as_sex5,
log.gender_description5,
log.gender_same_as_sex6,
log.gender_description6,
log.hasservicechargeschanged,
log.newservicecharges, # 136
log.deposit,
log.mscharge, # 136
]
end

21
app/services/bulk_upload/lettings/year2025/csv_parser.rb

@ -41,12 +41,7 @@ class BulkUpload::Lettings::Year2025::CsvParser
hash_rows = field_numbers
.zip(stripped_row)
.map do |field, value|
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
expected_is_integer = ROW_PARSER_CLASS.attribute_types[field].is_a?(ActiveModel::Type::Integer)
# we must be certain that the user entered a string that cannot be coerced to an integer
actual_is_non_empty_string = value.present? && Integer(value, exception: false).nil? && Float(value, exception: false).nil?
field_is_valid = !(expected_is_integer && actual_is_non_empty_string)
field_is_valid = value_is_valid_for_field(field, value)
correct_value = field_is_valid ? value : nil
@ -136,6 +131,20 @@ private
@normalised_string
end
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
def value_is_valid_for_field(field, value)
field_type = ROW_PARSER_CLASS.attribute_types[field]
if field_type.is_a?(ActiveModel::Type::Integer)
value.nil? || Integer(value, exception: false).present?
elsif field_type.is_a?(ActiveModel::Type::Decimal)
value.nil? || Float(value, exception: false).present?
else
true
end
end
def first_record_start_date
if with_headers?
year = row_parsers.first.field_10.to_s.strip.length.between?(1, 2) ? row_parsers.first.field_10.to_i + 2000 : row_parsers.first.field_10.to_i

25
app/services/bulk_upload/lettings/year2026/csv_parser.rb

@ -3,7 +3,6 @@ require "csv"
class BulkUpload::Lettings::Year2026::CsvParser
include CollectionTimeHelper
# TODO: CLDC-4162: Update when 2026 format is known
FIELDS = 147
FORM_YEAR = 2026
@ -28,8 +27,7 @@ class BulkUpload::Lettings::Year2026::CsvParser
end
def cols
# TODO: CLDC-4162: Update when 2026 format is known
@cols ||= ("A".."FA").to_a
@cols ||= ("A".."ER").to_a
end
def row_parsers
@ -42,12 +40,7 @@ class BulkUpload::Lettings::Year2026::CsvParser
hash_rows = field_numbers
.zip(stripped_row)
.map do |field, value|
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
expected_is_integer = ROW_PARSER_CLASS.attribute_types[field].is_a?(ActiveModel::Type::Integer)
# we must be certain that the user entered a string that cannot be coerced to an integer
actual_is_non_empty_string = value.present? && Integer(value, exception: false).nil? && Float(value, exception: false).nil?
field_is_valid = !(expected_is_integer && actual_is_non_empty_string)
field_is_valid = value_is_valid_for_field(field, value)
correct_value = field_is_valid ? value : nil
@ -146,4 +139,18 @@ private
Date.new(year, rows.first[8].to_i, rows.first[7].to_i)
end
end
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
def value_is_valid_for_field(field, value)
field_type = ROW_PARSER_CLASS.attribute_types[field]
if field_type.is_a?(ActiveModel::Type::Integer)
value.nil? || Integer(value, exception: false).present?
elsif field_type.is_a?(ActiveModel::Type::Decimal)
value.nil? || Float(value, exception: false).present?
else
true
end
end
end

21
app/services/bulk_upload/sales/year2025/csv_parser.rb

@ -41,12 +41,7 @@ class BulkUpload::Sales::Year2025::CsvParser
hash_rows = field_numbers
.zip(stripped_row)
.map do |field, value|
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
expected_is_integer = ROW_PARSER_CLASS.attribute_types[field].is_a?(ActiveModel::Type::Integer)
# we must be certain that the user entered a string that cannot be coerced to an integer
actual_is_non_empty_string = value.present? && Integer(value, exception: false).nil? && Float(value, exception: false).nil?
field_is_valid = !(expected_is_integer && actual_is_non_empty_string)
field_is_valid = value_is_valid_for_field(field, value)
correct_value = field_is_valid ? value : nil
@ -139,6 +134,20 @@ private
@normalised_string
end
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
def value_is_valid_for_field(field, value)
field_type = ROW_PARSER_CLASS.attribute_types[field]
if field_type.is_a?(ActiveModel::Type::Integer)
value.nil? || Integer(value, exception: false).present?
elsif field_type.is_a?(ActiveModel::Type::Decimal)
value.nil? || Float(value, exception: false).present?
else
true
end
end
def first_record_start_date
if with_headers?
year = row_parsers.first.field_3.to_s.strip.length.between?(1, 2) ? row_parsers.first.field_3.to_i + 2000 : row_parsers.first.field_3.to_i

2
app/services/bulk_upload/sales/year2025/row_parser.rb

@ -288,7 +288,7 @@ class BulkUpload::Sales::Year2025::RowParser
attribute :field_112, :integer
attribute :field_113, :decimal
attribute :field_114, :integer
attribute :field_114, :decimal
attribute :field_115, :decimal
attribute :field_116, :integer
attribute :field_117, :decimal

25
app/services/bulk_upload/sales/year2026/csv_parser.rb

@ -3,7 +3,6 @@ require "csv"
class BulkUpload::Sales::Year2026::CsvParser
include CollectionTimeHelper
# TODO: CLDC-4162: Update when 2026 format is known
FIELDS = 136
FORM_YEAR = 2026
@ -28,8 +27,7 @@ class BulkUpload::Sales::Year2026::CsvParser
end
def cols
# TODO: CLDC-4162: Update when 2026 format is known
@cols ||= ("A".."EF").to_a
@cols ||= ("A".."EG").to_a
end
def row_parsers
@ -42,12 +40,7 @@ class BulkUpload::Sales::Year2026::CsvParser
hash_rows = field_numbers
.zip(stripped_row)
.map do |field, value|
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
expected_is_integer = ROW_PARSER_CLASS.attribute_types[field].is_a?(ActiveModel::Type::Integer)
# we must be certain that the user entered a string that cannot be coerced to an integer
actual_is_non_empty_string = value.present? && Integer(value, exception: false).nil? && Float(value, exception: false).nil?
field_is_valid = !(expected_is_integer && actual_is_non_empty_string)
field_is_valid = value_is_valid_for_field(field, value)
correct_value = field_is_valid ? value : nil
@ -140,6 +133,20 @@ private
@normalised_string
end
# this is needed as a string passed to an int attribute is by default mapped to '0'.
# this is bad as some questions will accept a '0'. so you could enter something invalid and not be told about it
def value_is_valid_for_field(field, value)
field_type = ROW_PARSER_CLASS.attribute_types[field]
if field_type.is_a?(ActiveModel::Type::Integer)
value.nil? || Integer(value, exception: false).present?
elsif field_type.is_a?(ActiveModel::Type::Decimal)
value.nil? || Float(value, exception: false).present?
else
true
end
end
def first_record_start_date
if with_headers?
year = row_parsers.first.field_3.to_s.strip.length.between?(1, 2) ? row_parsers.first.field_3.to_i + 2000 : row_parsers.first.field_3.to_i

1058
app/services/bulk_upload/sales/year2026/row_parser.rb

File diff suppressed because it is too large Load Diff

32
spec/fixtures/files/2026_27_sales_bulk_upload.csv vendored

File diff suppressed because one or more lines are too long

2
spec/fixtures/files/sales_logs_csv_export_codes_26.csv vendored

File diff suppressed because one or more lines are too long

2
spec/fixtures/files/sales_logs_csv_export_labels_26.csv vendored

File diff suppressed because one or more lines are too long

1
spec/fixtures/variable_definitions/sales_download_26_27.csv vendored

@ -20,3 +20,4 @@ gender_description6,If 'No', enter person 6's gender identity
hasservicechargeschanged,Will the service charge change after this staircasing transaction takes place?
newservicecharges,What are the new total monthly service charges for the property?
hholdcount,In total, how many people live in the property?
hhtype,Household type

1 sexrab1,What was buyer 1's sex at birth?
20 hasservicechargeschanged,Will the service charge change after this staircasing transaction takes place?
21 newservicecharges,What are the new total monthly service charges for the property?
22 hholdcount,In total, how many people live in the property?
23 hhtype,Household type

2
spec/lib/tasks/log_variable_definitions_spec.rb

@ -6,7 +6,7 @@ RSpec.describe "log_variable_definitions" do
subject(:task) { Rake::Task["data_import:add_variable_definitions"] }
let(:path) { "spec/fixtures/variable_definitions" }
let(:total_variable_definitions_count) { 466 }
let(:total_variable_definitions_count) { 467 }
before do
Rake.application.rake_require("tasks/log_variable_definitions")

2
spec/services/bulk_upload/sales/validator_spec.rb

@ -127,7 +127,7 @@ RSpec.describe BulkUpload::Sales::Validator do
before do
log.owning_organisation = nil
log.saledate = date
file.write(log_to_csv.default_field_numbers_row)
file.write(log_to_csv.default_field_numbers_row_for_year(year))
file.write(log_to_csv.to_csv_row)
file.rewind
end

8
spec/services/bulk_upload/sales/year2026/csv_parser_spec.rb

@ -166,7 +166,7 @@ RSpec.describe BulkUpload::Sales::Year2026::CsvParser do
it "returns correct column" do
expect(service.column_for_field("field_1")).to eql("B")
expect(service.column_for_field("field_99")).to eql("CV")
expect(service.column_for_field("field_112")).to eql("DI")
end
end
end
@ -195,8 +195,8 @@ RSpec.describe BulkUpload::Sales::Year2026::CsvParser do
let(:field_values) { log_to_csv.to_2026_row }
before do
field_32_index = field_numbers.index(32)
field_values[field_32_index] = "abc" # should be an integer
field_34_index = field_numbers.index(34)
field_values[field_34_index] = "abc" # should be an integer
file.write(log_to_csv.custom_field_numbers_row(field_numbers:))
file.write(log_to_csv.to_custom_csv_row(field_values:))
@ -204,7 +204,7 @@ RSpec.describe BulkUpload::Sales::Year2026::CsvParser do
end
it "sets the invalid data to nil" do
expect(service.row_parsers[0].field_32).to be_nil
expect(service.row_parsers[0].field_34).to be_nil
end
end
end

512
spec/services/bulk_upload/sales/year2026/row_parser_spec.rb

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save