From f0868b81730374f962916da3d45d3c492b3088a5 Mon Sep 17 00:00:00 2001 From: Phil Lee Date: Tue, 24 Jan 2023 09:34:48 +0000 Subject: [PATCH] CLDC-1898 Bulk upload headers (#1210) * fix bulk upload age data types * bulk upload handles both with and without headers - headers are from the spreadsheet template - otherwise assume cell A1 is start of the dataset --- .../bulk_upload/lettings/csv_parser.rb | 10 +++-- .../bulk_upload/lettings/row_parser.rb | 16 ++++---- .../bulk_upload/lettings/csv_parser_spec.rb | 38 +++++++++++++++++++ .../bulk_upload/lettings/log_creator_spec.rb | 3 +- .../bulk_upload/lettings/validator_spec.rb | 3 +- spec/support/bulk_upload/log_to_csv.rb | 9 +++-- 6 files changed, 60 insertions(+), 19 deletions(-) create mode 100644 spec/services/bulk_upload/lettings/csv_parser_spec.rb diff --git a/app/services/bulk_upload/lettings/csv_parser.rb b/app/services/bulk_upload/lettings/csv_parser.rb index dde079840..b8e66678b 100644 --- a/app/services/bulk_upload/lettings/csv_parser.rb +++ b/app/services/bulk_upload/lettings/csv_parser.rb @@ -8,11 +8,11 @@ class BulkUpload::Lettings::CsvParser end def row_offset - 5 + with_headers? ? 5 : 0 end def col_offset - 1 + with_headers? ? 1 : 0 end def cols @@ -21,7 +21,7 @@ class BulkUpload::Lettings::CsvParser def row_parsers @row_parsers ||= body_rows.map do |row| - stripped_row = row[1..] + stripped_row = row[col_offset..] headers = ("field_1".."field_134").to_a hash = Hash[headers.zip(stripped_row)] @@ -39,6 +39,10 @@ class BulkUpload::Lettings::CsvParser private + def with_headers? + rows[0][0]&.match?(/\D+/) + end + def row_sep "\n" end diff --git a/app/services/bulk_upload/lettings/row_parser.rb b/app/services/bulk_upload/lettings/row_parser.rb index 47a53c103..d9a4a3e95 100644 --- a/app/services/bulk_upload/lettings/row_parser.rb +++ b/app/services/bulk_upload/lettings/row_parser.rb @@ -15,14 +15,14 @@ class BulkUpload::Lettings::RowParser attribute :field_9, :integer attribute :field_10, :string attribute :field_11, :integer - attribute :field_12, :string - attribute :field_13, :string - attribute :field_14, :string - attribute :field_15, :string - attribute :field_16, :string - attribute :field_17, :string - attribute :field_18, :string - attribute :field_19, :string + attribute :field_12, :integer + attribute :field_13, :integer + attribute :field_14, :integer + attribute :field_15, :integer + attribute :field_16, :integer + attribute :field_17, :integer + attribute :field_18, :integer + attribute :field_19, :integer attribute :field_20, :string attribute :field_21, :string attribute :field_22, :string diff --git a/spec/services/bulk_upload/lettings/csv_parser_spec.rb b/spec/services/bulk_upload/lettings/csv_parser_spec.rb new file mode 100644 index 000000000..6711d82cd --- /dev/null +++ b/spec/services/bulk_upload/lettings/csv_parser_spec.rb @@ -0,0 +1,38 @@ +require "rails_helper" + +RSpec.describe BulkUpload::Lettings::CsvParser do + subject(:service) { described_class.new(path:) } + + let(:path) { file_fixture("2022_23_lettings_bulk_upload.csv") } + + context "when parsing csv with headers" do + it "returns correct offsets" do + expect(service.row_offset).to eq(5) + expect(service.col_offset).to eq(1) + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_12).to eq(55) + end + end + + context "when parsing csv without headers" do + let(:file) { Tempfile.new } + let(:path) { file.path } + let(:log) { build(:lettings_log, :completed) } + + before do + file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_csv_row) + file.rewind + end + + it "returns correct offsets" do + expect(service.row_offset).to eq(0) + expect(service.col_offset).to eq(0) + end + + it "parses csv correctly" do + expect(service.row_parsers[0].field_12).to eql(log.age1) + end + end +end diff --git a/spec/services/bulk_upload/lettings/log_creator_spec.rb b/spec/services/bulk_upload/lettings/log_creator_spec.rb index acf297dcf..fc7d1cdc0 100644 --- a/spec/services/bulk_upload/lettings/log_creator_spec.rb +++ b/spec/services/bulk_upload/lettings/log_creator_spec.rb @@ -44,8 +44,7 @@ RSpec.describe BulkUpload::Lettings::LogCreator do end before do - 5.times { file.write "\n" } - file.write(BulkUpload::LogToCsv.new(log:).to_csv_row) + file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_csv_row) file.rewind end diff --git a/spec/services/bulk_upload/lettings/validator_spec.rb b/spec/services/bulk_upload/lettings/validator_spec.rb index 624345aac..e242858bf 100644 --- a/spec/services/bulk_upload/lettings/validator_spec.rb +++ b/spec/services/bulk_upload/lettings/validator_spec.rb @@ -70,8 +70,7 @@ RSpec.describe BulkUpload::Lettings::Validator do let(:path) { file.path } before do - file.write("\r\n" * 5) - file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n").to_csv_row) + file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_csv_row) file.rewind end diff --git a/spec/support/bulk_upload/log_to_csv.rb b/spec/support/bulk_upload/log_to_csv.rb index e7d1273d8..3b49f0b86 100644 --- a/spec/support/bulk_upload/log_to_csv.rb +++ b/spec/support/bulk_upload/log_to_csv.rb @@ -1,14 +1,15 @@ class BulkUpload::LogToCsv - attr_reader :log, :line_ending + attr_reader :log, :line_ending, :col_offset - def initialize(log:, line_ending: "\n") + def initialize(log:, line_ending: "\n", col_offset: 1) @log = log @line_ending = line_ending + @col_offset = col_offset end def to_csv_row [ - nil, # 0 + [nil] * col_offset, # 0 log.renttype, # 1 nil, nil, @@ -154,7 +155,7 @@ class BulkUpload::LogToCsv log.joint, renewal, line_ending, - ].join(",") + ].flatten.join(",") end def renewal