Create log creator and update row parser for sales

app/services/bulk_upload/sales/log_creator.rb

@@ -0,0 +1,70 @@
+class BulkUpload::Sales::LogCreator
+  attr_reader :bulk_upload, :path
+
+  def initialize(bulk_upload:, path:)
+    @bulk_upload = bulk_upload
+    @path = path
+  end
+
+  def call
+    row_parsers.each do |row_parser|
+      row_parser.valid?
+
+      next if row_parser.blank_row?
+
+      row_parser.log.blank_invalid_non_setup_fields!
+      row_parser.log.bulk_upload = bulk_upload
+      row_parser.log.skip_update_status = true
+      row_parser.log.status = "pending"
+
+      row_parser.log.status_cache = row_parser.log.calculate_status
+
+      begin
+        row_parser.log.save!
+      rescue StandardError => e
+        Sentry.capture_exception(e)
+      end
+    end
+  end
+
+private
+
+  def csv_parser
+    @csv_parser ||= case bulk_upload.year
+                    when 2022
+                      BulkUpload::Sales::Year2022::CsvParser.new(path:)
+                    when 2023
+                      BulkUpload::Sales::Year2023::CsvParser.new(path:)
+                    else
+                      raise "csv parser not found"
+                    end
+  end
+
+  def row_offset
+    csv_parser.row_offset
+  end
+
+  def col_offset
+    csv_parser.col_offset
+  end
+
+  def row_parsers
+    return @row_parsers if @row_parsers
+
+    @row_parsers = csv_parser.row_parsers
+
+    @row_parsers.each do |row_parser|
+      row_parser.bulk_upload = bulk_upload
+    end
+
+    @row_parsers
+  end
+
+  def body_rows
+    csv_parser.body_rows
+  end
+
+  def rows
+    csv_parser.rows
+  end
+end

app/services/bulk_upload/sales/year2022/csv_parser.rb

@@ -28,7 +28,7 @@ class BulkUpload::Sales::Year2022::CsvParser
       headers = ("field_1".."field_125").to_a
       hash = Hash[headers.zip(stripped_row)]
 
-      BulkUpload::Lettings::Year2022::RowParser.new(hash)
+      BulkUpload::Sales::Year2022::RowParser.new(hash)
     end
   end
 

app/services/bulk_upload/sales/year2022/row_parser.rb

@@ -130,18 +130,20 @@ class BulkUpload::Sales::Year2022::RowParser
     field_125: "Was a mortgage used for the purchase of this property? - Outright sale",
   }.freeze
 
+  attribute :bulk_upload
+
   attribute :field_1, :string
   attribute :field_2, :integer
   attribute :field_3, :integer
   attribute :field_4, :integer
   attribute :field_5
   attribute :field_6, :integer
-  attribute :field_7, :integer
+  attribute :field_7, :string
-  attribute :field_8, :integer
+  attribute :field_8, :string
-  attribute :field_9, :integer
+  attribute :field_9, :string
-  attribute :field_10, :integer
+  attribute :field_10, :string
-  attribute :field_11, :integer
+  attribute :field_11, :string
-  attribute :field_12, :integer
+  attribute :field_12, :string
   attribute :field_13, :string
   attribute :field_14, :string
   attribute :field_15, :string
@@ -149,10 +151,10 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_17, :string
   attribute :field_18, :string
   attribute :field_19, :string
-  attribute :field_20, :integer
+  attribute :field_20, :string
-  attribute :field_21, :integer
+  attribute :field_21, :string
-  attribute :field_22, :integer
+  attribute :field_22, :string
-  attribute :field_23, :integer
+  attribute :field_23, :string
   attribute :field_24, :integer
   attribute :field_25, :integer
   attribute :field_26, :integer
@@ -221,7 +223,7 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_89, :integer
   attribute :field_90, :integer
   attribute :field_91, :integer
-  attribute :field_92, :integer
+  attribute :field_92, :string
   attribute :field_93, :string
   attribute :field_94
   attribute :field_95, :integer
@@ -260,8 +262,6 @@ class BulkUpload::Sales::Year2022::RowParser
   # validates :field_4, numericality: { in: (1..999), allow_blank: true }
   # validates :field_4, presence: true, if: :field_4_presence_check
 
-  validate :validate_possible_answers
-
   # delegate :valid?, to: :native_object
   # delegate :errors, to: :native_object
 
@@ -269,40 +269,367 @@ class BulkUpload::Sales::Year2022::RowParser
     QUESTIONS[field]
   end
 
+  def attribute_set
+    @attribute_set ||= instance_variable_get(:@attributes)
+  end
+
+  def blank_row?
+    attribute_set
+      .to_hash
+      .reject { |k, _| %w[bulk_upload block_log_creation].include?(k) }
+      .values
+      .compact
+      .empty?
+  end
+
+  def log
+    @log ||= SalesLog.new(attributes_for_log)
+  end
+
 private
 
-  def native_object
+  def attributes_for_log
-    @native_object ||= SalesLog.new(attributes_for_log)
+    attributes = {}
+    attributes["purchid"] = field_1
+    attributes["saledate"] = saledate
+
+    attributes["noint"] = 2 if field_6 == 1
+
+    attributes["details_known_2"] = details_known?(2)
+    attributes["details_known_3"] = details_known?(3)
+    attributes["details_known_4"] = details_known?(4)
+    attributes["details_known_5"] = details_known?(5)
+    attributes["details_known_6"] = details_known?(6)
+
+    attributes["age1_known"] = age1_known?
+    attributes["age1"] = field_7 if attributes["age1_known"].zero? && field_7&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age2_known"] = age2_known?
+    attributes["age2"] = field_8 if attributes["age1_known"].zero? && field_8&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age3_known"] = age3_known?
+    attributes["age3"] = field_9 if attributes["age1_known"].zero? && field_9&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age4_known"] = age4_known?
+    attributes["age4"] = field_10 if attributes["age1_known"].zero? && field_10&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age5_known"] = age5_known?
+    attributes["age5"] = field_11 if attributes["age1_known"].zero? && field_11&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["age6_known"] = age6_known?
+    attributes["age6"] = field_12 if attributes["age1_known"].zero? && field_12&.match(/\A\d{1,3}\z|\AR\z/)
+
+    attributes["sex1"] = field_13
+    attributes["sex2"] = field_14
+    attributes["sex3"] = field_15
+    attributes["sex4"] = field_16
+    attributes["sex5"] = field_17
+    attributes["sex6"] = field_18
+
+    attributes["relat2"] = field_19
+    attributes["relat3"] = field_20
+    attributes["relat4"] = field_21
+    attributes["relat5"] = field_22
+    attributes["relat6"] = field_23
+
+    attributes["ecstat1"] = field_24
+    attributes["ecstat2"] = field_25
+    attributes["ecstat3"] = field_26
+    attributes["ecstat4"] = field_27
+    attributes["ecstat5"] = field_28
+    attributes["ecstat6"] = field_29
+
+    attributes["ethnic_group"] = ethnic_group_from_ethnic
+    attributes["ethnic"] = field_30
+    attributes["national"] = field_31
+    attributes["income1"] = field_32
+    attributes["income2"] = field_33
+    attributes["inc1mort"] = field_34
+    attributes["inc2mort"] = field_35
+    attributes["savings"] = field_36
+    attributes["prevown"] = field_37
+
+    attributes["prevten"] = field_39
+    attributes["prevloc"] = field_40
+    attributes["previous_la_known"] = previous_la_known
+    attributes["ppcodenk"] = field_43
+    attributes["ppostcode_full"] = ppostcode_full
+
+    attributes["pregyrha"] = field_44
+    attributes["pregla"] = field_45
+    attributes["pregghb"] = field_46
+    attributes["pregother"] = field_47
+    attributes["pregblank"] = 1 if [field_44, field_45, field_46, field_47].all?(&:blank?)
+
+    attributes["disabled"] = field_48
+    attributes["wheel"] = field_49
+    attributes["beds"] = field_50
+    attributes["proptype"] = field_51
+    attributes["builtype"] = field_52
+    attributes["la_known"] = field_53.present? ? 1 : 0
+    attributes["la"] = field_53
+    attributes["pcodenk"] = 0 if postcode_full.present? || field_43 == 1
+    attributes["postcode_full"] = postcode_full
+    attributes["wchair"] = field_56
+
+    attributes["type"] = sale_type
+
+    attributes["resale"] = field_58
+
+    attributes["hodate"] = hodate
+    attributes["exdate"] = exdate
+
+    attributes["lanomagr"] = field_65
+
+    attributes["frombeds"] = field_66
+    attributes["fromprop"] = field_67
+
+    attributes["value"] = value
+    attributes["equity"] = field_69
+    attributes["mortgage"] = mortgage
+    attributes["extrabor"] = extrabor
+    attributes["deposit"] = deposit
+    attributes["cashdis"] = field_73
+    attributes["mrent"] = field_74
+    attributes["mscharge"] = mscharge
+    attributes["grant"] = field_78
+    attributes["discount"] = field_79
+
+    attributes["othtype"] = field_85
+
+    attributes["owning_organisation_id"] = owning_organisation_id
+    attributes["created_by"] = created_by || bulk_upload.user
+    attributes["hhregres"] = hhregres
+    attributes["hhregresstill"] = hhregresstill
+    attributes["armedforcesspouse"] = field_97
+
+    attributes["mortgagelender"] = mortgagelender
+    attributes["mortgagelenderother"] = mortgagelenderother
+
+    attributes["hb"] = field_104
+
+    attributes["mortlen"] = mortlen
+
+    attributes["proplen"] = field_108
+    attributes["jointmore"] = field_109
+    attributes["proplen"] = field_110
+    attributes["staircase"] = field_111
+    attributes["privacynotice"] = field_112
+    attributes["ownershipsch"] = field_113
+    attributes["companybuy"] = field_114
+    attributes["buylivein"] = field_115
+    attributes["jointpur"] = field_116
+    attributes["buy1livein"] = field_117
+    attributes["buy2livein"] = field_118
+    attributes["hholdcount"] = field_119
+    attributes["stairbought"] = field_120
+    attributes["stairowned"] = field_121
+    attributes["socprevten"] = field_122
+    attributes["mortgageused"] = mortgageused
+
+    attributes
   end
 
-  def field_mapping
+  def saledate
-    {
+    Date.new(field_4 + 2000, field_3, field_2) if field_2.present? && field_3.present? && field_4.present?
-      field_117: :buy1livein,
+  rescue Date::Error
-    }
+    Date.new
   end
 
-  def validate_possible_answers
+  def hodate
-    field_mapping.each do |field, attribute|
+    Date.new(field_61 + 2000, field_60, field_59) if field_59.present? && field_60.present? && field_61.present?
-      possible_answers = FormHandler.instance.current_sales_form.questions.find { |q| q.id == attribute.to_s }.answer_options.keys
+  rescue Date::Error
+    Date.new
+  end
 
-      unless possible_answers.include?(public_send(field))
+  def exdate
-        errors.add(field, "Value supplied is not one of the permitted values")
+    Date.new(field_64 + 2000, field_63, field_62) if field_62.present? && field_63.present? && field_64.present?
-      end
+  rescue Date::Error
+    Date.new
+  end
+
+  def age1_known?
+    return 1 if field_7 == "R"
+    return 1 if field_7.blank?
+
+    0
+  end
+
+  [
+    { person: 2, field: :field_8 },
+    { person: 3, field: :field_9 },
+    { person: 4, field: :field_10 },
+    { person: 5, field: :field_11 },
+    { person: 6, field: :field_12 },
+  ].each do |hash|
+    define_method("age#{hash[:person]}_known?") do
+      return 1 if public_send(hash[:field]) == "R"
+      return 0 if send("person_#{hash[:person]}_present?")
+      return 1 if public_send(hash[:field]).blank?
+
+      0
     end
   end
 
-  def attributes_for_log
+  def person_2_present?
-    hash = field_mapping.invert
+    field_8.present? || field_14.present? || field_19.present?
-    attributes = {}
+  end
+
+  def person_3_present?
+    field_9.present? || field_15.present? || field_20.present?
+  end
 
-    hash.map do |k, v|
+  def person_4_present?
-      attributes[k] = public_send(v)
+    field_10.present? || field_16.present? || field_21.present?
+  end
+
+  def person_5_present?
+    field_11.present? || field_17.present? || field_22.present?
+  end
+
+  def person_6_present?
+    field_12.present? || field_18.present? || field_23.present?
+  end
+
+  def details_known?(person_n)
+    send("person_#{person_n}_present?") ? 0 : 1
+  end
+
+  def ethnic_group_from_ethnic
+    return nil if field_43.blank?
+
+    case field_43
+    when 1, 2, 3, 18
+      0
+    when 4, 5, 6, 7
+      1
+    when 8, 9, 10, 11, 15
+      2
+    when 12, 13, 14
+      3
+    when 16, 19
+      4
+    when 17
+      17
     end
+  end
 
-    attributes
+  def postcode_full
+    "#{field_54} #{field_55}" if field_54 && field_55
+  end
+
+  def ppostcode_full
+    "#{field_41} #{field_42}" if field_41 && field_42
   end
 
-  # def field_4_presence_check
+  def sale_type
-  #   [1, 3, 5, 7, 9, 11].include?(field_1)
+    case field_113
-  # end
+    when 1 then field_57
+    when 2 then field_76
+    when 3 then field_84
+    end
+  end
+
+  def value
+    case field_113
+    when 1 then field_68
+    when 2 then field_77
+    when 3 then field_87
+    end
+  end
+
+  def mortgage
+    case field_113
+    when 1 then field_70
+    when 2 then field_80
+    when 3 then field_88
+    end
+  end
+
+  def extrabor
+    case field_113
+    when 1 then field_71
+    when 2 then field_81
+    when 3 then field_89
+    end
+  end
+
+  def deposit
+    case field_113
+    when 1 then field_72
+    when 2 then field_82
+    when 3 then field_90
+    end
+  end
+
+  def mscharge
+    case field_113
+    when 1 then field_75
+    when 2 then field_83
+    when 3 then field_91
+    end
+  end
+
+  def mortgagelender
+    case field_113
+    when 1 then field_98
+    when 2 then field_100
+    when 3 then field_102
+    end
+  end
+
+  def mortgagelenderother
+    case field_113
+    when 1 then field_99
+    when 2 then field_101
+    when 3 then field_103
+    end
+  end
+
+  def mortlen
+    case field_113
+    when 1 then field_105
+    when 2 then field_106
+    when 3 then field_107
+    end
+  end
+
+  def mortgageused
+    case field_113
+    when 1 then field_123
+    when 2 then field_124
+    when 3 then field_125
+    end
+  end
+
+  def owning_organisation
+    Organisation.find_by_id_on_multiple_fields(field_92)
+  end
+
+  def owning_organisation_id
+    owning_organisation&.id
+  end
+
+  def created_by
+    @created_by ||= User.find_by(email: field_93)
+  end
+
+  def hhregres
+    case field_95
+    when 3 then 3
+    when 4, 5, 6 then 1
+    when 7 then 7
+    when 8 then 8
+    end
+  end
+
+  def hhregresstill
+    return unless hhregres == 1
+
+    field_95
+  end
+
+  def previous_la_known
+    field_40.present? ? 1 : 0
+  end
 end

spec/services/bulk_upload/sales/log_creator_spec.rb

@@ -0,0 +1,99 @@
+require "rails_helper"
+
+RSpec.describe BulkUpload::Sales::LogCreator do
+  subject(:service) { described_class.new(bulk_upload:, path:) }
+
+  let(:owning_org) { create(:organisation, old_visible_id: 123) }
+  let(:user) { create(:user, organisation: owning_org) }
+
+  let(:bulk_upload) { create(:bulk_upload, :sales, user:) }
+  let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+
+  describe "#call" do
+    context "when a valid csv with new log" do
+      it "creates a new log" do
+        expect { service.call }.to change(SalesLog, :count)
+      end
+
+      it "create a log with pending status" do
+        service.call
+        expect(SalesLog.last.status).to eql("pending")
+      end
+
+      it "associates log with bulk upload" do
+        service.call
+
+        log = SalesLog.last
+        expect(log.bulk_upload).to eql(bulk_upload)
+        expect(bulk_upload.sales_logs).to include(log)
+      end
+    end
+
+    context "when a valid csv with several blank rows" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) { SalesLog.new }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.rewind
+      end
+
+      it "ignores them and does not create the logs" do
+        expect { service.call }.not_to change(SalesLog, :count)
+      end
+    end
+
+    context "when a valid csv with row with one invalid non setup field" do
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+      let(:log) do
+        build(
+          :sales_log,
+          :completed,
+          age1: 5,
+          owning_organisation: owning_org,
+        )
+      end
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, col_offset: 0).to_2022_sales_csv_row)
+        file.rewind
+      end
+
+      it "creates the log" do
+        expect { service.call }.to change(SalesLog, :count).by(1)
+      end
+
+      it "blanks invalid field" do
+        service.call
+
+        record = SalesLog.last
+        expect(record.age1).to be_blank
+      end
+    end
+
+    context "when pre-creating logs" do
+      subject(:service) { described_class.new(bulk_upload:, path:) }
+
+      it "creates a new log" do
+        expect { service.call }.to change(SalesLog, :count)
+      end
+
+      it "creates a log with correct states" do
+        service.call
+
+        last_log = SalesLog.last
+
+        expect(last_log.status).to eql("pending")
+        expect(last_log.status_cache).to eql("completed")
+      end
+    end
+
+    context "when valid csv with existing log" do
+      xit "what should happen?"
+    end
+  end
+end

spec/support/bulk_upload/log_to_csv.rb

@@ -294,11 +294,9 @@ class BulkUpload::LogToCsv
       log.type, # discounted ownership
       log.value,
       log.grant,
-      log.grant,
       log.discount,
       log.mortgage, # 80
       log.extrabor,
-      log.extrabor,
       log.deposit,
       log.mscharge,