Update sales log validator

app/services/bulk_upload/sales/validator.rb

@@ -4,6 +4,7 @@ class BulkUpload::Sales::Validator
   attr_reader :bulk_upload, :path
 
   validate :validate_file_not_empty
+  validate :validate_min_columns
   validate :validate_max_columns
 
   def initialize(bulk_upload:, path:)
@@ -18,56 +19,83 @@ class BulkUpload::Sales::Validator
       row = index + row_offset + 1
 
       row_parser.errors.each do |error|
+        col = csv_parser.column_for_field(error.attribute.to_s)
+
         bulk_upload.bulk_upload_errors.create!(
           field: error.attribute,
-          error: error.type,
+          error: error.message,
           purchaser_code: row_parser.field_1,
           row:,
-          cell: "#{cols[field_number_for_attribute(error.attribute) + col_offset - 1]}#{row}",
+          cell: "#{col}#{row}",
+          col:,
+          category: error.options[:category],
         )
       end
     end
   end
 
-private
+  def create_logs?
+    return false if any_setup_errors?
+    return false if row_parsers.any?(&:block_log_creation?)
 
-  def field_number_for_attribute(attribute)
-    attribute.to_s.split("_").last.to_i
+    row_parsers.all? { |row_parser| row_parser.log.valid? }
   end
 
-  def rows
-    @rows ||= CSV.read(path, row_sep:)
+  def any_setup_errors?
+    bulk_upload
+      .bulk_upload_errors
+      .where(category: "setup")
+      .count
+      .positive?
   end
 
-  def body_rows
-    rows[row_offset..]
+private
+
+  def csv_parser
+    @csv_parser ||= case bulk_upload.year
+                    when 2022
+                      BulkUpload::Sales::Year2022::CsvParser.new(path:)
+                    when 2023
+                      BulkUpload::Sales::Year2023::CsvParser.new(path:)
+                    else
+                      raise "csv parser not found"
+                    end
   end
 
   def row_offset
-    5
+    csv_parser.row_offset
   end
 
   def col_offset
-    1
+    csv_parser.col_offset
+  end
+
+  def field_number_for_attribute(attribute)
+    attribute.to_s.split("_").last.to_i
   end
 
   def cols
-    @cols ||= ("A".."DV").to_a
+    csv_parser.cols
   end
 
   def row_parsers
-    @row_parsers ||= body_rows.map do |row|
-      stripped_row = row[col_offset..]
-      headers = ("field_1".."field_125").to_a
-      hash = Hash[headers.zip(stripped_row)]
+    return @row_parsers if @row_parsers
+
+    @row_parsers = csv_parser.row_parsers
+
+    @row_parsers.each do |row_parser|
+      row_parser.bulk_upload = bulk_upload
+    end
 
-      BulkUpload::Sales::Year2022::RowParser.new(hash)
+    @row_parsers
   end
+
+  def rows
+    csv_parser.rows
   end
 
-  def row_sep
-    "\r\n"
-    # "\n"
+  def body_rows
+    csv_parser.body_rows
   end
 
   def validate_file_not_empty
@@ -78,12 +106,20 @@ private
     end
   end
 
+  def validate_min_columns
+    return if halt_validations?
+
+    column_count = rows.map(&:size).min
+
+    errors.add(:base, :under_min_column_count) if column_count < csv_parser.class::MIN_COLUMNS
+  end
+
   def validate_max_columns
     return if halt_validations?
 
-    max_row_size = rows.map(&:size).max
+    column_count = rows.map(&:size).max
 
-    errors.add(:file, :max_row_size) if max_row_size > 126
+    errors.add(:base, :over_max_column_count) if column_count > csv_parser.class::MAX_COLUMNS
   end
 
   def halt_validations!

app/services/bulk_upload/sales/year2022/csv_parser.rb

@@ -51,7 +51,7 @@ private
   end
 
   def with_headers?
-    rows[0][0]&.match?(/\D+/)
+    rows.map { |r| r[0] }.any? { |cell| cell&.match?(/field number/i) }
   end
 
   def row_sep

app/services/bulk_upload/sales/year2022/row_parser.rb

@@ -259,30 +259,30 @@ class BulkUpload::Sales::Year2022::RowParser
   attribute :field_124, :integer
   attribute :field_125, :integer
 
-  validates :field_2, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (day)") }
-  validates :field_3, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (month)") }
-  validates :field_4, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (year)") }
-  validates :field_4, format: { with: /\A\d{2}\z/, message: I18n.t("validations.setup.saledate.year_not_two_digits") }
-
-  validates :field_113, presence: { message: I18n.t("validations.not_answered", question: "ownership type") }
-  validates :field_57, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :shared_ownership?
-  validates :field_76, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :discounted_ownership?
-  validates :field_84, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :outright_sale?
-  validates :field_115, presence: { message: I18n.t("validations.not_answered", question: "will the buyers live in the property") }, if: :outright_sale?
-  validates :field_116, presence: { message: I18n.t("validations.not_answered", question: "joint purchase") }, if: :joint_purchase_asked?
-  validates :field_114, presence: { message: I18n.t("validations.not_answered", question: "company buyer") }, if: :outright_sale?
-  validates :field_109, presence: { message: I18n.t("validations.not_answered", question: "more than 2 buyers") }, if: :joint_purchase?
-
-  validate :validate_nulls
-  validate :validate_valid_radio_option
-
-  validate :validate_owning_org_data_given
-  validate :validate_owning_org_exists
-  validate :validate_owning_org_permitted
-
-  validate :validate_created_by_exists
-  validate :validate_created_by_related
-  validate :validate_relevant_collection_window
+  validates :field_2, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (day)") }, on: :after_log
+  validates :field_3, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (month)") }, on: :after_log
+  validates :field_4, presence: { message: I18n.t("validations.not_answered", question: "sale completion date (year)") }, on: :after_log
+  validates :field_4, format: { with: /\A\d{2}\z/, message: I18n.t("validations.setup.saledate.year_not_two_digits") }, on: :after_log
+
+  validates :field_113, presence: { message: I18n.t("validations.not_answered", question: "ownership type") }, on: :after_log
+  validates :field_57, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :shared_ownership?, on: :after_log
+  validates :field_76, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :discounted_ownership?, on: :after_log
+  validates :field_84, presence: { message: I18n.t("validations.not_answered", question: "shared ownership type") }, if: :outright_sale?, on: :after_log
+  validates :field_115, presence: { message: I18n.t("validations.not_answered", question: "will the buyers live in the property") }, if: :outright_sale?, on: :after_log
+  validates :field_116, presence: { message: I18n.t("validations.not_answered", question: "joint purchase") }, if: :joint_purchase_asked?, on: :after_log
+  validates :field_114, presence: { message: I18n.t("validations.not_answered", question: "company buyer") }, if: :outright_sale?, on: :after_log
+  validates :field_109, presence: { message: I18n.t("validations.not_answered", question: "more than 2 buyers") }, if: :joint_purchase?, on: :after_log
+
+  validate :validate_nulls, on: :after_log
+  validate :validate_valid_radio_option, on: :before_log
+
+  validate :validate_owning_org_data_given, on: :after_log
+  validate :validate_owning_org_exists, on: :after_log
+  validate :validate_owning_org_permitted, on: :after_log
+
+  validate :validate_created_by_exists, on: :after_log
+  validate :validate_created_by_related, on: :after_log
+  validate :validate_relevant_collection_window, on: :after_log
 
   def self.question_for_field(field)
     QUESTIONS[field]
@@ -310,9 +310,13 @@ class BulkUpload::Sales::Year2022::RowParser
 
     return true if blank_row?
 
+    super(:before_log)
+    before_errors = errors.dup
+
     log.valid?
 
-    super
+    super(:after_log)
+    errors.merge!(before_errors)
 
     log.errors.each do |error|
       fields = field_mapping_for_errors[error.attribute] || []
@@ -915,11 +919,21 @@ private
 
       fields = field_mapping_for_errors[question_id.to_sym] || []
 
+      if setup_question?(question)
+        fields.each do |field|
+          if errors[field].present?
+            errors.add(field, I18n.t("validations.invalid_option", question: QUESTIONS[field]), category: :setup)
+          end
+        end
+      else
         fields.each do |field|
+          unless errors.any? { |e| fields.include?(e.attribute) }
             errors.add(field, I18n.t("validations.invalid_option", question: QUESTIONS[field]))
           end
         end
       end
+    end
+  end
 
   def validate_relevant_collection_window
     return if saledate.blank? || bulk_upload.form.blank?

spec/factories/sales_log.rb

@@ -8,6 +8,7 @@ FactoryBot.define do
       purchid { "PC123" }
       ownershipsch { 2 }
       type { 8 }
+      jointpur { 2 }
       saledate { Time.utc(2023, 2, 2, 10, 36, 49) }
     end
     trait :shared_ownership do

spec/fixtures/files/completed_2022_23_sales_bulk_upload.csv

@@ -0,0 +1,119 @@
+Question,What is the purchaser code?,What is the day of the sale completion date? - DD,What is the month of the sale completion date? - MM,What is the year of the sale completion date? - YY,[BLANK],Was the buyer interviewed for any of the answers you will provide on this log?,Age of Buyer 1,Age of Buyer 2 or Person 2,Age of Person 3,Age of Person 4,Age of Person 5,Age of Person 6,Gender identity of Buyer 1,Gender identity of Buyer 2 or Person 2,Gender identity of Person 3,Gender identity of Person 4,Gender identity of Person 5,Gender identity of Person 6,Person 2's relationship to lead tenant,Person 3's relationship to lead tenant,Person 4's relationship to lead tenant,Person 5's relationship to lead tenant,Person 6's relationship to lead tenant,Working situation of Buyer 1,Working situation of Buyer 2 or Person 2,Working situation of Person 3,Working situation of Person 4,Working situation of Person 5,Working situation of Person 6,What is the buyer 1's ethnic group?,What is buyer 1's nationality?,What is buyer 1's gross annual income?,What is buyer 2's gross annual income?,Was buyer 1's income used for a mortgage application?,Was buyer 2's income used for a mortgage application?,"What is the total amount the buyers had in savings before they paid any deposit for the property?
+
+To the nearest £10",Have any of the buyers previously owned a property?,[BLANK],What was buyer 1's previous tenure?,What is the local authority of buyer 1's last settled home,Part 1 of postcode of buyer 1's last settled home,Part 2 of postcode of buyer 1's last settled home,Do you know the postcode of buyer 1's last settled home?,Was the buyer registered with their PRP (HA)?,Was the buyer registered with the local authority?,Was the buyer registered with a Help to Buy agent?,Was the buyer registered with another PRP (HA)?,Does anyone in the household consider themselves to have a disability?,Does anyone in the household use a wheelchair?,How many bedrooms does the property have?,What type of unit is the property?,Which type of building is the property?,What is the local authority of the property?,Part 1 of postcode of property,Part 2 of postcode of property,Is the property built or adapted to wheelchair-user standards?,What is the type of shared ownership sale?,"Is this a resale?
+
+Shared ownership","What is the day of the practical completion or handover date? - DD
+
+Shared ownership","What is the month of the practical completion or handover date? - MM
+
+Shared ownership","What is the year of the practical completion or handover date? - YY
+
+Shared ownership","What is the day of the exchange of contracts date? - DD
+
+Shared ownership","What is the month of the exchange of contracts date? - MM
+
+Shared ownership","What is the year of the exchange of contracts date? - YY
+
+Shared ownership","Was the household re-housed under a local authority nominations agreement?
+
+Shared ownership","How many bedrooms did the buyer's previous property have?
+
+Shared ownership","What was the type of the buyer's previous property?
+
+Shared ownership","What was the full purchase price?
+
+Shared ownership","What was the initial percentage equity stake purchased?
+
+Shared ownership","What is the mortgage amount?
+
+Shared ownership","Does this include any extra borrowing?
+
+Shared ownership","How much was the cash deposit paid on the property?
+
+Shared ownership","How much cash discount was given through Social Homebuy?
+
+Shared ownership","What is the basic monthly rent?
+
+Shared ownership","What are the total monthly leasehold charges for the property?
+
+Shared ownership",What is the type of discounted ownership sale?,"What was the full purchase price?
+
+Discounted ownership","What was the amount of any loan, grant, discount or subsidy given?
+
+Discounted ownership","What was the percentage discount?
+
+Discounted ownership","What is the mortgage amount?
+
+Discounted ownership","Does this include any extra borrowing?
+
+Discounted ownership","How much was the cash deposit paid on the property?
+
+Discounted ownership","What are the total monthly leasehold charges for the property?
+
+Discounted ownership",What is the type of outright sale?,"What is the 'other' type of outright sale?
+
+Outright sale",[BLANK],"What is the full purchase price?
+
+Outright sale","What is the mortgage amount?
+
+Outright sale","Does this include any extra borrowing?
+
+Outright sale","How much was the cash deposit paid on the property?
+
+Outright sale","What are the total monthly leasehold charges for the property?
+
+Outright sale","Which organisation owned this property before the sale?
+
+Organisation's CORE ID",Username,BLANK,Has the buyer ever served in the UK Armed Forces and for how long?,[BLANK],Are any of the buyers a spouse or civil partner of a UK Armed Forces regular who died in service within the last 2 years?,"What is the name of the mortgage lender?
+
+Shared ownership","What is the name of the 'other' mortgage lender?
+
+Shared ownership","What is the name of the mortgage lender?
+
+Discounted ownership","What is the name of the 'other' mortgage lender?
+
+Discounted ownership","What is the name of the mortgage lender?
+
+Outright sale","What is the name of the 'other' mortgage lender?
+
+Outright sale",Were the buyers receiving any of these housing-related benefits immediately before buying this property?,"What is the length of the mortgage in years?
+
+Shared ownership","What is the length of the mortgage in years?
+
+Discounted ownership","What is the length of the mortgage in years?
+
+Outright sale","How long have the buyers been living in the property before the purchase?
+
+Discounted ownership",Are there more than two joint purchasers of this property?,"How long have the buyers been living in the property before the purchase?
+
+Shared ownership",Is this a staircasing transaction?,Data Protection question,Was this purchase made through an ownership scheme?,"Is the buyer a company?
+
+Outright sale",Will the buyers live in the property?,Is this a joint purchase?,Will buyer 1 live in the property?,Will buyer 2 live in the property?,"Besides the buyers, how many people live in the property?","What percentage of the property has been bought in this staircasing transaction?
+
+Shared ownership","What percentage of the property does the buyer now own in total?
+
+Shared ownership","What was the rent type of the buyer's previous property?
+
+Shared ownership","Was a mortgage used for the purchase of this property?
+
+Shared ownership","Was a mortgage used for the purchase of this property?
+
+Discounted ownership","Was a mortgage used for the purchase of this property?
+
+Outright sale"
+Values,Max 9 digits,1 - 31,1 - 12,19 - 23,,1 or null,"15 - 110
+or R",1 - 110 or R,,,,,"M, F, X or R",,,,,,"P, C, X or R",,,,,0 - 10,,,,,,1 - 19,"12 -13, 17 -19",0 - 99999,,1 or 2,1 or 2,0 - 999990,1 - 3,,1 - 7 or 9,ONS CODE - E + 9 digits,XXX(X),XXX,1 or null,,,,,1 - 3,1 - 3,1 - 9,1 - 4 or 9,1 or 2,ONS CODE E + 9 digits,XXX(X),XXX,1 - 3,"2, 16, 18, 24, 28 or 30-31",1 or 2,1 - 31,1 - 12,19 - 23,1 - 31,1 - 12,19 - 23,1 - 3,1 - 9,1 - 4 or 9,0 - 999999,0 - 100,0 - 999999,1 - 3,0 - 999999,,0 - 999.99,,"8, 9, 14, 21, 22, 27 or 29",0 - 999999,,0 - 100,0 - 999999,1 - 3,0 - 999999,0 - 999.99,10 or 12,,,0 - 999999,,1-3,0 - 999999,0-999.99,Up to 7 digits,Username of CORE account this sales log should be assigned to,,3 - 8,,4 - 7,1 - 40,,1 - 40,,1 - 40,,1 - 4, Integer <=60, Integer <=60, Integer <=60, Integer <=80,1 - 3, Integer <=80,1 - 3,1,1 - 3,1 - 2,1 - 2,1 - 2,1 - 2,1 - 2,0 - 5,1 - 100,1 - 100,1-3 or 9-10,1 - 2,1 - 2,1 - 2
+Can be Null?,No,,,,,No,No,"If fields 14, 19 and 25 are all also null","If fields 15, 20 and 26 are all also null","If fields 16, 21 and 27 are all also null","If fields 17, 22 and 28 are all also null","If fields 18, 23 and 29 are all also null",No,"If fields 8, 19 and 25 are all also null","If fields 9, 20 and 26 are also null","If fields 10, 21 and 27 are all also null","If fields 11, 22 and 28 are all also null","If fields 12, 23 and 29 are all also null","If fields 8, 14 and 25 are all also null","If fields 9, 15 and 26 are all also null","If fields 10, 16 and 27 are all also null","If fields 11, 17 and 28 are all also null","If fields 12, 18 and 29 are all also null",If field 6 = 1,"If fields 8, 14 and 19 are all also null","If fields 9, 15 and 20 are all also null","If fields 10, 16 and 21 are all also null","If fields 11, 17 and 22 are all also null","If fields 12, 18 and 23 are all also null",If field 6 =  1,,,If field 116 = 2,If field 32  is null,If field 116 = 2,If field 6 = 1,,,If field 6 = 1,No,If field 43 = 1,,If fields 41 and 42 BOTH have valid entries,Yes,,,,If field 6 = 1,,No,,,,,,,If field 113 = 2 or 3,,,,,,,,,"If field 113 = 2 or 3
+OR
+field 39 = 3 - 7 or 9",,If field 113 = 2 or 3,,,,,"If field 57 is null, 2, 16, 24 or 28",If field 113 = 2 or 3,,If field 113 = 1 or 3,If field 76 is null,"If field 76 is null, 9 or 14","If field 76 is null, 8, 21 or 22",If field 113 = 1 or 3,,,,If field 113 = 1 or 2,If field 84 is null or 10,,If field 113 = 1 or 2,,,,,No,Yes,,No,,No,If field 113 = 2 or 3,"If field 113 = 2 or 3
+OR
+If field 98 is not 40",If field 113 = 1 or 3,"If field 113 = 1 or 3
+OR
+If field 100 is not 40",If field 113 = 1 or 2,"If field 113 = 1 or 2
+OR
+If field 102 is not 40",No,If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2,If field 113 = 1 or 3,If field 116 = 2,If field 113 = 2 or 3,If field 113 = 2 or 3,No,No,If field 113 = 1 or 2,If field 113 = 1 or 2,No,No,If field 116 = 2,No,If field 113 = 2 or 3,If field 113 = 2 or 3,"If field 113 = 1 or 2
+OR
+If field 39 = 3 - 9",If field 113 = 2 or 3,If field 113 = 1 or 3,If field 113 = 1 or 2
+Bulk upload format and duplicate check,Yes,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Field number,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125
+,22 test BU,22,2,23,,1,32,32,,,,,M,F,,,,,R,,,,,1,2,,,,,12,18,30000,15000,1,1,20000,3,,1,E09000008,A1,1AA,1,,1,1,,3,3,2,1,1,E09000008,CR0,4BB,3,2,2,23,3,22,30,3,22,3,1,1,250000,25,42500,3,20000,,800,200,,,,,,,,,,,,,,,,,3,,,3,,5,1,,,,,,4,20,,,,2,5,1,1,1,,1,1,1,1,0,10,10,1,1,,

spec/services/bulk_upload/sales/log_creator_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe BulkUpload::Sales::LogCreator do
   let(:user) { create(:user, organisation: owning_org) }
 
   let(:bulk_upload) { create(:bulk_upload, :sales, user:) }
-  let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+  let(:path) { file_fixture("completed_2022_23_sales_bulk_upload.csv") }
 
   describe "#call" do
     context "when a valid csv with new log" do

spec/services/bulk_upload/sales/validator_spec.rb

@@ -3,7 +3,9 @@ require "rails_helper"
 RSpec.describe BulkUpload::Sales::Validator do
   subject(:validator) { described_class.new(bulk_upload:, path:) }
 
-  let(:bulk_upload) { create(:bulk_upload) }
+  let(:user) { create(:user, organisation:) }
+  let(:organisation) { create(:organisation, old_visible_id: "3") }
+  let(:bulk_upload) { create(:bulk_upload, user:) }
   let(:path) { file.path }
   let(:file) { Tempfile.new }
 
@@ -14,6 +16,18 @@ RSpec.describe BulkUpload::Sales::Validator do
       end
     end
 
+    context "when file has too few columns" do
+      before do
+        file.write("a," * 112)
+        file.write("\n")
+        file.rewind
+      end
+
+      it "is not valid" do
+        expect(validator).not_to be_valid
+      end
+    end
+
     context "when file has too many columns" do
       before do
         file.write((%w[a] * 127).join(","))
@@ -28,20 +42,194 @@ RSpec.describe BulkUpload::Sales::Validator do
     context "when incorrect headers"
   end
 
-  context "when a valid csv that contains errors" do
+  describe "#call" do
+    context "when a valid csv" do
+      let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+
+      it "create validation error with correct values" do
+        validator.call
+
+        error = BulkUploadError.find_by(row: "6", field: "field_92", category: "setup")
+
+        expect(error.field).to eql("field_92")
+        expect(error.error).to eql("The owning organisation code is incorrect")
+        expect(error.purchaser_code).to eql("22 test BU")
+        expect(error.row).to eql("6")
+        expect(error.cell).to eql("CO6")
+        expect(error.col).to eql("CO")
+      end
+    end
+
+    context "with unix line endings" do
+      let(:fixture_path) { file_fixture("2022_23_sales_bulk_upload.csv") }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        string = File.read(fixture_path)
+        string.gsub!("\r\n", "\n")
+        file.write(string)
+        file.rewind
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+    end
+
+    context "without headers" do
+      let(:log) { build(:sales_log, :completed) }
+      let(:file) { Tempfile.new }
+      let(:path) { file.path }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "creates validation errors" do
+        expect { validator.call }.to change(BulkUploadError, :count)
+      end
+    end
+  end
+
+  describe "#create_logs?" do
+    context "when all logs are valid" do
+      let(:target_path) { file_fixture("completed_2022_23_sales_bulk_upload.csv") }
+
+      before do
+        target_array = File.open(target_path).readlines
+        target_array[0..118].each do |line|
+          file.write line
+        end
+        file.rewind
+      end
+
+      it "returns truthy" do
+        validator.call
+        expect(validator).to be_create_logs
+      end
+    end
+
+    context "when there is an invalid log" do
       let(:path) { file_fixture("2022_23_sales_bulk_upload.csv") }
 
-    it "persists bulk upload errors" do
-      expect {
+      it "returns falsey" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a log is not valid?" do
+      let(:log_1) { build(:sales_log, :completed, created_by: user) }
+      let(:log_2) { build(:sales_log, :completed, created_by: user) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0, overrides: { organisation_id: "random" }).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns false" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when all logs valid?" do
+      let(:log_1) { build(:sales_log, :completed, created_by: user) }
+      let(:log_2) { build(:sales_log, :completed, created_by: user) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns true" do
+        validator.call
+        expect(validator).to be_create_logs
+      end
+    end
+
+    context "when a single log wants to block log creation" do
+      let(:unaffiliated_org) { create(:organisation) }
+
+      let(:log_1) { build(:sales_log, :completed, created_by: user, owning_organisation: unaffiliated_org) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "will not create logs" do
+        validator.call
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a log has incomplete setup secion" do
+      let(:log) { build(:sales_log, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+
+      before do
+        file.write(BulkUpload::LogToCsv.new(log:, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+        file.close
+      end
+
+      it "returns false" do
         validator.call
-      }.to change(BulkUploadError, :count).by(1)
+        expect(validator).not_to be_create_logs
+      end
+    end
+
+    context "when a column has error rate below absolute threshold" do
+      context "when a column is over 60% error threshold" do
+        let(:log_1) { build(:sales_log, :completed, created_by: user) }
+        let(:log_2) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_3) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_4) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_5) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+
+        before do
+          file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_3, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_4, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_5, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.close
         end
 
-    it "populates purchaser_code" do
+        it "returns true" do
           validator.call
+          expect(validator).to be_create_logs
+        end
+      end
+
+      context "when a column is under 60% error threshold" do
+        let(:log_1) { build(:sales_log, :completed, created_by: user) }
+        let(:log_2) { build(:sales_log, :completed, created_by: user) }
+        let(:log_3) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_4) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
+        let(:log_5) { build(:sales_log, :in_progress, created_by: user, saledate: Time.zone.local(2022, 5, 1)) }
 
-      error = BulkUploadError.last
-      expect(error.purchaser_code).to eql("1")
+        before do
+          file.write(BulkUpload::LogToCsv.new(log: log_1, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_2, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_3, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_4, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.write(BulkUpload::LogToCsv.new(log: log_5, line_ending: "\r\n", col_offset: 0).to_2022_sales_csv_row)
+          file.close
+        end
+
+        it "returns true" do
+          validator.call
+          expect(validator).to be_create_logs
+        end
+      end
     end
   end
 end

spec/support/bulk_upload/log_to_csv.rb

@@ -310,7 +310,7 @@ class BulkUpload::LogToCsv
       log.deposit, # 90
       log.mscharge,
 
-      log.owning_organisation&.old_visible_id,
+      overrides[:organisation_id] || log.owning_organisation&.old_visible_id,
       log.created_by&.email,
       nil,
       hhregres,