From 1bdcc767b521ef8c4934b8af2bc45e545e018011 Mon Sep 17 00:00:00 2001
From: Rachael Booth <Rachael.Booth@softwire.com>
Date: Tue, 20 Feb 2024 17:46:38 +0000
Subject: [PATCH 1/6] CLDC-3227: Add validations to reasonother content (#2236)

* CLDC-3227: Add soft validation for reason other when it's likely a standard category

* CLDC-3227: Add hard validation for homeless indicators in other reason for leaving last settled home

* CLDC-3227: Update hard validation regex to ignore all non alphabet characters surrounding a match

* CLDC-3227: Add unit tests

* Refactor household validations spec to remove dependancy on fake 2021 form

* Add card number to question

* Move informative text to translations as well

* CLDC-3227: Allow neighbouring non-alphabet characters in soft validation check

* Remove debug lines

* Remove blank line

* Remove bonus whitespace

* Deal with appearance in csv export

* Only do regexp.union once

* Fix informative text in value check
---
 .../lettings/pages/reasonother_value_check.rb |  23 ++++
 .../questions/reasonother_value_check.rb      |  14 ++
 .../subsections/household_situation.rb        |   1 +
 .../validations/household_validations.rb      |  20 +++
 app/models/validations/soft_validations.rb    |  38 +++++
 app/services/csv/lettings_log_csv_service.rb  |   2 +-
 config/locales/en.yml                         |   6 +
 ...easonother_value_check_to_lettings_logs.rb |   5 +
 db/schema.rb                                  |   3 +-
 spec/factories/lettings_log.rb                |   1 +
 .../lettings_log_csv_export_codes_23.csv      |   4 +-
 .../lettings_log_csv_export_codes_24.csv      |   4 +-
 .../lettings_log_csv_export_labels_23.csv     |   4 +-
 .../lettings_log_csv_export_labels_24.csv     |   4 +-
 .../subsections/household_situation_spec.rb   |  86 +++++++++---
 .../validations/household_validations_spec.rb | 130 ++++++++++++------
 .../validations/soft_validations_spec.rb      |  38 +++++
 17 files changed, 310 insertions(+), 73 deletions(-)
 create mode 100644 app/models/form/lettings/pages/reasonother_value_check.rb
 create mode 100644 app/models/form/lettings/questions/reasonother_value_check.rb
 create mode 100644 db/migrate/20240209153215_add_reasonother_value_check_to_lettings_logs.rb

diff --git a/app/models/form/lettings/pages/reasonother_value_check.rb b/app/models/form/lettings/pages/reasonother_value_check.rb
new file mode 100644
index 000000000..ccda997a7
--- /dev/null
+++ b/app/models/form/lettings/pages/reasonother_value_check.rb
@@ -0,0 +1,23 @@
+class Form::Lettings::Pages::ReasonotherValueCheck < ::Form::Page
+  def initialize(id, hsh, subsection)
+    super
+    @id = "reasonother_value_check"
+    @depends_on = [{ "reasonother_might_be_existing_category?" => true }]
+    @title_text = {
+      "translation" => "soft_validations.reasonother.title_text",
+      "arguments" => [{ "key" => "reasonother", "i18n_template" => "reasonother" }],
+    }
+    @informative_text = {
+      "translation" => "soft_validations.reasonother.informative_text",
+      "arguments" => [],
+    }
+  end
+
+  def questions
+    @questions ||= [Form::Lettings::Questions::ReasonotherValueCheck.new(nil, nil, self)]
+  end
+
+  def interruption_screen_question_ids
+    %w[reason reasonother]
+  end
+end
diff --git a/app/models/form/lettings/questions/reasonother_value_check.rb b/app/models/form/lettings/questions/reasonother_value_check.rb
new file mode 100644
index 000000000..865f38764
--- /dev/null
+++ b/app/models/form/lettings/questions/reasonother_value_check.rb
@@ -0,0 +1,14 @@
+class Form::Lettings::Questions::ReasonotherValueCheck < ::Form::Question
+  def initialize(id, hsh, page)
+    super
+    @id = "reasonother_value_check"
+    @check_answer_label = "Reason other confirmation"
+    @header = "Are you sure this doesn’t fit an existing category?"
+    @type = "interruption_screen"
+    @check_answers_card_number = 0
+    @answer_options = ANSWER_OPTIONS
+    @hidden_in_check_answers = { "depends_on" => [{ "reasonother_value_check" => 0 }, { "reasonother_value_check" => 1 }] }
+  end
+
+  ANSWER_OPTIONS = { "0" => { "value" => "Yes" }, "1" => { "value" => "No" } }.freeze
+end
diff --git a/app/models/form/lettings/subsections/household_situation.rb b/app/models/form/lettings/subsections/household_situation.rb
index 9db7c1f04..6646d7230 100644
--- a/app/models/form/lettings/subsections/household_situation.rb
+++ b/app/models/form/lettings/subsections/household_situation.rb
@@ -12,6 +12,7 @@ class Form::Lettings::Subsections::HouseholdSituation < ::Form::Subsection
       Form::Lettings::Pages::TimeOnWaitingList.new(nil, nil, self),
       Form::Lettings::Pages::ReasonForLeavingLastSettledHome.new(nil, nil, self),
       Form::Lettings::Pages::ReasonForLeavingLastSettledHomeRenewal.new(nil, nil, self),
+      (Form::Lettings::Pages::ReasonotherValueCheck.new(nil, nil, self) if form.start_year_after_2024?),
       Form::Lettings::Pages::PreviousHousingSituation.new(nil, nil, self),
       Form::Lettings::Pages::PreviousHousingSituationRenewal.new(nil, nil, self),
       Form::Lettings::Pages::Homelessness.new("homelessness", nil, self),
diff --git a/app/models/validations/household_validations.rb b/app/models/validations/household_validations.rb
index cd15d09cf..c7e034e94 100644
--- a/app/models/validations/household_validations.rb
+++ b/app/models/validations/household_validations.rb
@@ -9,6 +9,20 @@ module Validations::HouseholdValidations
     end
   end
 
+  PHRASES_INDICATING_HOMELESSNESS = [
+    "Homeless",
+    "Homelessness",
+    "Temporary accommodation",
+    "Temp accommodation",
+    "TA",
+    "Sleeping rough",
+    "Rough sleeping",
+  ].freeze
+
+  PHRASES_INDICATING_HOMELESSNESS_REGEX = Regexp.union(
+    PHRASES_INDICATING_HOMELESSNESS.map { |phrase| Regexp.new("\\A[^[:alpha:]]*#{phrase}[^[:alpha:]]*\\Z", Regexp::IGNORECASE) },
+  )
+
   def validate_reason_for_leaving_last_settled_home(record)
     if record.reason == 32 && record.underoccupation_benefitcap != 4
       record.errors.add :underoccupation_benefitcap, I18n.t("validations.household.underoccupation_benefitcap.dont_know_required")
@@ -20,6 +34,12 @@ module Validations::HouseholdValidations
       record.errors.add :referral, I18n.t("validations.household.referral.reason_permanently_decanted")
       record.errors.add :reason, I18n.t("validations.household.reason.not_internal_transfer")
     end
+
+    return unless record.form.start_year_after_2024?
+
+    if record.reason == 20 && PHRASES_INDICATING_HOMELESSNESS_REGEX.match?(record.reasonother)
+      record.errors.add :reason, I18n.t("validations.household.reason.other_not_settled")
+    end
   end
 
   def validate_armed_forces(record)
diff --git a/app/models/validations/soft_validations.rb b/app/models/validations/soft_validations.rb
index 92685afe0..74afef27d 100644
--- a/app/models/validations/soft_validations.rb
+++ b/app/models/validations/soft_validations.rb
@@ -133,6 +133,44 @@ module Validations::SoftValidations
     weekly_value(supcharg) > max
   end
 
+  PHRASES_LIKELY_TO_INDICATE_EXISTING_REASON_CATEGORY = [
+    "Decant",
+    "Decanted",
+    "Refugee",
+    "Asylum",
+    "Ukraine",
+    "Ukrainian",
+    "Army",
+    "Military",
+    "Domestic Abuse",
+    "Domestic Violence",
+    "DA",
+    "DV",
+    "Relationship breakdown",
+    "Overcrowding",
+    "Overcrowded",
+    "Too small",
+    "More space",
+    "Bigger property",
+    "Damp",
+    "Mould",
+    "Fire",
+    "Repossession",
+    "Death",
+    "Deceased",
+    "Passed away",
+    "Prison",
+    "Hospital",
+  ].freeze
+
+  PHRASES_LIKELY_TO_INDICATE_EXISTING_REASON_CATEGORY_REGEX = Regexp.union(
+    PHRASES_LIKELY_TO_INDICATE_EXISTING_REASON_CATEGORY.map { |phrase| Regexp.new("\\b[^[:alpha]]*#{phrase}[^[:alpha:]]*\\b", Regexp::IGNORECASE) },
+  )
+
+  def reasonother_might_be_existing_category?
+    PHRASES_LIKELY_TO_INDICATE_EXISTING_REASON_CATEGORY_REGEX.match?(reasonother)
+  end
+
 private
 
   def details_known_or_lead_tenant?(tenant_number)
diff --git a/app/services/csv/lettings_log_csv_service.rb b/app/services/csv/lettings_log_csv_service.rb
index e9ebd875a..e67363188 100644
--- a/app/services/csv/lettings_log_csv_service.rb
+++ b/app/services/csv/lettings_log_csv_service.rb
@@ -296,7 +296,7 @@ module Csv
       "letting_allocation_unknown" => %w[letting_allocation_none],
     }.freeze
 
-    SUPPORT_ONLY_ATTRIBUTES = %w[net_income_value_check first_time_property_let_as_social_housing postcode_known is_la_inferred totchild totelder totadult net_income_known previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old la prevloc updated_by_id bulk_upload_id uprn_confirmed].freeze
+    SUPPORT_ONLY_ATTRIBUTES = %w[net_income_value_check first_time_property_let_as_social_housing postcode_known is_la_inferred totchild totelder totadult net_income_known previous_la_known is_previous_la_inferred age1_known age2_known age3_known age4_known age5_known age6_known age7_known age8_known details_known_2 details_known_3 details_known_4 details_known_5 details_known_6 details_known_7 details_known_8 wrent wscharge wpschrge wsupchrg wtcharge wtshortfall rent_value_check old_form_id old_id retirement_value_check tshortfall_known pregnancy_value_check hhtype new_old la prevloc updated_by_id bulk_upload_id uprn_confirmed reasonother_value_check].freeze
 
     def lettings_log_attributes
       ordered_questions = FormHandler.instance.ordered_lettings_questions_for_all_years
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 90962e08d..b322f9759 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -543,6 +543,7 @@ en:
       reason:
         not_internal_transfer: "Answer cannot be ‘permanently decanted from another property owned by this landlord’ as you told us the source of referral for this tenancy was not an internal transfer"
         renewal_reason_needed: 'The reason for leaving must be "End of assured shorthold tenancy - no fault" or "End of fixed term tenancy - no fault" if the letting is a renewal'
+        other_not_settled: "Please give the reason for the tenant leaving their last settled home. This is where they were living before they became homeless, were living in temporary accommodation or sleeping rough"
       condition_effects:
         no_choices: "You cannot answer this question as you told us nobody in the household has a physical or mental health condition (or other illness) expected to last 12 months or more"
       postcode:
@@ -740,6 +741,11 @@ Make sure these answers are correct."
     deposit_and_mortgage:
       title_text: "You told us the mortgage amount was %{mortgage}, the cash deposit was %{deposit} and the discount was %{discount}."
       hint_text: "We would expect the mortgage amount and the deposit added together to be the same as the purchase price minus the discount."
+    reasonother:
+      title_text: "You told us that the tenant’s main reason for leaving their last settled home was %{reasonother}"
+      informative_text: "The reason you have entered looks very similar to one of the existing response categories.
+                         Please check the categories and select the appropriate one.
+                         If the existing categories are not suitable, please confirm here to move onto the next question."
 
   devise:
     email:
diff --git a/db/migrate/20240209153215_add_reasonother_value_check_to_lettings_logs.rb b/db/migrate/20240209153215_add_reasonother_value_check_to_lettings_logs.rb
new file mode 100644
index 000000000..54486b3ae
--- /dev/null
+++ b/db/migrate/20240209153215_add_reasonother_value_check_to_lettings_logs.rb
@@ -0,0 +1,5 @@
+class AddReasonotherValueCheckToLettingsLogs < ActiveRecord::Migration[7.0]
+  def change
+    add_column :lettings_logs, :reasonother_value_check, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 658e82190..6959c4bf3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_01_30_084707) do
+ActiveRecord::Schema[7.0].define(version: 2024_02_09_153215) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -305,6 +305,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_01_30_084707) do
     t.integer "duplicate_set_id"
     t.integer "nationality_all"
     t.integer "nationality_all_group"
+    t.integer "reasonother_value_check"
     t.integer "accessible_register"
     t.index ["bulk_upload_id"], name: "index_lettings_logs_on_bulk_upload_id"
     t.index ["created_by_id"], name: "index_lettings_logs_on_created_by_id"
diff --git a/spec/factories/lettings_log.rb b/spec/factories/lettings_log.rb
index 23ba06bf1..6f4cf951c 100644
--- a/spec/factories/lettings_log.rb
+++ b/spec/factories/lettings_log.rb
@@ -10,6 +10,7 @@ FactoryBot.define do
       renewal { 0 }
       needstype { 1 }
       rent_type { 1 }
+      declaration { 1 }
     end
     trait :in_progress do
       status { 1 }
diff --git a/spec/fixtures/files/lettings_log_csv_export_codes_23.csv b/spec/fixtures/files/lettings_log_csv_export_codes_23.csv
index 873c703b0..086f250e0 100644
--- a/spec/fixtures/files/lettings_log_csv_export_codes_23.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_codes_23.csv
@@ -1,2 +1,2 @@
-id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,nationality_all,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,2,1,,2,HIJKLMN,ABCDEFG,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,1,4,,1,4,0,0,2,35,,F,0,2,,13,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,nationality_all,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,reasonother_value_check,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,2,1,,2,HIJKLMN,ABCDEFG,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,1,4,,1,4,0,0,2,35,,F,0,2,,13,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_codes_24.csv b/spec/fixtures/files/lettings_log_csv_export_codes_24.csv
index 832ceed45..e752f4237 100644
--- a/spec/fixtures/files/lettings_log_csv_export_codes_24.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_codes_24.csv
@@ -1,2 +1,2 @@
-id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,declaration,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,nationality_all,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,accessible_register,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2024-04-01T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,2,1,,2,HIJKLMN,ABCDEFG,1,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,4,,1,4,0,0,2,35,,F,0,2,13,,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,0,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,declaration,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,nationality_all,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,reasonother_value_check,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,accessible_register,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2024-04-01T00:00:00+01:00,1,,,2023,DLUHC,DLUHC,1,7,0,2023-11-26,2,2,1,,2,HIJKLMN,ABCDEFG,1,1,0,,,fake address,,London,,NW9 5LL,false,Barnet,E09000003,0,2,6,2,2,7,1,1,3,2023-11-24,,,1,2023-11-25,,3,1,4,,2,,4,,1,4,0,0,2,35,,F,0,2,13,,0,0,P,32,M,6,1,R,-9,R,10,0,R,-9,R,10,,,,,,,,,,,,,,,,,,,,,1,4,1,2,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,2,7,4,,,6,2,1,0,TN23 6LZ,1,false,Ashford,E07000105,1,0,1,0,0,0,0,0,1,0,,2,,0,0,268,1,,6,1,1,,0,2,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,1,0,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_labels_23.csv b/spec/fixtures/files/lettings_log_csv_export_labels_23.csv
index a9c491ed9..4f2c43d5d 100644
--- a/spec/fixtures/files/lettings_log_csv_export_labels_23.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_labels_23.csv
@@ -1,2 +1,2 @@
-id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,nationality_all,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,Affordable Rent,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,,Yes,4,0,0,2,35,,Female,White,Irish,,Tenant prefers not to say,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,declaration,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,nationality_all,national,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,reasonother_value_check,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2023-11-26T00:00:00+00:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,Affordable Rent,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,Yes,4,,Yes,4,0,0,2,35,,Female,White,Irish,,Tenant prefers not to say,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/fixtures/files/lettings_log_csv_export_labels_24.csv b/spec/fixtures/files/lettings_log_csv_export_labels_24.csv
index 0e345b837..2d15ea4b3 100644
--- a/spec/fixtures/files/lettings_log_csv_export_labels_24.csv
+++ b/spec/fixtures/files/lettings_log_csv_export_labels_24.csv
@@ -1,2 +1,2 @@
-id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,declaration,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,nationality_all,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,accessible_register,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
-,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2024-04-01T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,Affordable Rent,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,4,,Yes,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,No,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
+id,status,duplicate_set_id,created_by,is_dpo,created_at,updated_by,updated_at,creation_method,old_id,old_form_id,collection_start_year,owning_organisation_name,managing_organisation_name,needstype,lettype,renewal,startdate,renttype,renttype_detail,irproduct,irproduct_other,lar,tenancycode,propcode,declaration,postcode_known,uprn_known,uprn,uprn_confirmed,address_line1,address_line2,town_or_city,county,postcode_full,is_la_inferred,la_label,la,first_time_property_let_as_social_housing,unitletas,rsnvac,newprop,offered,unittype_gn,builtype,wchair,beds,voiddate,vacdays,void_date_value_check,majorrepairs,mrcdate,major_repairs_date_value_check,joint,startertenancy,tenancy,tenancyother,tenancylength,sheltered,hhmemb,pregnancy_value_check,refused,hhtype,totchild,totelder,totadult,age1,retirement_value_check,sex1,ethnic_group,ethnic,national,nationality_all,ecstat1,details_known_2,relat2,age2,sex2,ecstat2,details_known_3,relat3,age3,sex3,ecstat3,details_known_4,relat4,age4,sex4,ecstat4,details_known_5,relat5,age5,sex5,ecstat5,details_known_6,relat6,age6,sex6,ecstat6,details_known_7,relat7,age7,sex7,ecstat7,details_known_8,relat8,age8,sex8,ecstat8,armedforces,leftreg,reservist,preg_occ,housingneeds,housingneeds_type,housingneeds_a,housingneeds_b,housingneeds_c,housingneeds_f,housingneeds_g,housingneeds_h,housingneeds_other,illness,illness_type_4,illness_type_5,illness_type_2,illness_type_6,illness_type_7,illness_type_3,illness_type_9,illness_type_8,illness_type_1,illness_type_10,layear,waityear,reason,reasonother,reasonother_value_check,prevten,new_old,homeless,ppcodenk,ppostcode_full,previous_la_known,is_previous_la_inferred,prevloc_label,prevloc,reasonpref,rp_homeless,rp_insan_unsat,rp_medwel,rp_hardship,rp_dontknow,cbl,cap,chr,accessible_register,letting_allocation_none,referral,referral_value_check,net_income_known,incref,earnings,incfreq,net_income_value_check,hb,has_benefits,benefits,household_charge,nocharge,period,is_carehome,chcharge,wchchrg,carehome_charges_value_check,brent,wrent,rent_value_check,scharge,wscharge,pscharge,wpschrge,supcharg,wsupchrg,tcharge,wtcharge,scharge_value_check,pscharge_value_check,supcharg_value_check,hbrentshortfall,tshortfall_known,tshortfall,wtshortfall,scheme_code,scheme_service_name,scheme_sensitive,SCHTYPE,scheme_registered_under_care_act,scheme_owning_organisation_name,scheme_primary_client_group,scheme_has_other_client_group,scheme_secondary_client_group,scheme_support_type,scheme_intended_stay,scheme_created_at,location_code,location_postcode,location_name,location_units,location_type_of_unit,location_mobility_type,location_local_authority,location_startdate
+,completed,,s.port@jeemayle.com,false,2023-11-26T00:00:00+00:00,,2024-04-01T00:00:00+01:00,single log,,,2023,DLUHC,DLUHC,General needs,Affordable rent general needs local authority,No,2023-11-26,Affordable Rent,Affordable Rent,Rent to Buy,,No,HIJKLMN,ABCDEFG,Yes,Yes,No,,,fake address,,London,,NW9 5LL,No,Barnet,E09000003,No,Affordable rent basis,Tenant abandoned property,No,2,House,Purpose built,Yes,3,2023-11-24,,,Yes,2023-11-25,,Don’t know,Yes,Assured Shorthold Tenancy (AST) – Fixed term,,2,,4,,Yes,4,0,0,2,35,,Female,White,Irish,Tenant prefers not to say,,Other,Yes,Partner,32,Male,Not seeking work,No,Prefers not to say,Not known,Prefers not to say,Prefers not to say,Yes,Person prefers not to say,Not known,Person prefers not to say,Person prefers not to say,,,,,,,,,,,,,,,,,,,,,Yes – the person is a current or former regular,No – they left up to and including 5 years ago,Yes,No,Yes,Fully wheelchair accessible housing,Yes,No,No,No,No,No,No,Yes,No,No,Yes,No,No,No,No,No,No,No,Less than 1 year,1 year but under 2 years,Loss of tied accommodation,,,Other supported housing,2,No,Yes,TN23 6LZ,Yes,No,Ashford,E07000105,Yes,,Yes,,,,No,No,Yes,No,,Tenant applied directly (no referral or nomination),,Yes,No,268,Weekly,,Universal Credit housing element,Yes,All,,No,Every 2 weeks,,,,,200.0,100.0,,50.0,25.0,40.0,20.0,35.0,17.5,325.0,162.5,,,,Yes,Yes,12.0,6.0,,,,,,,,,,,,,,,,,,,,
diff --git a/spec/models/form/lettings/subsections/household_situation_spec.rb b/spec/models/form/lettings/subsections/household_situation_spec.rb
index 803f873f8..dd6f20d89 100644
--- a/spec/models/form/lettings/subsections/household_situation_spec.rb
+++ b/spec/models/form/lettings/subsections/household_situation_spec.rb
@@ -6,33 +6,75 @@ RSpec.describe Form::Lettings::Subsections::HouseholdSituation, type: :model do
   let(:subsection_id) { nil }
   let(:subsection_definition) { nil }
   let(:section) { instance_double(Form::Lettings::Sections::Household) }
+  let(:form) { instance_double(Form) }
+
+  before do
+    allow(section).to receive(:form).and_return(form)
+  end
 
   it "has correct section" do
     expect(household_situation.section).to eq(section)
   end
 
-  it "has correct pages" do
-    expect(household_situation.pages.map(&:id)).to eq(
-      %w[
-        time_lived_in_local_authority
-        time_on_waiting_list
-        reason_for_leaving_last_settled_home
-        reason_for_leaving_last_settled_home_renewal
-        previous_housing_situation
-        previous_housing_situation_renewal
-        homelessness
-        previous_postcode
-        previous_local_authority
-        reasonable_preference
-        reasonable_preference_reason
-        allocation_system
-        referral
-        referral_prp
-        referral_supported_housing
-        referral_supported_housing_prp
-        referral_value_check
-      ],
-    )
+  context "with form year before 2024" do
+    before do
+      allow(form).to receive(:start_year_after_2024?).and_return(false)
+    end
+
+    it "has correct pages" do
+      expect(household_situation.pages.map(&:id)).to eq(
+        %w[
+          time_lived_in_local_authority
+          time_on_waiting_list
+          reason_for_leaving_last_settled_home
+          reason_for_leaving_last_settled_home_renewal
+          previous_housing_situation
+          previous_housing_situation_renewal
+          homelessness
+          previous_postcode
+          previous_local_authority
+          reasonable_preference
+          reasonable_preference_reason
+          allocation_system
+          referral
+          referral_prp
+          referral_supported_housing
+          referral_supported_housing_prp
+          referral_value_check
+        ],
+      )
+    end
+  end
+
+  context "with form year >= 2024" do
+    before do
+      allow(form).to receive(:start_year_after_2024?).and_return(true)
+    end
+
+    it "has correct pages" do
+      expect(household_situation.pages.map(&:id)).to eq(
+        %w[
+          time_lived_in_local_authority
+          time_on_waiting_list
+          reason_for_leaving_last_settled_home
+          reason_for_leaving_last_settled_home_renewal
+          reasonother_value_check
+          previous_housing_situation
+          previous_housing_situation_renewal
+          homelessness
+          previous_postcode
+          previous_local_authority
+          reasonable_preference
+          reasonable_preference_reason
+          allocation_system
+          referral
+          referral_prp
+          referral_supported_housing
+          referral_supported_housing_prp
+          referral_value_check
+        ],
+      )
+    end
   end
 
   it "has the correct id" do
diff --git a/spec/models/validations/household_validations_spec.rb b/spec/models/validations/household_validations_spec.rb
index 849046c65..5a48d4b3f 100644
--- a/spec/models/validations/household_validations_spec.rb
+++ b/spec/models/validations/household_validations_spec.rb
@@ -4,11 +4,15 @@ RSpec.describe Validations::HouseholdValidations do
   subject(:household_validator) { validator_class.new }
 
   let(:validator_class) { Class.new { include Validations::HouseholdValidations } }
-  let(:record) { FactoryBot.create(:lettings_log) }
-  let(:fake_2021_2022_form) { Form.new("spec/fixtures/forms/2021_2022.json") }
+  let(:log_date) { Time.zone.now }
+  let(:record) { FactoryBot.create(:lettings_log, :setup_completed, startdate: log_date) }
 
   before do
-    allow(FormHandler.instance).to receive(:current_lettings_form).and_return(fake_2021_2022_form)
+    Timecop.freeze(log_date + 1)
+  end
+
+  after do
+    Timecop.return
   end
 
   describe "reasonable preference validations" do
@@ -53,6 +57,44 @@ RSpec.describe Validations::HouseholdValidations do
         household_validator.validate_reason_for_leaving_last_settled_home(record)
         expect(record.errors["reasonother"]).to be_empty
       end
+
+      context "when form year is before 2024" do
+        let(:log_date) { Time.zone.local(2024, 1, 1) }
+
+        it "does not validate the content of reasonother for phrases indicating homelessness" do
+          record.reason = 20
+          record.reasonother = "Temp accommodation"
+          household_validator.validate_reason_for_leaving_last_settled_home(record)
+          expect(record.errors["reason"]).to be_empty
+        end
+      end
+
+      context "when form year is >= 2024" do
+        let(:log_date) { Time.zone.local(2024, 4, 1) }
+
+        context "when checking the content of reasonother" do
+          it "validates that the reason doesn't match phrase indicating homelessness" do
+            record.reason = 20
+            record.reasonother = "Temp accommodation"
+            household_validator.validate_reason_for_leaving_last_settled_home(record)
+            expect(record.errors["reason"]).to include(I18n.t("validations.household.reason.other_not_settled"))
+          end
+
+          it "allows reasons that don't exactly match a phrase indicating homelessness" do
+            record.reason = 20
+            record.reasonother = "Not quite homeless but some other reason"
+            household_validator.validate_reason_for_leaving_last_settled_home(record)
+            expect(record.errors["reason"]).to be_empty
+          end
+
+          it "ignores surrounding non-alphabet characters and casing when determining a match" do
+            record.reason = 20
+            record.reasonother = "  0homelessness ! "
+            household_validator.validate_reason_for_leaving_last_settled_home(record)
+            expect(record.errors["reason"]).to include(I18n.t("validations.household.reason.other_not_settled"))
+          end
+        end
+      end
     end
 
     context "when reason is not other" do
@@ -231,6 +273,26 @@ RSpec.describe Validations::HouseholdValidations do
   end
 
   describe "household member validations" do
+    it "validates that the number of household members cannot be less than 1" do
+      record.hhmemb = 0
+      household_validator.validate_numeric_min_max(record)
+      expect(record.errors["hhmemb"])
+        .to include(match I18n.t("validations.numeric.within_range", field: "Number of household members", min: 1, max: 8))
+    end
+
+    it "validates that the number of household members cannot be more than 8" do
+      record.hhmemb = 9
+      household_validator.validate_numeric_min_max(record)
+      expect(record.errors["hhmemb"])
+        .to include(match I18n.t("validations.numeric.within_range", field: "Number of household members", min: 1, max: 8))
+    end
+
+    it "expects that the number of other household members is between the min and max" do
+      record.hhmemb = 5
+      household_validator.validate_numeric_min_max(record)
+      expect(record.errors["hhmemb"]).to be_empty
+    end
+
     it "validates that only 1 partner exists" do
       record.relat2 = "P"
       record.relat3 = "P"
@@ -394,26 +456,6 @@ RSpec.describe Validations::HouseholdValidations do
         expect(record.errors["sex2"]).to be_empty
         expect(record.errors["age2"]).to be_empty
       end
-
-      it "validates that the number of household members cannot be less than 0" do
-        record.hhmemb = -1
-        household_validator.validate_numeric_min_max(record)
-        expect(record.errors["hhmemb"])
-          .to include(match I18n.t("validations.numeric.within_range", field: "Number of Household Members", min: 0, max: 8))
-      end
-
-      it "validates that the number of household members cannot be more than 8" do
-        record.hhmemb = 9
-        household_validator.validate_numeric_min_max(record)
-        expect(record.errors["hhmemb"])
-          .to include(match I18n.t("validations.numeric.within_range", field: "Number of Household Members", min: 0, max: 8))
-      end
-
-      it "expects that the number of other household members is between the min and max" do
-        record.hhmemb = 5
-        household_validator.validate_numeric_min_max(record)
-        expect(record.errors["hhmemb"]).to be_empty
-      end
     end
 
     context "when the household contains a retired female" do
@@ -651,24 +693,30 @@ RSpec.describe Validations::HouseholdValidations do
           .to be_empty
       end
 
-      it "prevten cannot be 3" do
-        record.referral = 1
-        record.prevten = 3
-        household_validator.validate_previous_housing_situation(record)
-        expect(record.errors["prevten"])
-          .to include(match I18n.t("validations.household.prevten.internal_transfer", prevten: ""))
-        expect(record.errors["referral"])
-          .to include(match I18n.t("validations.household.referral.prevten_invalid", prevten: ""))
-      end
-
-      it "prevten cannot be 4, 10, 13, 19, 23, 24, 25, 26, 28, 29" do
-        record.referral = 1
-        record.prevten = 4
-        household_validator.validate_previous_housing_situation(record)
-        expect(record.errors["prevten"])
-          .to include(match I18n.t("validations.household.prevten.internal_transfer", prevten: ""))
-        expect(record.errors["referral"])
-          .to include(match I18n.t("validations.household.referral.prevten_invalid", prevten: ""))
+      [
+        { code: 3, label: "Private sector tenancy" },
+        { code: 4, label: "Tied housing or rented with job" },
+        { code: 7, label: "Direct access hostel" },
+        { code: 10, label: "Hospital" },
+        { code: 13, label: "Children’s home or foster care" },
+        { code: 14, label: "Bed and breakfast" },
+        { code: 19, label: "Rough sleeping" },
+        { code: 23, label: "Mobile home or caravan" },
+        { code: 24, label: "Home Office Asylum Support" },
+        { code: 25, label: "Any other accommodation" },
+        { code: 26, label: "Owner occupation (private)" },
+        { code: 28, label: "Living with friends or family" },
+        { code: 29, label: "Prison or approved probation hostel" },
+      ].each do |prevten|
+        it "prevten cannot be #{prevten[:code]}" do
+          record.referral = 1
+          record.prevten = prevten[:code]
+          household_validator.validate_previous_housing_situation(record)
+          expect(record.errors["prevten"])
+            .to include(match I18n.t("validations.household.prevten.internal_transfer", prevten: prevten[:label]))
+          expect(record.errors["referral"])
+            .to include(match I18n.t("validations.household.referral.prevten_invalid", prevten: ""))
+        end
       end
     end
   end
diff --git a/spec/models/validations/soft_validations_spec.rb b/spec/models/validations/soft_validations_spec.rb
index 8f00799ff..ad87b3c5b 100644
--- a/spec/models/validations/soft_validations_spec.rb
+++ b/spec/models/validations/soft_validations_spec.rb
@@ -1017,4 +1017,42 @@ RSpec.describe Validations::SoftValidations do
       end
     end
   end
+
+  describe "reasonother_might_be_existing_category?" do
+    it "returns true if reasonother is exactly in the 'likely existing category' list" do
+      record.reasonother = "Domestic Abuse"
+
+      expect(record).to be_reasonother_might_be_existing_category
+    end
+
+    it "returns true if any word of reasonother is exactly in the 'likely existing category' list" do
+      record.reasonother = "Was decanted from somewhere"
+
+      expect(record).to be_reasonother_might_be_existing_category
+    end
+
+    it "is not case sensitive when matching" do
+      record.reasonother = "domestic abuse"
+
+      expect(record).to be_reasonother_might_be_existing_category
+    end
+
+    it "returns false if no part of reasonother is in the 'likely existing category' list" do
+      record.reasonother = "other"
+
+      expect(record).not_to be_reasonother_might_be_existing_category
+    end
+
+    it "returns false if match to the 'likely existing category' list is only part of a word" do
+      record.reasonother = "wasdecanted"
+
+      expect(record).not_to be_reasonother_might_be_existing_category
+    end
+
+    it "ignores neighbouring non-alphabet for matching" do
+      record.reasonother = "1Domestic abuse."
+
+      expect(record).to be_reasonother_might_be_existing_category
+    end
+  end
 end

From c9345ecaf153d408e293f72ea599355cfd7f78ad Mon Sep 17 00:00:00 2001
From: natdeanlewissoftwire
 <94526761+natdeanlewissoftwire@users.noreply.github.com>
Date: Wed, 21 Feb 2024 09:23:43 +0000
Subject: [PATCH 2/6] CLDC-3202 Update year filters for 24/25 (#2216)

* feat: replace 21/22 filter with 24/25 filter, add feature toggle so this appears everywhere but prod

* feat: update filters automatically year-on-year

* feat: update filters automatically year-on-year and test

* refactor: lint

* refactor: minor renaming

* feat: time travel for PO

* feat: time travel for PO without breaking tests

* feat: update tests

* Revert time travelling

---------

Co-authored-by: Kat <katrina@kosiak.co.uk>
---
 app/helpers/collection_time_helper.rb |  4 +++
 app/helpers/filters_helper.rb         | 12 ++++++++-
 spec/features/organisation_spec.rb    |  8 +++---
 spec/helpers/filters_helper_spec.rb   | 38 ++++++++++++++++++++++-----
 4 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/app/helpers/collection_time_helper.rb b/app/helpers/collection_time_helper.rb
index 6f8ef62fc..4381aa43a 100644
--- a/app/helpers/collection_time_helper.rb
+++ b/app/helpers/collection_time_helper.rb
@@ -46,6 +46,10 @@ module CollectionTimeHelper
     current_collection_start_date - 1.year
   end
 
+  def archived_collection_start_year
+    current_collection_start_year - 2
+  end
+
   def quarter_for_date(date: Time.zone.now)
     quarters = [
       { quarter: "Q3", cutoff_date: Time.zone.local(2024, 1, 12), start_date: Time.zone.local(2023, 10, 1), end_date: Time.zone.local(2023, 12, 31) },
diff --git a/app/helpers/filters_helper.rb b/app/helpers/filters_helper.rb
index f35caf31f..8841a0d36 100644
--- a/app/helpers/filters_helper.rb
+++ b/app/helpers/filters_helper.rb
@@ -1,4 +1,6 @@
 module FiltersHelper
+  include CollectionTimeHelper
+
   def filter_selected?(filter, value, filter_type)
     return false unless session[session_name_for(filter_type)]
 
@@ -93,7 +95,11 @@ module FiltersHelper
   end
 
   def collection_year_options
-    { "2023": "2023/24", "2022": "2022/23", "2021": "2021/22" }
+    {
+      current_collection_start_year.to_s => year_combo(current_collection_start_year),
+      previous_collection_start_year.to_s => year_combo(previous_collection_start_year),
+      archived_collection_start_year.to_s => year_combo(archived_collection_start_year),
+    }
   end
 
   def filters_applied_text(filter_type)
@@ -174,4 +180,8 @@ private
       end
     end
   end
+
+  def year_combo(year)
+    "#{year}/#{year - 2000 + 1}"
+  end
 end
diff --git a/spec/features/organisation_spec.rb b/spec/features/organisation_spec.rb
index 43c0a9d8e..f9bb8f450 100644
--- a/spec/features/organisation_spec.rb
+++ b/spec/features/organisation_spec.rb
@@ -196,9 +196,9 @@ RSpec.describe "User Features" do
       end
 
       it "can filter lettings logs by year" do
-        check("years-2021-field")
+        check("years-2022-field")
         click_button("Apply filters")
-        expect(page).to have_current_path("/organisations/#{org_id}/lettings-logs?years[]=&years[]=2021&status[]=&needstypes[]=&assigned_to=all&user=&owning_organisation_select=all&owning_organisation=&managing_organisation_select=all&managing_organisation=")
+        expect(page).to have_current_path("/organisations/#{org_id}/lettings-logs?years[]=&years[]=2022&status[]=&needstypes[]=&assigned_to=all&user=&owning_organisation_select=all&owning_organisation=&managing_organisation_select=all&managing_organisation=")
         expect(page).not_to have_link first_log.id.to_s, href: "/lettings-logs/#{first_log.id}"
       end
 
@@ -241,9 +241,9 @@ RSpec.describe "User Features" do
         organisation.sales_logs.map(&:id).each do |sales_log_id|
           expect(page).to have_link sales_log_id.to_s, href: "/sales-logs/#{sales_log_id}"
         end
-        check("years-2021-field")
+        check("years-2022-field")
         click_button("Apply filters")
-        expect(page).to have_current_path("/organisations/#{org_id}/sales-logs?years[]=&years[]=2021&status[]=&assigned_to=all&user=&owning_organisation_select=all&owning_organisation=&managing_organisation_select=all&managing_organisation=")
+        expect(page).to have_current_path("/organisations/#{org_id}/sales-logs?years[]=&years[]=2022&status[]=&assigned_to=all&user=&owning_organisation_select=all&owning_organisation=&managing_organisation_select=all&managing_organisation=")
         expect(page).not_to have_link first_log.id.to_s, href: "/sales-logs/#{first_log.id}"
       end
     end
diff --git a/spec/helpers/filters_helper_spec.rb b/spec/helpers/filters_helper_spec.rb
index f60cc483e..52a9595fc 100644
--- a/spec/helpers/filters_helper_spec.rb
+++ b/spec/helpers/filters_helper_spec.rb
@@ -240,12 +240,38 @@ RSpec.describe FiltersHelper do
   end
 
   describe "#collection_year_options" do
-    it "includes 2023/2024 option" do
-      expect(collection_year_options).to eq(
-        {
-          "2021": "2021/22", "2022": "2022/23", "2023": "2023/24"
-        },
-      )
+    context "with 23/24 as the current collection year" do
+      around do |example|
+        Timecop.freeze(Time.zone.local(2023, 5, 1)) do
+          example.run
+        end
+        Timecop.return
+      end
+
+      it "has the correct options" do
+        expect(collection_year_options).to eq(
+          {
+            "2023" => "2023/24", "2022" => "2022/23", "2021" => "2021/22"
+          },
+        )
+      end
+    end
+
+    context "with 24/25 as the current collection year" do
+      around do |example|
+        Timecop.freeze(Time.zone.local(2024, 5, 1)) do
+          example.run
+        end
+        Timecop.return
+      end
+
+      it "has the correct options" do
+        expect(collection_year_options).to eq(
+          {
+            "2024" => "2024/25", "2023" => "2023/24", "2022" => "2022/23"
+          },
+        )
+      end
     end
   end
 

From 769ba5ab43d3b12f388dd00591248e2d43a8a5e2 Mon Sep 17 00:00:00 2001
From: Rachael Booth <Rachael.Booth@softwire.com>
Date: Wed, 21 Feb 2024 11:48:07 +0000
Subject: [PATCH 3/6] CLDC-3237: Update crown (#2253)

---
 Gemfile      | 2 +-
 Gemfile.lock | 8 ++++----
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Gemfile b/Gemfile
index aae99d51e..d80a159dd 100644
--- a/Gemfile
+++ b/Gemfile
@@ -18,7 +18,7 @@ gem "jsbundling-rails"
 # Reduces boot times through caching; required in config/boot.rb
 gem "bootsnap", ">= 1.4.4", require: false
 # GOV UK frontend components
-gem "govuk-components", "~> 5.0"
+gem "govuk-components", "~> 5.1"
 # GOV UK component form builder DSL
 gem "govuk_design_system_formbuilder", "~> 5.0"
 # Convert Markdown into GOV.UK frontend-styled HTML
diff --git a/Gemfile.lock b/Gemfile.lock
index 0970a7721..74e787baf 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -171,10 +171,10 @@ GEM
       raabro (~> 1.4)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    govuk-components (5.0.2)
+    govuk-components (5.1.0)
       html-attributes-utils (~> 1.0.0, >= 1.0.0)
       pagy (~> 6.0)
-      view_component (>= 3.9, < 3.10)
+      view_component (>= 3.9, < 3.11)
     govuk_design_system_formbuilder (5.0.0)
       actionview (>= 6.1)
       activemodel (>= 6.1)
@@ -237,7 +237,7 @@ GEM
       childprocess (>= 0.6.3, < 5)
       iniparse (~> 1.4)
       rexml (~> 3.2)
-    pagy (6.2.0)
+    pagy (6.5.0)
     paper_trail (14.0.0)
       activerecord (>= 6.0)
       request_store (~> 1.4)
@@ -458,7 +458,7 @@ DEPENDENCIES
   erb_lint
   factory_bot_rails
   faker
-  govuk-components (~> 5.0)
+  govuk-components (~> 5.1)
   govuk_design_system_formbuilder (~> 5.0)
   govuk_markdown
   jsbundling-rails
diff --git a/package.json b/package.json
index 5b34103fe..dedf7b6b7 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
     "css-loader": "^6.7.1",
     "custom-event-polyfill": "^1.0.7",
     "file-loader": "^6.2.0",
-    "govuk-frontend": "5.0.0",
+    "govuk-frontend": "5.1.0",
     "html5shiv": "^3.7.3",
     "intersection-observer": "^0.12.0",
     "mini-css-extract-plugin": "^2.6.0",
diff --git a/yarn.lock b/yarn.lock
index d6e84caac..1286245d6 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3245,10 +3245,10 @@ govuk-frontend@4.7.0:
   resolved "https://registry.yarnpkg.com/govuk-frontend/-/govuk-frontend-4.7.0.tgz#69950b6c2e69f435ffe9aa60d8dee232dac977de"
   integrity sha512-0OsdCusF5qvLWwKziU8zqxiC0nq6WP0ZQuw51ymZ/1V0tO71oIKMlSLN2S9bm8RcEGSoidPt2A34gKxePrLjvg==
 
-govuk-frontend@5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/govuk-frontend/-/govuk-frontend-5.0.0.tgz#c08a4d1115fb31eb39b6d19979c627f816185dd7"
-  integrity sha512-3WSfvQ+3kw/q/m8jrq/t8XnMUA8D2r0uhGyZaDbIh1gWTJBQzJBHbHiKYI9nc9ixIXdCFsc9RozkgEm57a795g==
+govuk-frontend@5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/govuk-frontend/-/govuk-frontend-5.1.0.tgz#55e520940b587ddd023e96251efaa076acc9bd5f"
+  integrity sha512-Dc3J+uOI4i2VR3BVyfxbf6qVjTT4n4bBqbD0/Io6feP8pt/4IfKdP1vWimZf+BwMKKMXacw10hmdy5UcD6Cr8w==
 
 govuk-prototype-kit@^13.14.1:
   version "13.16.0"

From f9ac59b860abee84b0aad431ab5985e8c5ac4123 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Wed, 21 Feb 2024 14:13:02 +0000
Subject: [PATCH 4/6] CLDC-3186 Display reported by question (#2228)

* Remove feature toggle

* Remove merge organisatiions toggle

* Update managing org routing for 2024

* Update bulk upload
---
 .../sales_log_summary_component.html.erb      |   2 +-
 app/models/form/lettings/pages/stock_owner.rb |  10 +-
 .../form/lettings/questions/stock_owner.rb    |   6 +-
 .../form/sales/pages/managing_organisation.rb |  18 +-
 .../form/sales/pages/owning_organisation.rb   |  11 +-
 .../sales/questions/owning_organisation_id.rb |  75 ++++-----
 .../bulk_upload/sales/year2023/row_parser.rb  |   4 +-
 .../bulk_upload/sales/year2024/row_parser.rb  |  13 +-
 app/services/feature_toggle.rb                |   8 -
 app/views/logs/_log_filters.html.erb          |   2 +-
 app/views/organisations/show.html.erb         |   4 +-
 .../sales/pages/managing_organisation_spec.rb | 154 +++++++++++++++++-
 spec/requests/sales_logs_controller_spec.rb   |  24 ---
 .../sales/year2024/row_parser_spec.rb         |  68 +++++---
 14 files changed, 254 insertions(+), 145 deletions(-)

diff --git a/app/components/sales_log_summary_component.html.erb b/app/components/sales_log_summary_component.html.erb
index 82acc6e14..5c0668301 100644
--- a/app/components/sales_log_summary_component.html.erb
+++ b/app/components/sales_log_summary_component.html.erb
@@ -32,7 +32,7 @@
               <dd class="app-metadata__definition"><%= log.owning_organisation&.name %></dd>
             </div>
           <% end %>
-          <% if log.managing_organisation && FeatureToggle.sales_managing_organisation_enabled? %>
+          <% if log.managing_organisation %>
             <div class="app-metadata__item">
               <dt class="app-metadata__term">Reported by</dt>
               <dd class="app-metadata__definition"><%= log.managing_organisation&.name %></dd>
diff --git a/app/models/form/lettings/pages/stock_owner.rb b/app/models/form/lettings/pages/stock_owner.rb
index 07fe01327..a653e7fe2 100644
--- a/app/models/form/lettings/pages/stock_owner.rb
+++ b/app/models/form/lettings/pages/stock_owner.rb
@@ -14,16 +14,10 @@ class Form::Lettings::Pages::StockOwner < ::Form::Page
     return false unless current_user
     return true if current_user.support?
 
-    stock_owners = if FeatureToggle.merge_organisations_enabled?
-                     current_user.organisation.stock_owners + current_user.organisation.absorbed_organisations.where(holds_own_stock: true)
-                   else
-                     current_user.organisation.stock_owners
-                   end
+    stock_owners = current_user.organisation.stock_owners + current_user.organisation.absorbed_organisations.where(holds_own_stock: true)
 
     if current_user.organisation.holds_own_stock?
-      if FeatureToggle.merge_organisations_enabled? && current_user.organisation.absorbed_organisations.any?(&:holds_own_stock?)
-        return true
-      end
+      return true if current_user.organisation.absorbed_organisations.any?(&:holds_own_stock?)
       return true if stock_owners.count >= 1
 
       log.update!(owning_organisation: current_user.organisation)
diff --git a/app/models/form/lettings/questions/stock_owner.rb b/app/models/form/lettings/questions/stock_owner.rb
index c78c4a080..6b5e77c6d 100644
--- a/app/models/form/lettings/questions/stock_owner.rb
+++ b/app/models/form/lettings/questions/stock_owner.rb
@@ -67,11 +67,7 @@ class Form::Lettings::Questions::StockOwner < ::Form::Question
   def hidden_in_check_answers?(_log, user = nil)
     return false if user.support?
 
-    stock_owners = if FeatureToggle.merge_organisations_enabled?
-                     user.organisation.stock_owners + user.organisation.absorbed_organisations.where(holds_own_stock: true)
-                   else
-                     user.organisation.stock_owners
-                   end
+    stock_owners = user.organisation.stock_owners + user.organisation.absorbed_organisations.where(holds_own_stock: true)
 
     if user.organisation.holds_own_stock?
       stock_owners.count.zero?
diff --git a/app/models/form/sales/pages/managing_organisation.rb b/app/models/form/sales/pages/managing_organisation.rb
index 0c9ed7337..3d8e59383 100644
--- a/app/models/form/sales/pages/managing_organisation.rb
+++ b/app/models/form/sales/pages/managing_organisation.rb
@@ -12,10 +12,20 @@ class Form::Sales::Pages::ManagingOrganisation < ::Form::Page
 
   def routed_to?(log, current_user)
     return false unless current_user
-    return false unless current_user.support?
-    return false unless FeatureToggle.sales_managing_organisation_enabled?
-    return false unless log.owning_organisation
 
-    log.owning_organisation.managing_agents.count >= 1
+    if form.start_year_after_2024?
+      organisation = current_user.support? ? log.owning_organisation : current_user.organisation
+
+      return false unless organisation
+      return false if log.owning_organisation != organisation && !organisation.holds_own_stock?
+      return true unless organisation.holds_own_stock?
+
+      organisation.managing_agents.count >= 1
+    else
+      return false unless current_user.support?
+      return false unless log.owning_organisation
+
+      log.owning_organisation.managing_agents.count >= 1
+    end
   end
 end
diff --git a/app/models/form/sales/pages/owning_organisation.rb b/app/models/form/sales/pages/owning_organisation.rb
index d8787a456..f0c9e4e68 100644
--- a/app/models/form/sales/pages/owning_organisation.rb
+++ b/app/models/form/sales/pages/owning_organisation.rb
@@ -13,19 +13,12 @@ class Form::Sales::Pages::OwningOrganisation < ::Form::Page
   def routed_to?(log, current_user)
     return false unless current_user
     return true if current_user.support?
-    return false unless FeatureToggle.sales_managing_organisation_enabled?
     return true if has_multiple_stock_owners_with_own_stock?(current_user)
 
-    stock_owners = if FeatureToggle.merge_organisations_enabled?
-                     current_user.organisation.stock_owners.where(holds_own_stock: true) + current_user.organisation.absorbed_organisations.where(holds_own_stock: true)
-                   else
-                     current_user.organisation.stock_owners.where(holds_own_stock: true)
-                   end
+    stock_owners = current_user.organisation.stock_owners.where(holds_own_stock: true) + current_user.organisation.absorbed_organisations.where(holds_own_stock: true)
 
     if current_user.organisation.holds_own_stock?
-      if FeatureToggle.merge_organisations_enabled? && current_user.organisation.absorbed_organisations.any?(&:holds_own_stock?)
-        return true
-      end
+      return true if current_user.organisation.absorbed_organisations.any?(&:holds_own_stock?)
       return true if stock_owners.count >= 1
 
       log.update!(owning_organisation: current_user.organisation)
diff --git a/app/models/form/sales/questions/owning_organisation_id.rb b/app/models/form/sales/questions/owning_organisation_id.rb
index 47a10993d..a4c700e05 100644
--- a/app/models/form/sales/questions/owning_organisation_id.rb
+++ b/app/models/form/sales/questions/owning_organisation_id.rb
@@ -14,7 +14,7 @@ class Form::Sales::Questions::OwningOrganisationId < ::Form::Question
     return answer_opts unless user
     return answer_opts unless log
 
-    if FeatureToggle.sales_managing_organisation_enabled? && !user.support?
+    unless user.support?
       if log.owning_organisation_id.present?
         answer_opts[log.owning_organisation.id] = log.owning_organisation.name
       end
@@ -28,43 +28,36 @@ class Form::Sales::Questions::OwningOrganisationId < ::Form::Question
       end
     end
 
-    if FeatureToggle.merge_organisations_enabled?
-      if log.owning_organisation_id.present?
-        answer_opts[log.owning_organisation.id] = log.owning_organisation.name
-      end
+    if log.owning_organisation_id.present?
+      answer_opts[log.owning_organisation.id] = log.owning_organisation.name
+    end
 
-      recently_absorbed_organisations = user.organisation.absorbed_organisations.merged_during_open_collection_period
-      if !user.support? && user.organisation.holds_own_stock?
-        answer_opts[user.organisation.id] = if recently_absorbed_organisations.exists? && user.organisation.available_from.present?
-                                              "#{user.organisation.name} (Your organisation, active as of #{user.organisation.available_from.to_fs(:govuk_date)})"
-                                            else
-                                              "#{user.organisation.name} (Your organisation)"
-                                            end
-      end
+    recently_absorbed_organisations = user.organisation.absorbed_organisations.merged_during_open_collection_period
+    if !user.support? && user.organisation.holds_own_stock?
+      answer_opts[user.organisation.id] = if recently_absorbed_organisations.exists? && user.organisation.available_from.present?
+                                            "#{user.organisation.name} (Your organisation, active as of #{user.organisation.available_from.to_fs(:govuk_date)})"
+                                          else
+                                            "#{user.organisation.name} (Your organisation)"
+                                          end
+    end
 
-      if user.support?
-        Organisation.where(holds_own_stock: true).find_each do |org|
-          if org.merge_date.present?
-            answer_opts[org.id] = "#{org.name} (inactive as of #{org.merge_date.to_fs(:govuk_date)})" if org.merge_date >= FormHandler.instance.start_date_of_earliest_open_for_editing_collection_period
-          elsif org.absorbed_organisations.merged_during_open_collection_period.exists? && org.available_from.present?
-            answer_opts[org.id] = "#{org.name} (active as of #{org.available_from.to_fs(:govuk_date)})"
-          else
-            answer_opts[org.id] = org.name
-          end
-        end
-      else
-        recently_absorbed_organisations.each do |absorbed_org|
-          answer_opts[absorbed_org.id] = merged_organisation_label(absorbed_org.name, absorbed_org.merge_date) if absorbed_org.holds_own_stock?
+    if user.support?
+      Organisation.where(holds_own_stock: true).find_each do |org|
+        if org.merge_date.present?
+          answer_opts[org.id] = "#{org.name} (inactive as of #{org.merge_date.to_fs(:govuk_date)})" if org.merge_date >= FormHandler.instance.start_date_of_earliest_open_for_editing_collection_period
+        elsif org.absorbed_organisations.merged_during_open_collection_period.exists? && org.available_from.present?
+          answer_opts[org.id] = "#{org.name} (active as of #{org.available_from.to_fs(:govuk_date)})"
+        else
+          answer_opts[org.id] = org.name
         end
       end
-
-      answer_opts
     else
-      Organisation.select(:id, :name).each_with_object(answer_opts) do |organisation, hsh|
-        hsh[organisation.id] = organisation.name
-        hsh
+      recently_absorbed_organisations.each do |absorbed_org|
+        answer_opts[absorbed_org.id] = merged_organisation_label(absorbed_org.name, absorbed_org.merge_date) if absorbed_org.holds_own_stock?
       end
     end
+
+    answer_opts
   end
 
   def displayed_answer_options(log, user = nil)
@@ -82,18 +75,14 @@ class Form::Sales::Questions::OwningOrganisationId < ::Form::Question
   end
 
   def hidden_in_check_answers?(_log, user = nil)
-    if FeatureToggle.merge_organisations_enabled?
-      return false if user.support?
+    return false if user.support?
 
-      stock_owners = user.organisation.stock_owners + user.organisation.absorbed_organisations.where(holds_own_stock: true)
+    stock_owners = user.organisation.stock_owners + user.organisation.absorbed_organisations.where(holds_own_stock: true)
 
-      if user.organisation.holds_own_stock?
-        stock_owners.count.zero?
-      else
-        stock_owners.count <= 1
-      end
+    if user.organisation.holds_own_stock?
+      stock_owners.count.zero?
     else
-      !current_user.support?
+      stock_owners.count <= 1
     end
   end
 
@@ -104,11 +93,7 @@ class Form::Sales::Questions::OwningOrganisationId < ::Form::Question
 private
 
   def selected_answer_option_is_derived?(_log)
-    if FeatureToggle.merge_organisations_enabled?
-      true
-    else
-      false
-    end
+    true
   end
 
   def merged_organisation_label(name, merge_date)
diff --git a/app/services/bulk_upload/sales/year2023/row_parser.rb b/app/services/bulk_upload/sales/year2023/row_parser.rb
index 93f15395f..0759d7845 100644
--- a/app/services/bulk_upload/sales/year2023/row_parser.rb
+++ b/app/services/bulk_upload/sales/year2023/row_parser.rb
@@ -455,12 +455,12 @@ class BulkUpload::Sales::Year2023::RowParser
 
   validate :validate_owning_org_data_given, on: :after_log
   validate :validate_owning_org_exists, on: :after_log
-  validate :validate_owning_org_owns_stock, on: :after_log if FeatureToggle.sales_managing_organisation_enabled?
+  validate :validate_owning_org_owns_stock, on: :after_log
   validate :validate_owning_org_permitted, on: :after_log
 
   validate :validate_created_by_exists, on: :after_log
   validate :validate_created_by_related, on: :after_log
-  validate :validate_managing_org_related, on: :after_log if FeatureToggle.sales_managing_organisation_enabled?
+  validate :validate_managing_org_related, on: :after_log
   validate :validate_relevant_collection_window, on: :after_log
   validate :validate_incomplete_soft_validations, on: :after_log
 
diff --git a/app/services/bulk_upload/sales/year2024/row_parser.rb b/app/services/bulk_upload/sales/year2024/row_parser.rb
index 80748c0bc..4e0f689a4 100644
--- a/app/services/bulk_upload/sales/year2024/row_parser.rb
+++ b/app/services/bulk_upload/sales/year2024/row_parser.rb
@@ -455,12 +455,12 @@ class BulkUpload::Sales::Year2024::RowParser
 
   validate :validate_owning_org_data_given, on: :after_log
   validate :validate_owning_org_exists, on: :after_log
-  validate :validate_owning_org_owns_stock, on: :after_log if FeatureToggle.sales_managing_organisation_enabled?
+  validate :validate_owning_org_owns_stock, on: :after_log
   validate :validate_owning_org_permitted, on: :after_log
 
   validate :validate_created_by_exists, on: :after_log
   validate :validate_created_by_related, on: :after_log
-  validate :validate_managing_org_related, on: :after_log if FeatureToggle.sales_managing_organisation_enabled?
+  validate :validate_managing_org_related, on: :after_log
   validate :validate_relevant_collection_window, on: :after_log
   validate :validate_incomplete_soft_validations, on: :after_log
 
@@ -732,6 +732,7 @@ private
       discount: %i[field_116],
       othtype: %i[field_12],
       owning_organisation_id: %i[field_1],
+      managing_organisation_id: [:field_2],
       created_by: %i[field_3],
       hhregres: %i[field_72],
       hhregresstill: %i[field_73],
@@ -1211,9 +1212,7 @@ private
   end
 
   def managing_organisation
-    return owning_organisation if created_by&.organisation&.absorbed_organisations&.include?(owning_organisation)
-
-    created_by&.organisation || bulk_upload.user.organisation
+    Organisation.find_by_id_on_multiple_fields(field_2)
   end
 
   def nationality_group(nationality_value)
@@ -1228,8 +1227,8 @@ private
     if owning_organisation && managing_organisation && !owning_organisation.can_be_managed_by?(organisation: managing_organisation)
       block_log_creation!
 
-      if errors[:field_3].blank?
-        errors.add(:field_3, "This user belongs to an organisation that does not have a relationship with the owning organisation", category: :setup)
+      if errors[:field_2].blank?
+        errors.add(:field_2, "This organisation does not have a relationship with the owning organisation", category: :setup)
       end
     end
   end
diff --git a/app/services/feature_toggle.rb b/app/services/feature_toggle.rb
index 38c1372e1..e00e89b90 100644
--- a/app/services/feature_toggle.rb
+++ b/app/services/feature_toggle.rb
@@ -28,10 +28,6 @@ class FeatureToggle
     false
   end
 
-  def self.merge_organisations_enabled?
-    true
-  end
-
   def self.deduplication_flow_enabled?
     true
   end
@@ -47,8 +43,4 @@ class FeatureToggle
   def self.service_moved?
     false
   end
-
-  def self.sales_managing_organisation_enabled?
-    true
-  end
 end
diff --git a/app/views/logs/_log_filters.html.erb b/app/views/logs/_log_filters.html.erb
index 16053c805..ea7496bdd 100644
--- a/app/views/logs/_log_filters.html.erb
+++ b/app/views/logs/_log_filters.html.erb
@@ -93,7 +93,7 @@
           } %>
         <% end %>
 
-        <% if (current_user.support? || non_support_with_managing_orgs?) && (user_or_org_lettings_path? || FeatureToggle.sales_managing_organisation_enabled?) %>
+        <% if current_user.support? || non_support_with_managing_orgs? %>
           <%= render partial: "filters/radio_filter", locals: {
             f:,
             options: {
diff --git a/app/views/organisations/show.html.erb b/app/views/organisations/show.html.erb
index 21f722920..e2e07e28c 100644
--- a/app/views/organisations/show.html.erb
+++ b/app/views/organisations/show.html.erb
@@ -36,9 +36,7 @@
       <% end %>
       <%= data_sharing_agreement_row(organisation: @organisation, user: current_user, summary_list:) %>
     <% end %>
-    <% if FeatureToggle.merge_organisations_enabled? %>
-      <p>To report a merge or update your organisation details, <%= govuk_link_to "contact the helpdesk", GlobalConstants::HELPDESK_URL %>.</p>
-    <% end %>
+    <p>To report a merge or update your organisation details, <%= govuk_link_to "contact the helpdesk", GlobalConstants::HELPDESK_URL %>.</p>
     <%= render partial: "organisations/merged_organisation_details" %>
   </div>
 </div>
diff --git a/spec/models/form/sales/pages/managing_organisation_spec.rb b/spec/models/form/sales/pages/managing_organisation_spec.rb
index 0a34554cb..8c411401d 100644
--- a/spec/models/form/sales/pages/managing_organisation_spec.rb
+++ b/spec/models/form/sales/pages/managing_organisation_spec.rb
@@ -5,9 +5,13 @@ RSpec.describe Form::Sales::Pages::ManagingOrganisation, type: :model do
 
   let(:page_id) { nil }
   let(:page_definition) { nil }
-  let(:subsection) { instance_double(Form::Subsection) }
+  let(:subsection) { instance_double(Form::Subsection, form:) }
   let(:form) { instance_double(Form) }
 
+  before do
+    allow(form).to receive(:start_year_after_2024?).and_return(false)
+  end
+
   it "has correct subsection" do
     expect(page.subsection).to eq(subsection)
   end
@@ -32,8 +36,8 @@ RSpec.describe Form::Sales::Pages::ManagingOrganisation, type: :model do
     expect(page.depends_on).to be nil
   end
 
-  describe "#routed_to?" do
-    let(:log) { create(:lettings_log) }
+  describe "#routed_to? with 2023 logs" do
+    let(:log) { create(:sales_log) }
     let(:organisation) { create(:organisation) }
 
     context "when user nil" do
@@ -54,7 +58,7 @@ RSpec.describe Form::Sales::Pages::ManagingOrganisation, type: :model do
       let(:user) { create(:user, :support) }
 
       context "when owning_organisation not set" do
-        let(:log) { create(:lettings_log, owning_organisation: nil) }
+        let(:log) { create(:sales_log, owning_organisation: nil) }
 
         it "is not shown" do
           expect(page.routed_to?(log, user)).to eq(false)
@@ -103,4 +107,146 @@ RSpec.describe Form::Sales::Pages::ManagingOrganisation, type: :model do
       end
     end
   end
+
+  describe "#routed_to? with 2024 logs" do
+    let(:log) { create(:sales_log) }
+    let(:organisation) { create(:organisation) }
+
+    before do
+      allow(form).to receive(:start_year_after_2024?).and_return(true)
+    end
+
+    context "when user nil" do
+      it "is not shown" do
+        expect(page.routed_to?(log, nil)).to eq(false)
+      end
+    end
+
+    context "when support" do
+      context "when does not hold own stock" do
+        let(:user) do
+          create(:user, :support, organisation: create(:organisation, holds_own_stock: false))
+        end
+        let(:log) { create(:sales_log, owning_organisation: user.organisation) }
+
+        it "is shown" do
+          expect(page.routed_to?(log, user)).to eq(true)
+        end
+      end
+
+      context "when owning_organisation not set" do
+        let(:user) { create(:user, :support) }
+        let(:log) { create(:sales_log, owning_organisation: nil) }
+
+        it "is not shown" do
+          expect(page.routed_to?(log, user)).to eq(false)
+        end
+      end
+
+      context "when holds own stock" do
+        let(:user) do
+          create(:user, :support, organisation: create(:organisation, holds_own_stock: true))
+        end
+
+        context "with 0 managing_agents" do
+          it "is not shown" do
+            expect(page.routed_to?(log, user)).to eq(false)
+          end
+        end
+
+        context "with >1 managing_agents" do
+          before do
+            create(:organisation_relationship, parent_organisation: log.owning_organisation)
+            create(:organisation_relationship, parent_organisation: log.owning_organisation)
+          end
+
+          it "is shown" do
+            expect(page.routed_to?(log, user)).to eq(true)
+          end
+        end
+
+        context "with 1 managing_agents" do
+          let(:managing_agent) { create(:organisation) }
+
+          before do
+            create(
+              :organisation_relationship,
+              child_organisation: managing_agent,
+              parent_organisation: log.owning_organisation,
+            )
+          end
+
+          it "is shown" do
+            expect(page.routed_to?(log, user)).to eq(true)
+          end
+        end
+      end
+    end
+
+    context "when not support" do
+      context "when does not hold own stock" do
+        let(:user) { create(:user, :data_coordinator, organisation: create(:organisation, holds_own_stock: false)) }
+
+        context "and the user's organisation is selected as owning organisation" do
+          let(:log) { create(:sales_log, owning_organisation: user.organisation) }
+
+          it "is shown" do
+            expect(page.routed_to?(log, user)).to eq(true)
+          end
+        end
+
+        context "and a different than the user's organisation is selected as owning organisation" do
+          let(:stock_owner) { create(:organisation, holds_own_stock: true) }
+          let(:log) { create(:sales_log, owning_organisation: stock_owner) }
+
+          before do
+            create(:organisation_relationship, parent_organisation: stock_owner, child_organisation: user.organisation)
+          end
+
+          it "is not shown" do
+            expect(page.routed_to?(log, user)).to eq(false)
+          end
+        end
+      end
+
+      context "when holds own stock" do
+        let(:user) do
+          create(:user, :data_coordinator, organisation: create(:organisation, holds_own_stock: true))
+        end
+
+        context "with 0 managing_agents" do
+          it "is not shown" do
+            expect(page.routed_to?(log, user)).to eq(false)
+          end
+        end
+
+        context "with >1 managing_agents" do
+          before do
+            create(:organisation_relationship, parent_organisation: user.organisation)
+            create(:organisation_relationship, parent_organisation: user.organisation)
+          end
+
+          it "is shown" do
+            expect(page.routed_to?(log, user)).to eq(true)
+          end
+        end
+
+        context "with 1 managing_agents" do
+          let(:managing_agent) { create(:organisation) }
+
+          before do
+            create(
+              :organisation_relationship,
+              child_organisation: managing_agent,
+              parent_organisation: user.organisation,
+            )
+          end
+
+          it "is shown" do
+            expect(page.routed_to?(log, user)).to eq(true)
+          end
+        end
+      end
+    end
+  end
 end
diff --git a/spec/requests/sales_logs_controller_spec.rb b/spec/requests/sales_logs_controller_spec.rb
index a9bf07883..6ad991934 100644
--- a/spec/requests/sales_logs_controller_spec.rb
+++ b/spec/requests/sales_logs_controller_spec.rb
@@ -161,30 +161,6 @@ RSpec.describe SalesLogsController, type: :request do
               expect(sales_log.managing_organisation.name).to eq("User org")
             end
           end
-
-          context "when the user's org doesn't hold stock and merge_organisations_enabled is false" do
-            let(:organisation) { FactoryBot.create(:organisation, name: "User org", holds_own_stock: false) }
-            let(:user) { FactoryBot.create(:user, :data_coordinator, organisation:) }
-
-            before do
-              RequestHelper.stub_http_requests
-              sign_in user
-              allow(FeatureToggle).to receive(:merge_organisations_enabled?).and_return(false)
-              post "/sales-logs", headers:
-            end
-
-            it "does not set owning organisation" do
-              created_id = response.location.match(/[0-9]+/)[0]
-              sales_log = SalesLog.find_by(id: created_id)
-              expect(sales_log.owning_organisation).to be_nil
-            end
-
-            it "sets managing organisation as the user organisation" do
-              created_id = response.location.match(/[0-9]+/)[0]
-              sales_log = SalesLog.find_by(id: created_id)
-              expect(sales_log.managing_organisation.name).to eq("User org")
-            end
-          end
         end
       end
     end
diff --git a/spec/services/bulk_upload/sales/year2024/row_parser_spec.rb b/spec/services/bulk_upload/sales/year2024/row_parser_spec.rb
index 2656ff1c0..8866d3de2 100644
--- a/spec/services/bulk_upload/sales/year2024/row_parser_spec.rb
+++ b/spec/services/bulk_upload/sales/year2024/row_parser_spec.rb
@@ -9,11 +9,13 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
   let(:bulk_upload) { create(:bulk_upload, :sales, user:, year: 2024) }
   let(:user) { create(:user, organisation: owning_org) }
   let(:owning_org) { create(:organisation, :with_old_visible_id) }
+  let(:managing_org) { create(:organisation, :with_old_visible_id) }
+
   let(:setup_section_params) do
     {
       bulk_upload:,
       field_1: owning_org.old_visible_id, # organisation
-      field_2: owning_org.old_visible_id, # organisation
+      field_2: managing_org.old_visible_id, # organisation
       field_3: user.email, # user
       field_4: now.day.to_s, # sale day
       field_5: now.month.to_s, # sale month
@@ -31,7 +33,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
     {
       bulk_upload:,
       field_1: owning_org.old_visible_id,
-      field_2: owning_org.old_visible_id,
+      field_2: managing_org.old_visible_id,
 
       field_4: "12",
       field_5: "5",
@@ -114,6 +116,8 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
   end
 
   around do |example|
+    create(:organisation_relationship, parent_organisation: owning_org, child_organisation: managing_org)
+
     Timecop.freeze(Time.zone.local(2025, 2, 22)) do
       Singleton.__init__(FormHandler)
       example.run
@@ -287,7 +291,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_17 field_18 field_4 field_5 field_6 field_8])
+        expect(errors).to eql(%i[field_1 field_17 field_18 field_2 field_4 field_5 field_6 field_8])
       end
     end
 
@@ -303,7 +307,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_15 field_17 field_18 field_4 field_5 field_6 field_9])
+        expect(errors).to eql(%i[field_1 field_15 field_17 field_18 field_2 field_4 field_5 field_6 field_9])
       end
     end
 
@@ -321,7 +325,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_16 field_17 field_18 field_4 field_5 field_6])
+        expect(errors).to eql(%i[field_1 field_16 field_17 field_18 field_2 field_4 field_5 field_6])
       end
     end
 
@@ -338,7 +342,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_10 field_15 field_17 field_18 field_4 field_5 field_6])
+        expect(errors).to eql(%i[field_1 field_10 field_15 field_17 field_18 field_2 field_4 field_5 field_6])
       end
     end
 
@@ -356,7 +360,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_17 field_18 field_4 field_5 field_6 field_8])
+        expect(errors).to eql(%i[field_1 field_17 field_18 field_2 field_4 field_5 field_6 field_8])
       end
     end
 
@@ -372,7 +376,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_11 field_13 field_14 field_17 field_18 field_4 field_5 field_6])
+        expect(errors).to eql(%i[field_1 field_11 field_13 field_14 field_17 field_18 field_2 field_4 field_5 field_6])
       end
     end
 
@@ -390,7 +394,7 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       it "has errors on correct setup fields" do
         errors = parser.errors.select { |e| e.options[:category] == :setup }.map(&:attribute).sort
 
-        expect(errors).to eql(%i[field_1 field_12 field_14 field_15 field_17 field_18 field_4 field_5 field_6])
+        expect(errors).to eql(%i[field_1 field_12 field_14 field_15 field_17 field_18 field_2 field_4 field_5 field_6])
       end
     end
 
@@ -1446,39 +1450,55 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
       let(:attributes) { setup_section_params }
 
       context "when user is part of the owning organisation" do
-        it "sets managing organisation to the users organisation" do
+        it "sets managing organisation to the correct organisation" do
           parser.valid?
           expect(parser.log.owning_organisation_id).to be(owning_org.id)
-          expect(parser.log.managing_organisation_id).to be(owning_org.id)
+          expect(parser.log.managing_organisation_id).to be(managing_org.id)
         end
       end
 
-      context "when user is part of an organisation affiliated with owning org" do
-        let(:managing_agent) { create(:organisation) }
-        let(:user) { create(:user, organisation: managing_agent) }
-        let(:attributes) { setup_section_params }
+      context "when blank" do
+        let(:attributes) { { bulk_upload:, field_2: "" } }
 
-        before do
-          create(:organisation_relationship, child_organisation: managing_agent, parent_organisation: owning_org)
+        it "is not permitted as setup error" do
+          parser.valid?
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_2 }.message).to eql("You must answer reported by")
         end
 
+        it "blocks log creation" do
+          parser.valid?
+          expect(parser).to be_block_log_creation
+        end
+      end
+
+      context "when cannot find managing org" do
+        let(:attributes) { { bulk_upload:, field_2: "donotexist" } }
+
         it "is not permitted as setup error" do
           parser.valid?
-          expect(parser.log.owning_organisation_id).to be(owning_org.id)
-          expect(parser.log.managing_organisation_id).to be(managing_agent.id)
+          setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
+
+          expect(setup_errors.find { |e| e.attribute == :field_2 }.message).to eql("You must answer reported by")
+        end
+
+        it "blocks log creation" do
+          parser.valid?
+          expect(parser).to be_block_log_creation
         end
       end
 
-      context "when user is part of an organisation not affiliated with owning org" do
-        let(:unaffiliated_org) { create(:organisation) }
-        let(:user) { create(:user, organisation: unaffiliated_org) }
-        let(:attributes) { setup_section_params }
+      context "when not affiliated with managing org" do
+        let(:unaffiliated_org) { create(:organisation, :with_old_visible_id) }
+
+        let(:attributes) { { bulk_upload:, field_1: owning_org.old_visible_id, field_2: unaffiliated_org.old_visible_id } }
 
         it "is not permitted as setup error" do
           parser.valid?
           setup_errors = parser.errors.select { |e| e.options[:category] == :setup }
 
-          expect(setup_errors.find { |e| e.attribute == :field_3 }.message).to eql("This user belongs to an organisation that does not have a relationship with the owning organisation")
+          expect(setup_errors.find { |e| e.attribute == :field_2 }.message).to eql("This organisation does not have a relationship with the owning organisation")
         end
 
         it "blocks log creation" do

From 703bf023a3a585f566336000ca2809957d7515d0 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Thu, 22 Feb 2024 08:06:18 +0000
Subject: [PATCH 5/6] CLDC-3195 Download schemes per organisation (fix) (#2237)

* Post to organisation send csv email path

* Download stock owner schemes

* Add parent schemes to the user model directly
---
 app/controllers/organisations_controller.rb |  2 +-
 app/jobs/scheme_email_csv_job.rb            |  6 +-
 app/models/user.rb                          |  2 +-
 spec/features/organisation_spec.rb          | 13 ++++
 spec/jobs/scheme_email_csv_job_spec.rb      | 81 ++++++++++++++-------
 5 files changed, 75 insertions(+), 29 deletions(-)

diff --git a/app/controllers/organisations_controller.rb b/app/controllers/organisations_controller.rb
index 0952d12ac..4612e6bc9 100644
--- a/app/controllers/organisations_controller.rb
+++ b/app/controllers/organisations_controller.rb
@@ -33,7 +33,7 @@ class OrganisationsController < ApplicationController
     organisation_schemes = Scheme.where(owning_organisation: [@organisation] + @organisation.parent_organisations)
     unpaginated_filtered_schemes = filter_manager.filtered_schemes(organisation_schemes, search_term, session_filters)
 
-    render "schemes/download_csv", locals: { search_term:, post_path: email_csv_schemes_path, download_type: params[:download_type], schemes: unpaginated_filtered_schemes }
+    render "schemes/download_csv", locals: { search_term:, post_path: schemes_email_csv_organisation_path, download_type: params[:download_type], schemes: unpaginated_filtered_schemes }
   end
 
   def email_schemes_csv
diff --git a/app/jobs/scheme_email_csv_job.rb b/app/jobs/scheme_email_csv_job.rb
index 1dad752c5..9c3060cb9 100644
--- a/app/jobs/scheme_email_csv_job.rb
+++ b/app/jobs/scheme_email_csv_job.rb
@@ -6,7 +6,11 @@ class SchemeEmailCsvJob < ApplicationJob
   EXPIRATION_TIME = 24.hours.to_i
 
   def perform(user, search_term = nil, filters = {}, all_orgs = false, organisation = nil, download_type = "combined") # rubocop:disable Style/OptionalBooleanParameter - sidekiq can't serialise named params
-    unfiltered_schemes = organisation.present? && user.support? ? Scheme.where(owning_organisation_id: organisation.id) : user.schemes
+    unfiltered_schemes = if organisation.present? && user.support?
+                           Scheme.where(owning_organisation: [organisation] + organisation.parent_organisations)
+                         else
+                           user.schemes
+                         end
     filtered_schemes = FilterManager.filter_schemes(unfiltered_schemes, search_term, filters, all_orgs, user)
     csv_string = Csv::SchemeCsvService.new(download_type:).prepare_csv(filtered_schemes)
 
diff --git a/app/models/user.rb b/app/models/user.rb
index aba948425..0fc21172a 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -113,7 +113,7 @@ class User < ApplicationRecord
     if support?
       Scheme.all
     else
-      Scheme.filter_by_owning_organisation(organisation.absorbed_organisations + [organisation])
+      Scheme.filter_by_owning_organisation(organisation.absorbed_organisations + [organisation] + organisation.parent_organisations)
     end
   end
 
diff --git a/spec/features/organisation_spec.rb b/spec/features/organisation_spec.rb
index f9bb8f450..598be9e05 100644
--- a/spec/features/organisation_spec.rb
+++ b/spec/features/organisation_spec.rb
@@ -327,6 +327,19 @@ RSpec.describe "User Features" do
       end
     end
 
+    context "when viewing schemes for specific organisation" do
+      before do
+        create(:scheme, owning_organisation: organisation)
+        visit("/organisations/#{org_id}/schemes")
+      end
+
+      it "allows downloading schemes csv for the specific org" do
+        click_link("Download schemes (CSV)")
+        click_button("Send email")
+        expect(page).to have_current_path("/organisations/#{org_id}/schemes/csv-confirmation")
+      end
+    end
+
     context "and the organisation does not hold housing stock" do
       before do
         organisation.update!(holds_own_stock: false)
diff --git a/spec/jobs/scheme_email_csv_job_spec.rb b/spec/jobs/scheme_email_csv_job_spec.rb
index d23177ab3..5ddaa91a6 100644
--- a/spec/jobs/scheme_email_csv_job_spec.rb
+++ b/spec/jobs/scheme_email_csv_job_spec.rb
@@ -6,41 +6,52 @@ describe SchemeEmailCsvJob do
   test_url = :test_url
 
   let(:job) { described_class.new }
+  let(:storage_service) { instance_double(Storage::S3Service, write_file: nil, get_presigned_url: test_url) }
+  let(:mailer) { instance_double(CsvDownloadMailer, send_csv_download_mail: nil) }
   let(:user) { FactoryBot.create(:user) }
-  let(:storage_service) { instance_double(Storage::S3Service) }
-  let(:mailer) { instance_double(CsvDownloadMailer) }
-  let(:scheme_csv_service) { instance_double(Csv::SchemeCsvService) }
-  let(:search_term) { "meaning" }
-  let(:filters) { { "owning_organisation" => organisation.id, "status" => %w[active] } }
-  let(:all_orgs) { false }
-  let(:organisation) { build(:organisation) }
-  let(:download_type) { "combined" }
-  let(:schemes) { build_list(:scheme, 5, owning_organisation: organisation) }
-  let(:locations) { build_list(:location, 5, scheme: schemes.first) }
 
   before do
     allow(Storage::S3Service).to receive(:new).and_return(storage_service)
-    allow(storage_service).to receive(:write_file)
-    allow(storage_service).to receive(:get_presigned_url).and_return(test_url)
-
-    allow(Csv::SchemeCsvService).to receive(:new).and_return(scheme_csv_service)
-    allow(scheme_csv_service).to receive(:prepare_csv).and_return("")
-
     allow(CsvDownloadMailer).to receive(:new).and_return(mailer)
-    allow(mailer).to receive(:send_csv_download_mail)
   end
 
   context "when exporting" do
+    let(:scheme_csv_service) { instance_double(Csv::SchemeCsvService) }
+    let(:organisation) { user.organisation }
+    let(:download_type) { "combined" }
+    let(:schemes) { create_list(:scheme, 1, owning_organisation: organisation) }
+
     before do
-      allow(FilterManager).to receive(:filter_schemes).and_return(schemes)
+      create_list(:location, 2, scheme: schemes.first)
     end
 
     context "when download type schemes" do
       let(:download_type) { "schemes" }
 
-      it "uses an appropriate filename in S3" do
-        expect(storage_service).to receive(:write_file).with(/schemes-.*\.csv/, anything)
-        job.perform(user)
+      it "uses an appropriate filename in S3 and exports the correct schemes" do
+        expect(storage_service).to receive(:write_file).with(/schemes-.*\.csv/, anything) do |_, content|
+          expect(content).not_to be_nil
+          expect(content).not_to be_nil
+          expect(CSV.parse(content).count).to eq(2)
+        end
+        job.perform(user, nil, {}, nil, nil, download_type)
+      end
+
+      context "and there are stock owner schemes" do
+        let(:parent_organisation) { create(:organisation) }
+
+        before do
+          create(:scheme, owning_organisation: parent_organisation)
+          create(:organisation_relationship, parent_organisation:, child_organisation: organisation)
+        end
+
+        it "exports the correct number of schemes" do
+          expect(storage_service).to receive(:write_file).with(/schemes-.*\.csv/, anything) do |_, content|
+            expect(content).not_to be_nil
+            expect(CSV.parse(content).count).to eq(3)
+          end
+          job.perform(user, nil, {}, nil, nil, download_type)
+        end
       end
     end
 
@@ -49,7 +60,7 @@ describe SchemeEmailCsvJob do
 
       it "uses an appropriate filename in S3" do
         expect(storage_service).to receive(:write_file).with(/locations-.*\.csv/, anything)
-        job.perform(user)
+        job.perform(user, nil, {}, nil, nil, download_type)
       end
     end
 
@@ -58,7 +69,7 @@ describe SchemeEmailCsvJob do
 
       it "uses an appropriate filename in S3" do
         expect(storage_service).to receive(:write_file).with(/schemes-and-locations.*\.csv/, anything)
-        job.perform(user)
+        job.perform(user, nil, {}, nil, nil, download_type)
       end
     end
 
@@ -67,12 +78,27 @@ describe SchemeEmailCsvJob do
       job.perform(user, nil, {}, nil, organisation, "combined")
     end
 
-    it "calls the filter manager with the arguments provided" do
-      expect(FilterManager).to receive(:filter_schemes).with(a_kind_of(ActiveRecord::Relation), search_term, filters, all_orgs, user)
-      job.perform(user, search_term, filters, all_orgs, organisation, "combined")
+    context "when resources are filtered" do
+      let(:search_term) { "meaning" }
+      let(:filters) { { "owning_organisation" => organisation.id, "status" => %w[active] } }
+      let(:all_orgs) { false }
+
+      before do
+        allow(Csv::SchemeCsvService).to receive(:new).and_return(scheme_csv_service)
+        allow(scheme_csv_service).to receive(:prepare_csv).and_return("")
+        allow(FilterManager).to receive(:filter_schemes).and_return(schemes)
+      end
+
+      it "calls the filter manager with the arguments provided" do
+        expect(FilterManager).to receive(:filter_schemes).with(a_kind_of(ActiveRecord::Relation), search_term, filters, all_orgs, user)
+        job.perform(user, search_term, filters, all_orgs, organisation, "combined")
+      end
     end
 
     it "creates a SchemeCsvService with the correct download type" do
+      allow(Csv::SchemeCsvService).to receive(:new).and_return(scheme_csv_service)
+      allow(scheme_csv_service).to receive(:prepare_csv).and_return("")
+
       expect(Csv::SchemeCsvService).to receive(:new).with(download_type: "schemes")
       job.perform(user, nil, {}, nil, nil, "schemes")
       expect(Csv::SchemeCsvService).to receive(:new).with(download_type: "locations")
@@ -82,6 +108,9 @@ describe SchemeEmailCsvJob do
     end
 
     it "passes the schemes returned by the filter manager to the csv service" do
+      allow(Csv::SchemeCsvService).to receive(:new).and_return(scheme_csv_service)
+      allow(scheme_csv_service).to receive(:prepare_csv).and_return("")
+
       expect(scheme_csv_service).to receive(:prepare_csv).with(schemes)
       job.perform(user, nil, {}, nil, nil, "combined")
     end

From 83f43b21ee2e6af35c93c099e797c036f957c3c5 Mon Sep 17 00:00:00 2001
From: kosiakkatrina <54268893+kosiakkatrina@users.noreply.github.com>
Date: Thu, 22 Feb 2024 08:07:21 +0000
Subject: [PATCH 6/6] CLDC-3217 Validate 24/25 sales field numbers (#2233)

* Validate 24/25 sales field numbers

* Add fields check to 2023 sales
---
 app/services/bulk_upload/sales/validator.rb              | 9 +++++++++
 app/services/bulk_upload/sales/year2023/csv_parser.rb    | 7 +++++++
 app/services/bulk_upload/sales/year2024/csv_parser.rb    | 7 +++++++
 .../bulk_upload/sales/year2023/csv_parser_spec.rb        | 4 ++++
 .../bulk_upload/sales/year2024/csv_parser_spec.rb        | 4 ++++
 5 files changed, 31 insertions(+)

diff --git a/app/services/bulk_upload/sales/validator.rb b/app/services/bulk_upload/sales/validator.rb
index 91fa50096..32e9f7533 100644
--- a/app/services/bulk_upload/sales/validator.rb
+++ b/app/services/bulk_upload/sales/validator.rb
@@ -5,6 +5,7 @@ class BulkUpload::Sales::Validator
   attr_reader :bulk_upload, :path
 
   validate :validate_file_not_empty
+  validate :validate_field_numbers_count
   validate :validate_max_columns
   validate :validate_missing_required_headers
   validate :validate_correct_template
@@ -166,6 +167,14 @@ private
 
     if csv_parser.missing_required_headers?
       errors.add :base, I18n.t("activemodel.errors.models.bulk_upload/sales/validator.attributes.base.no_headers", guidance_link: bulk_upload_sales_log_url(id: "guidance", form: { year: bulk_upload.year }, host: ENV["APP_HOST"], anchor: "using-the-bulk-upload-template"))
+    end
+  end
+
+  def validate_field_numbers_count
+    return if halt_validations?
+
+    unless csv_parser.correct_field_count?
+      errors.add(:base, :wrong_field_numbers_count)
       halt_validations!
     end
   end
diff --git a/app/services/bulk_upload/sales/year2023/csv_parser.rb b/app/services/bulk_upload/sales/year2023/csv_parser.rb
index 42b0764d9..c6c07e1d5 100644
--- a/app/services/bulk_upload/sales/year2023/csv_parser.rb
+++ b/app/services/bulk_upload/sales/year2023/csv_parser.rb
@@ -4,6 +4,7 @@ class BulkUpload::Sales::Year2023::CsvParser
   include CollectionTimeHelper
 
   MAX_COLUMNS = 142
+  FIELDS = 135
   FORM_YEAR = 2023
 
   attr_reader :path
@@ -59,6 +60,12 @@ class BulkUpload::Sales::Year2023::CsvParser
     false
   end
 
+  def correct_field_count?
+    valid_field_numbers_count = field_numbers.count { |f| f != "field_blank" }
+
+    valid_field_numbers_count == FIELDS
+  end
+
 private
 
   def default_field_numbers
diff --git a/app/services/bulk_upload/sales/year2024/csv_parser.rb b/app/services/bulk_upload/sales/year2024/csv_parser.rb
index e53f0ec28..2dc9d38a1 100644
--- a/app/services/bulk_upload/sales/year2024/csv_parser.rb
+++ b/app/services/bulk_upload/sales/year2024/csv_parser.rb
@@ -3,6 +3,7 @@ require "csv"
 class BulkUpload::Sales::Year2024::CsvParser
   include CollectionTimeHelper
 
+  FIELDS = 131
   MAX_COLUMNS = 142
   FORM_YEAR = 2024
 
@@ -59,6 +60,12 @@ class BulkUpload::Sales::Year2024::CsvParser
     !with_headers?
   end
 
+  def correct_field_count?
+    valid_field_numbers_count = field_numbers.count { |f| f != "field_blank" }
+
+    valid_field_numbers_count == FIELDS
+  end
+
 private
 
   def default_field_numbers
diff --git a/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb b/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb
index 72c19e5d1..6738eb3f3 100644
--- a/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb
+++ b/spec/services/bulk_upload/sales/year2023/csv_parser_spec.rb
@@ -28,6 +28,10 @@ RSpec.describe BulkUpload::Sales::Year2023::CsvParser do
     it "parses csv correctly" do
       expect(service.row_parsers[0].field_19).to eql(log.uprn)
     end
+
+    it "counts the number of valid field numbers correctly" do
+      expect(service).to be_correct_field_count
+    end
   end
 
   context "when parsing csv with headers in arbitrary order" do
diff --git a/spec/services/bulk_upload/sales/year2024/csv_parser_spec.rb b/spec/services/bulk_upload/sales/year2024/csv_parser_spec.rb
index e4391212a..ef90bd834 100644
--- a/spec/services/bulk_upload/sales/year2024/csv_parser_spec.rb
+++ b/spec/services/bulk_upload/sales/year2024/csv_parser_spec.rb
@@ -28,6 +28,10 @@ RSpec.describe BulkUpload::Sales::Year2024::CsvParser do
     it "parses csv correctly" do
       expect(service.row_parsers[0].field_22).to eql(log.uprn)
     end
+
+    it "counts the number of valid field numbers correctly" do
+      expect(service).to be_correct_field_count
+    end
   end
 
   context "when parsing csv with headers in arbitrary order" do