Merge branch 'main' into CLDC-3619-duplicate-schemes-after-merge

.github/workflows/staging_pipeline.yml

@@ -14,6 +14,11 @@ defaults:
   run:
     shell: bash
 
+env:
+  app_repo_role: arn:aws:iam::815624722760:role/core-application-repo
+  aws_region: eu-west-2
+  repository: core
+
 jobs:
   test:
     name: Tests
@@ -364,3 +369,48 @@ jobs:
       environment: staging
     permissions:
       id-token: write
+
+  performance:
+    needs: [aws_deploy]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-region: ${{ env.aws_region }}
+          role-to-assume: ${{ env.app_repo_role }}
+
+      - name: Configure AWS credentials for the environment
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::107155005276:role/core-staging-deployment
+          role-chaining: true
+
+      - name: Run Performance Test
+        env:
+          ad_hoc_task_definition: core-staging-ad-hoc
+          cluster: core-staging-app
+          service: core-staging-app
+        run: |
+          echo $cluster
+          network=$(aws ecs describe-services --cluster $cluster --services $service --query services[0].networkConfiguration)
+          overrides='{
+            "containerOverrides": [{
+              "name": "app",
+              "command": ["bash", "-c", "export email=$STAGING_PERFORMANCE_TEST_EMAIL && export password=$STAGING_PERFORMANCE_TEST_PASSWORD && sh ./lib/tasks/performance_test.sh"]
+            }]
+          }'
+          arn=$(aws ecs run-task --cluster $cluster --task-definition $ad_hoc_task_definition --network-configuration "$network" --overrides "$overrides" --group performance --launch-type FARGATE --query tasks[0].taskArn)
+
+          echo "Waiting for performance tests to run"
+          task_id=${arn##*/}
+          task_id=${task_id%*\"}
+
+          aws ecs wait tasks-stopped --cluster $cluster --tasks $task_id
+
+          code=$(aws ecs describe-tasks --cluster $cluster --tasks $task_id --query "tasks[0].containers[0].exitCode")
+          if [ "$code == 0" ]; then exit 0; else exit 1; fi

Dockerfile

@@ -31,6 +31,9 @@ EXPOSE ${PORT}
 
 RUN adduser --system --no-create-home nonroot
 
+RUN apk add curl
+RUN apk add apache2-utils
+
 FROM base as test
 
 RUN bundle config set without ""
@@ -67,6 +70,9 @@ RUN mkdir -p tmp log
 RUN chown -R nonroot tmp log
 RUN chown nonroot db/schema.rb
 
+RUN mkdir -p performance_test
+RUN chown -R nonroot performance_test
+
 USER nonroot
 
 CMD bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0

app/components/bulk_upload_error_row_component.html.erb

@@ -9,6 +9,8 @@
 
   <div class="govuk-summary-card__content">
     <% potential_errors, critical_errors = bulk_upload_errors.partition { |error| error.category == "soft_validation" } %>
+
+    <% if critical_errors.any? %>
       <h2 class="govuk-heading-m">Critical errors</h2>
       <p class="govuk-body">These errors must be fixed to complete your logs.</p>
       <%= govuk_table do |table| %>
@@ -32,6 +34,7 @@
           <% end %>
         <% end %>
       <% end %>
+    <% end %>
 
     <% if potential_errors.any? %>
       <h2 class="govuk-heading-m">Potential errors</h2>

app/components/create_log_actions_component.html.erb

@@ -3,7 +3,7 @@
     <% if create_button_href.present? %>
       <%= govuk_button_to create_button_copy, create_button_href, class: "govuk-!-margin-right-6" %>
     <% end %>
-    <% if upload_button_href.present? %>
+    <% if upload_button_href.present? && !user.support? %>
       <%= govuk_button_link_to upload_button_copy, upload_button_href, secondary: true %>
     <% end %>
   </div>

app/controllers/bulk_upload_lettings_logs_controller.rb

@@ -4,9 +4,9 @@ class BulkUploadLettingsLogsController < ApplicationController
 
   def start
     if have_choice_of_year?
-      redirect_to bulk_upload_lettings_log_path(id: "year")
+      redirect_to bulk_upload_lettings_log_path(id: "year", form: { organisation_id: params[:organisation_id] }.compact)
     else
-      redirect_to bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: current_year })
+      redirect_to bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: current_year, organisation_id: params[:organisation_id] }.compact)
     end
   end
 
@@ -60,6 +60,6 @@ private
   end
 
   def form_params
-    params.fetch(:form, {}).permit(:year, :needstype, :file)
+    params.fetch(:form, {}).permit(:year, :needstype, :file, :organisation_id)
   end
 end

app/controllers/bulk_upload_sales_logs_controller.rb

@@ -4,9 +4,9 @@ class BulkUploadSalesLogsController < ApplicationController
 
   def start
     if have_choice_of_year?
-      redirect_to bulk_upload_sales_log_path(id: "year")
+      redirect_to bulk_upload_sales_log_path(id: "year", form: { organisation_id: params[:organisation_id] }.compact)
     else
-      redirect_to bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: current_year })
+      redirect_to bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: current_year, organisation_id: params[:organisation_id] }.compact)
     end
   end
 
@@ -58,6 +58,6 @@ private
   end
 
   def form_params
-    params.fetch(:form, {}).permit(:year, :file)
+    params.fetch(:form, {}).permit(:year, :file, :organisation_id)
   end
 end

app/helpers/application_helper.rb

@@ -2,6 +2,7 @@ module ApplicationHelper
   include Pagy::Frontend
 
   def browser_title(title, pagy, *resources)
+    title = sanitize(title)&.gsub("&", "&")
     if resources.any? { |r| r.present? && r.errors.present? }
       "Error: #{[title, t('service_name'), 'GOV.UK'].select(&:present?).join(' - ')}"
     else

app/helpers/user_helper.rb

@@ -56,4 +56,23 @@ module UserHelper
       user.errors.add(attribute, message)
     end
   end
+
+  def display_pending_email_change_banner?(user)
+    user.unconfirmed_email.present? && user.email != user.unconfirmed_email
+  end
+
+  def pending_email_change_title_text(current_user, user)
+    if current_user == user
+      "You have requested to change your email address to #{user.unconfirmed_email}."
+    else
+      "There has been a request to change this user’s email address to #{user.unconfirmed_email}."
+    end
+  end
+
+  def pending_email_change_banner_text(current_user)
+    text = "A confirmation link has been sent to the new email address. The current email will continue to work until the change is confirmed."
+    text += " Deactivating this user will cancel the email change request." if current_user.support? || current_user.data_coordinator?
+
+    text
+  end
 end

app/models/forms/bulk_upload_lettings/checking_file.rb

@@ -6,14 +6,19 @@ module Forms
       include Rails.application.routes.url_helpers
 
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       def view_path
         "bulk_upload_lettings_logs/forms/checking_file"
       end
 
       def back_path
+        if organisation_id.present?
+          lettings_logs_organisation_path(organisation_id)
+        else
           bulk_upload_lettings_log_path(id: "start")
         end
+      end
 
       def year_combo
         "#{year}/#{year + 1 - 2000}"

app/models/forms/bulk_upload_lettings/guidance.rb

@@ -7,6 +7,7 @@ module Forms
 
       attribute :year, :integer
       attribute :referrer
+      attribute :organisation_id, :integer
 
       def view_path
         "bulk_upload_shared/guidance"
@@ -15,7 +16,7 @@ module Forms
       def back_path
         case referrer
         when "prepare-your-file"
-          bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: })
+          bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year:, organisation_id: }.compact)
         when "home"
           root_path
         else

app/models/forms/bulk_upload_lettings/needstype.rb

@@ -7,6 +7,7 @@ module Forms
 
       attribute :needstype, :integer
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       validates :needstype, presence: true
 
@@ -19,11 +20,11 @@ module Forms
       end
 
       def back_path
-        bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year:, needstype: })
+        bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year:, needstype:, organisation_id: }.compact)
       end
 
       def next_path
-        bulk_upload_lettings_log_path(id: "upload-your-file", form: { year:, needstype: })
+        bulk_upload_lettings_log_path(id: "upload-your-file", form: { year:, needstype:, organisation_id: }.compact)
       end
 
       def year_combo

app/models/forms/bulk_upload_lettings/prepare_your_file.rb

@@ -7,6 +7,7 @@ module Forms
 
       attribute :year, :integer
       attribute :needstype, :integer
+      attribute :organisation_id, :integer
 
       def view_path
         case year
@@ -19,15 +20,16 @@ module Forms
 
       def back_path
         if have_choice_of_year?
-          Rails.application.routes.url_helpers.bulk_upload_lettings_log_path(id: "year", form: { year: })
+          Rails.application.routes.url_helpers.bulk_upload_lettings_log_path(id: "year", form: { year:, organisation_id: }.compact)
+        elsif organisation_id.present?
+          lettings_logs_organisation_path(organisation_id)
         else
           Rails.application.routes.url_helpers.lettings_logs_path
         end
       end
 
       def next_path
-        page_id = year == 2022 ? "needstype" : "upload-your-file"
+        bulk_upload_lettings_log_path(id: "upload-your-file", form: { year:, needstype:, organisation_id: }.compact)
-        bulk_upload_lettings_log_path(id: page_id, form: { year:, needstype: })
       end
 
       def legacy_template_path

app/models/forms/bulk_upload_lettings/upload_your_file.rb

@@ -11,6 +11,7 @@ module Forms
       attribute :needstype, :integer
       attribute :file
       attribute :current_user
+      attribute :organisation_id, :integer
 
       validates :file, presence: true
       validate :validate_file_is_csv
@@ -20,8 +21,7 @@ module Forms
       end
 
       def back_path
-        page_id = year == 2022 ? "needstype" : "prepare-your-file"
+        bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year:, needstype:, organisation_id: }.compact)
-        bulk_upload_lettings_log_path(id: page_id, form: { year:, needstype: })
       end
 
       def year_combo
@@ -29,7 +29,7 @@ module Forms
       end
 
       def next_path
-        bulk_upload_lettings_log_path(id: "checking-file", form: { year: })
+        bulk_upload_lettings_log_path(id: "checking-file", form: { year:, organisation_id: }.compact)
       end
 
       def save!
@@ -39,6 +39,7 @@ module Forms
           year:,
           needstype:,
           filename: file.original_filename,
+          organisation_id: (organisation_id if current_user.support?) || current_user.organisation_id,
         )
 
         storage_service.write_file(bulk_upload.identifier, File.read(file.path))

app/models/forms/bulk_upload_lettings/year.rb

@@ -6,6 +6,7 @@ module Forms
       include Rails.application.routes.url_helpers
 
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       validates :year, presence: true
 
@@ -20,11 +21,15 @@ module Forms
       end
 
       def back_path
+        if organisation_id.present?
+          lettings_logs_organisation_path(organisation_id)
+        else
           lettings_logs_path
         end
+      end
 
       def next_path
-        bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year: })
+        bulk_upload_lettings_log_path(id: "prepare-your-file", form: { year:, organisation_id: }.compact)
       end
 
       def save!

app/models/forms/bulk_upload_sales/checking_file.rb

@@ -6,14 +6,19 @@ module Forms
       include Rails.application.routes.url_helpers
 
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       def view_path
         "bulk_upload_sales_logs/forms/checking_file"
       end
 
       def back_path
+        if organisation_id.present?
+          sales_logs_organisation_path(organisation_id)
+        else
           bulk_upload_sales_log_path(id: "start")
         end
+      end
 
       def year_combo
         "#{year}/#{year + 1 - 2000}"

app/models/forms/bulk_upload_sales/guidance.rb

@@ -7,6 +7,7 @@ module Forms
 
       attribute :year, :integer
       attribute :referrer
+      attribute :organisation_id, :integer
 
       def view_path
         "bulk_upload_shared/guidance"
@@ -15,7 +16,7 @@ module Forms
       def back_path
         case referrer
         when "prepare-your-file"
-          bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: })
+          bulk_upload_sales_log_path(id: "prepare-your-file", form: { year:, organisation_id: }.compact)
         when "home"
           root_path
         else

app/models/forms/bulk_upload_sales/prepare_your_file.rb

@@ -6,6 +6,7 @@ module Forms
       include Rails.application.routes.url_helpers
 
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       def view_path
         case year
@@ -18,14 +19,16 @@ module Forms
 
       def back_path
         if have_choice_of_year?
-          Rails.application.routes.url_helpers.bulk_upload_sales_log_path(id: "year", form: { year: })
+          Rails.application.routes.url_helpers.bulk_upload_sales_log_path(id: "year", form: { year: }.compact)
+        elsif organisation_id.present?
+          sales_logs_organisation_path(organisation_id)
         else
           Rails.application.routes.url_helpers.sales_logs_path
         end
       end
 
       def next_path
-        bulk_upload_sales_log_path(id: "upload-your-file", form: { year: })
+        bulk_upload_sales_log_path(id: "upload-your-file", form: { year:, organisation_id: }.compact)
       end
 
       def legacy_template_path

app/models/forms/bulk_upload_sales/upload_your_file.rb

@@ -10,6 +10,7 @@ module Forms
       attribute :year, :integer
       attribute :file
       attribute :current_user
+      attribute :organisation_id, :integer
 
       validates :file, presence: true
       validate :validate_file_is_csv
@@ -19,7 +20,7 @@ module Forms
       end
 
       def back_path
-        bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: })
+        bulk_upload_sales_log_path(id: "prepare-your-file", form: { year:, organisation_id: }.compact)
       end
 
       def year_combo
@@ -27,7 +28,7 @@ module Forms
       end
 
       def next_path
-        bulk_upload_sales_log_path(id: "checking-file", form: { year: })
+        bulk_upload_sales_log_path(id: "checking-file", form: { year:, organisation_id: }.compact)
       end
 
       def save!
@@ -36,6 +37,7 @@ module Forms
           log_type: BulkUpload.log_types[:sales],
           year:,
           filename: file.original_filename,
+          organisation_id: (organisation_id if current_user.support?) || current_user.organisation_id,
         )
 
         storage_service.write_file(bulk_upload.identifier, File.read(file.path))

app/models/forms/bulk_upload_sales/year.rb

@@ -6,6 +6,7 @@ module Forms
       include Rails.application.routes.url_helpers
 
       attribute :year, :integer
+      attribute :organisation_id, :integer
 
       validates :year, presence: true
 
@@ -20,11 +21,15 @@ module Forms
       end
 
       def back_path
+        if organisation_id.present?
+          sales_logs_organisation_path(organisation_id)
+        else
           sales_logs_path
         end
+      end
 
       def next_path
-        bulk_upload_sales_log_path(id: "prepare-your-file", form: { year: })
+        bulk_upload_sales_log_path(id: "prepare-your-file", form: { year:, organisation_id: }.compact)
       end
 
       def save!

app/services/bulk_upload/lettings/year2024/row_parser.rb

@@ -432,6 +432,7 @@ class BulkUpload::Lettings::Year2024::RowParser
 
   validate :validate_assigned_to_exists, on: :after_log
   validate :validate_assigned_to_related, on: :after_log
+  validate :validate_assigned_to_when_support, on: :after_log
   validate :validate_all_charges_given, on: :after_log, if: proc { is_carehome.zero? }
 
   validate :validate_address_option_found, on: :after_log, unless: -> { supported_housing? }
@@ -582,6 +583,12 @@ private
     end
   end
 
+  def validate_assigned_to_when_support
+    if field_3.blank? && bulk_upload.user.support?
+      errors.add(:field_3, category: :setup, message: I18n.t("validations.not_answered", question: "what is the CORE username of the account this letting log should be assigned to?"))
+    end
+  end
+
   def validate_assigned_to_related
     return unless assigned_to
     return if assigned_to.organisation == owning_organisation || assigned_to.organisation == managing_organisation
@@ -891,14 +898,19 @@ private
   end
 
   def validate_owning_org_permitted
-    if owning_organisation && !bulk_upload.user.organisation.affiliated_stock_owners.include?(owning_organisation)
+    return unless owning_organisation
+    return if bulk_upload_organisation.affiliated_stock_owners.include?(owning_organisation)
+
     block_log_creation!
 
-      if errors[:field_1].blank?
+    return if errors[:field_1].present?
+
+    if bulk_upload.user.support?
+      errors.add(:field_1, "This owning organisation is not affiliated with #{bulk_upload_organisation.name}", category: :setup)
+    else
       errors.add(:field_1, "You do not have permission to add logs for this owning organisation", category: :setup)
     end
   end
-  end
 
   def validate_all_charges_given
     return if supported_housing? && field_125 == 1
@@ -1137,7 +1149,7 @@ private
     attributes["renewal"] = renewal
     attributes["scheme"] = scheme
     attributes["location"] = location
-    attributes["assigned_to"] = assigned_to || bulk_upload.user
+    attributes["assigned_to"] = assigned_to || (bulk_upload.user.support? ? nil : bulk_upload.user)
     attributes["created_by"] = bulk_upload.user
     attributes["needstype"] = field_4
     attributes["rent_type"] = RENT_TYPE_BU_MAPPING[field_11]
@@ -1625,4 +1637,8 @@ private
   def reason_is_other?
     field_98 == 20
   end
+
+  def bulk_upload_organisation
+    Organisation.find(bulk_upload.organisation_id)
+  end
 end

app/services/bulk_upload/sales/year2024/row_parser.rb

@@ -462,6 +462,7 @@ class BulkUpload::Sales::Year2024::RowParser
 
   validate :validate_assigned_to_exists, on: :after_log
   validate :validate_assigned_to_related, on: :after_log
+  validate :validate_assigned_to_when_support, on: :after_log
   validate :validate_managing_org_related, on: :after_log
   validate :validate_relevant_collection_window, on: :after_log
   validate :validate_incomplete_soft_validations, on: :after_log
@@ -925,7 +926,7 @@ private
 
     attributes["owning_organisation"] = owning_organisation
     attributes["managing_organisation"] = managing_organisation
-    attributes["assigned_to"] = assigned_to || bulk_upload.user
+    attributes["assigned_to"] = assigned_to || (bulk_upload.user.support? ? nil : bulk_upload.user)
     attributes["created_by"] = bulk_upload.user
     attributes["hhregres"] = field_72
     attributes["hhregresstill"] = field_73
@@ -1302,14 +1303,19 @@ private
   end
 
   def validate_owning_org_permitted
-    if owning_organisation && !bulk_upload.user.organisation.affiliated_stock_owners.include?(owning_organisation)
+    return unless owning_organisation
+    return if bulk_upload_organisation.affiliated_stock_owners.include?(owning_organisation)
+
     block_log_creation!
 
-      if errors[:field_1].blank?
+    return if errors[:field_1].present?
+
+    if bulk_upload.user.support?
+      errors.add(:field_1, "This owning organisation is not affiliated with #{bulk_upload_organisation.name}", category: :setup)
+    else
       errors.add(:field_1, "You do not have permission to add logs for this owning organisation", category: :setup)
     end
   end
-  end
 
   def validate_assigned_to_exists
     return if field_3.blank?
@@ -1319,6 +1325,12 @@ private
     end
   end
 
+  def validate_assigned_to_when_support
+    if field_3.blank? && bulk_upload.user.support?
+      errors.add(:field_3, category: :setup, message: I18n.t("validations.not_answered", question: "what is the CORE username of the account this sales log should be assigned to?"))
+    end
+  end
+
   def validate_assigned_to_related
     return unless assigned_to
     return if assigned_to.organisation == owning_organisation || assigned_to.organisation == managing_organisation
@@ -1492,4 +1504,8 @@ private
   def valid_nationality_options
     %w[0] + GlobalConstants::COUNTRIES_ANSWER_OPTIONS.keys # 0 is "Prefers not to say"
   end
+
+  def bulk_upload_organisation
+    Organisation.find(bulk_upload.organisation_id)
+  end
 end

app/views/bulk_upload_lettings_logs/forms/checking_file.html.erb

@@ -6,6 +6,7 @@
   <div class="govuk-grid-column-two-thirds">
     <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "prepare-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %>
 
       <span class="govuk-caption-l">Upload lettings logs in bulk (<%= @form.year_combo %>)</span>
       <h1 class="govuk-heading-l">We’re checking the file</h1>

app/views/bulk_upload_lettings_logs/forms/needstype.erb

@@ -7,6 +7,7 @@
     <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "needstype"), method: :patch do |f| %>
       <%= f.govuk_error_summary %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %>
 
       <%= f.govuk_collection_radio_buttons :needstype,
             @form.options,

app/views/bulk_upload_lettings_logs/forms/prepare_your_file_2024.html.erb

@@ -6,6 +6,7 @@
   <div class="govuk-grid-column-two-thirds">
     <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "prepare-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %>
 
       <span class="govuk-caption-l">Upload lettings logs in bulk (<%= @form.year_combo %>)</span>
       <h1 class="govuk-heading-l">Prepare your file</h1>

app/views/bulk_upload_lettings_logs/forms/upload_your_file.html.erb

@@ -7,6 +7,7 @@
     <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "upload-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
       <%= f.hidden_field :needstype %>
+      <%= f.hidden_field :organisation_id %>
 
       <%= f.govuk_error_summary %>
 

app/views/bulk_upload_lettings_logs/forms/year.html.erb

@@ -4,6 +4,7 @@
 
 <%= form_with model: @form, scope: :form, url: bulk_upload_lettings_log_path(id: "year"), method: :patch do |f| %>
   <%= f.govuk_error_summary %>
+  <%= f.hidden_field :organisation_id %>
 
   <%= f.govuk_collection_radio_buttons :year,
         @form.options,

app/views/bulk_upload_lettings_results/show.html.erb

@@ -25,4 +25,4 @@
   </div>
 </div>
 
-<%= govuk_button_link_to "Upload your file again", start_bulk_upload_lettings_logs_path %>
+<%= govuk_button_link_to "Upload your file again", start_bulk_upload_lettings_logs_path(organisation_id: @bulk_upload.organisation_id) %>

app/views/bulk_upload_lettings_results/summary.html.erb

@@ -29,4 +29,4 @@
   <% end %>
 </div>
 
-<%= govuk_button_link_to "Upload your file again", start_bulk_upload_lettings_logs_path %>
+<%= govuk_button_link_to "Upload your file again", start_bulk_upload_lettings_logs_path(organisation_id: @bulk_upload.organisation_id) %>

app/views/bulk_upload_sales_logs/forms/checking_file.html.erb

@@ -6,6 +6,7 @@
   <div class="govuk-grid-column-two-thirds">
     <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "prepare-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %><%= f.hidden_field :organisation_id %>
 
       <span class="govuk-caption-l">Upload sales logs in bulk (<%= @form.year_combo %>)</span>
       <h1 class="govuk-heading-l">We’re checking the file</h1>

app/views/bulk_upload_sales_logs/forms/prepare_your_file_2024.html.erb

@@ -6,6 +6,7 @@
   <div class="govuk-grid-column-two-thirds">
     <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "prepare-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %>
 
       <span class="govuk-caption-l">Upload sales logs in bulk (<%= @form.year_combo %>)</span>
       <h1 class="govuk-heading-l">Prepare your file</h1>

app/views/bulk_upload_sales_logs/forms/upload_your_file.html.erb

@@ -6,6 +6,7 @@
   <div class="govuk-grid-column-two-thirds">
     <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "upload-your-file"), method: :patch do |f| %>
       <%= f.hidden_field :year %>
+      <%= f.hidden_field :organisation_id %>
 
       <%= f.govuk_error_summary %>
 

app/views/bulk_upload_sales_logs/forms/year.html.erb

@@ -4,6 +4,7 @@
 
 <%= form_with model: @form, scope: :form, url: bulk_upload_sales_log_path(id: "year"), method: :patch do |f| %>
   <%= f.govuk_error_summary %>
+  <%= f.hidden_field :organisation_id %>
 
   <%= f.govuk_collection_radio_buttons :year,
         @form.options,

app/views/bulk_upload_sales_results/show.html.erb

@@ -25,4 +25,4 @@
   </div>
 </div>
 
-<%= govuk_button_link_to "Upload your file again", start_bulk_upload_sales_logs_path %>
+<%= govuk_button_link_to "Upload your file again", start_bulk_upload_sales_logs_path(organisation_id: @bulk_upload.organisation_id) %>

app/views/bulk_upload_sales_results/summary.html.erb

@@ -29,4 +29,4 @@
   <% end %>
 </div>
 
-<%= govuk_button_link_to "Upload your file again", start_bulk_upload_sales_logs_path %>
+<%= govuk_button_link_to "Upload your file again", start_bulk_upload_sales_logs_path(organisation_id: @bulk_upload.organisation_id) %>

app/views/logs/_create_for_org_actions.html.erb

@@ -1,10 +1,12 @@
 <div class="govuk-button-group app-filter-toggle">
   <% if @organisation.data_protection_confirmed? %>
     <% if current_page?(controller: 'organisations', action: 'lettings_logs') %>
-      <%= govuk_button_to "Create a new lettings log for this organisation", lettings_logs_path(lettings_log: { owning_organisation_id: @organisation.id }, method: :post) %>
+      <%= govuk_button_to "Create a new lettings log for this organisation", lettings_logs_path(lettings_log: { owning_organisation_id: @organisation.id }, method: :post), class: "govuk-!-margin-right-6" %>
+      <%= govuk_button_link_to "Upload lettings logs in bulk", bulk_upload_lettings_log_path(id: "start", organisation_id: @organisation.id), secondary: true %>
 <% end %>
     <% if current_page?(controller: 'organisations', action: 'sales_logs') %>
-      <%= govuk_button_to "Create a new sales log for this organisation", sales_logs_path(sales_log: { owning_organisation_id: @organisation.id }, method: :post) %>
+      <%= govuk_button_to "Create a new sales log for this organisation", sales_logs_path(sales_log: { owning_organisation_id: @organisation.id }, method: :post), class: "govuk-!-margin-right-6" %>
+      <%= govuk_button_link_to "Upload sales logs in bulk", bulk_upload_sales_log_path(id: "start", organisation_id: @organisation.id), secondary: true %>
 <% end %>
   <% end %>
 </div>

app/views/schemes/index.html.erb

@@ -6,7 +6,9 @@
 <%= render partial: "organisations/headings", locals: current_user.support? ? { main: "Supported housing schemes", sub: nil } : { main: "Supported housing schemes", sub: current_user.organisation.name } %>
 
 <% if SchemePolicy.new(current_user, nil).create? %>
+  <div class="govuk-button-group govuk-!-margin-bottom-6">
     <%= govuk_button_link_to "Create a new supported housing scheme", new_scheme_path, html: { method: :post } %>
+  </div>
 <% end %>
 <div class="app-filter-layout" data-controller="filter-layout">
   <%= render partial: "schemes/scheme_filters" %>

app/views/users/index.html.erb

@@ -6,7 +6,9 @@
 <%= render partial: "organisations/headings", locals: current_user.support? ? { main: "Users", sub: nil } : { main: "Users", sub: current_user.organisation.name } %>
 
 <% if current_user.data_coordinator? || current_user.support? %>
+  <div class="govuk-button-group govuk-!-margin-bottom-6">
     <%= govuk_button_link_to "Invite user", new_user_path, html: { method: :get } %>
+  </div>
 <% end %>
 <div class="app-filter-layout" data-controller="filter-layout">
   <%= render partial: "users/user_filters" %>

app/views/users/show.html.erb

@@ -11,6 +11,15 @@
   <% end %>
 <% end %>
 
+<% if display_pending_email_change_banner?(@user) %>
+  <%= govuk_notification_banner(title_text: "Important") do %>
+    <p class="govuk-notification-banner__heading govuk-!-width-full" style="max-width: fit-content">
+      <%= pending_email_change_title_text(current_user, @user) %>
+    </p>
+    <%= pending_email_change_banner_text(current_user) %>
+  <% end %>
+<% end %>
+
 <div class="govuk-grid-row">
   <div class="govuk-grid-column-two-thirds">
     <h1 class="govuk-heading-l">

db/migrate/20240905092332_add_organisation_id_to_bulk_uploads.rb

@@ -0,0 +1,5 @@
+class AddOrganisationIdToBulkUploads < ActiveRecord::Migration[7.0]
+  def change
+    add_column :bulk_uploads, :organisation_id, :integer
+  end
+end

db/migrate/20240923145326_add_validation_checked_field.rb

@@ -0,0 +1,5 @@
+class AddValidationCheckedField < ActiveRecord::Migration[7.0]
+  def change
+    add_column :log_validations, :checked, :boolean
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2024_09_20_144611) do
+ActiveRecord::Schema[7.0].define(version: 2024_09_23_145326) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -42,6 +42,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_09_20_144611) do
     t.text "choice"
     t.integer "total_logs_count"
     t.string "rent_type_fix_status", default: "not_applied"
+    t.integer "organisation_id"
     t.integer "moved_user_id"
     t.index ["identifier"], name: "index_bulk_uploads_on_identifier", unique: true
     t.index ["user_id"], name: "index_bulk_uploads_on_user_id"
@@ -410,6 +411,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_09_20_144611) do
     t.string "other_validated_models"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.boolean "checked"
   end
 
   create_table "logs_exports", force: :cascade do |t|

lib/tasks/performance_test.sh

@@ -0,0 +1,124 @@
+cd performance_test
+
+# Lettings logs page
+echo "Get token"
+TOKEN=$(curl -c token_cookies.txt -s https://staging.submit-social-housing-data.levellingup.gov.uk/account/sign-in | grep '<meta name="csrf-token"' | sed -n 's/.*content="\([^"]*\)".*/\1/p')
+
+echo "Logging in..."
+echo $email
+curl -L -o nul -c login_cookies.txt -b token_cookies.txt -X POST https://staging.submit-social-housing-data.levellingup.gov.uk/account/sign-in \
+  -d "user[email]=$email" \
+  -d "user[password]=$password" \
+  -d "authenticity_token=$TOKEN"
+
+COOKIES=$(awk '/_data_collector_session/ { print $6, $7 }' login_cookies.txt | tr ' ' '=')
+
+echo "Running lettings logs page performance test..."
+ab -n 50 -c 50 -l -C "$COOKIES" 'https://staging.submit-social-housing-data.levellingup.gov.uk/lettings-logs?years[]=2024&status[]=completed' > performance_lettings_test_results.txt
+file="performance_lettings_test_results.txt"
+
+failed_requests=$(grep "Failed requests:" "$file" | awk '{print $3}')
+non_2xx_responses=$(grep "Non-2xx responses:" "$file" | awk '{print $3}')
+time_per_request_all=$(grep "Time per request:" "$file" | awk 'NR==2{print $4}')
+requests_per_second=$(grep "Requests per second:" "$file" | awk '{print $4}')
+
+
+if [ "$failed_requests" -gt 0 ]; then
+  echo "Lettings logs: Performance test failed - $failed_requests failed requests"
+  exit 1
+fi
+
+if [ "$non_2xx_responses" -ne 0 ] && [ -n "$non_2xx_responses" ]; then
+  echo "Lettings logs: Performance test failed: There were $non_2xx_responses non-2xx responses."
+  exit 1
+fi
+
+if (( $(echo "$time_per_request_all > 250" | bc -l) )); then
+  echo "Lettings logs: Performance test failed - Time per request across all concurrent requests is more than 250 ms: $time_per_request_all ms"
+  exit 1
+fi
+
+if (( $(echo "$requests_per_second < 5" | bc -l) )); then
+  echo "Lettings logs: Performance test failed - Requests per second is less than 5: $requests_per_second"
+  exit 1
+fi
+
+echo "Lettings logs page test passed: No failed requests and no non-2xx responses."
+
+
+# Sales logs page
+echo "Running sales logs page performance test..."
+ab -n 50 -c 50 -l -C "$COOKIES" 'https://staging.submit-social-housing-data.levellingup.gov.uk/sales-logs?years[]=2024&status[]=completed' > performance_sales_test_results.txt
+file="performance_sales_test_results.txt"
+
+failed_requests=$(grep "Failed requests:" "$file" | awk '{print $3}')
+non_2xx_responses=$(grep "Non-2xx responses:" "$file" | awk '{print $3}')
+time_per_request_all=$(grep "Time per request:" "$file" | awk 'NR==2{print $4}')
+requests_per_second=$(grep "Requests per second:" "$file" | awk '{print $4}')
+
+
+if [ "$failed_requests" -gt 0 ]; then
+  echo "Sales logs: Performance test failed - $failed_requests failed requests"
+  exit 1
+fi
+
+if [ "$non_2xx_responses" -ne 0 ] && [ -n "$non_2xx_responses" ]; then
+  echo "Sales logs: Performance test failed: There were $non_2xx_responses non-2xx responses."
+  exit 1
+fi
+
+if (( $(echo "$time_per_request_all > 250" | bc -l) )); then
+  echo "Sales logs: Performance test failed - Time per request across all concurrent requests is more than 250 ms: $time_per_request_all ms"
+  exit 1
+fi
+
+if (( $(echo "$requests_per_second < 5" | bc -l) )); then
+  echo "Sales logs: Performance test failed - Requests per second is less than 5: $requests_per_second"
+  exit 1
+fi
+
+echo "Sales logs page test passed: No failed requests and no non-2xx responses."
+
+
+# Post data to a log test
+page_content=$(curl -b login_cookies.txt -s 'https://staging.submit-social-housing-data.levellingup.gov.uk/lettings-logs?years[]=2024&status[]=completed')
+completed_log_link=$(echo "$page_content" | sed -n 's/.*<a class="govuk-link" href="\([^"]*lettings-logs[^"]*\)".*/\1/p' | head -n 1)
+echo "testing post to $completed_log_link"
+
+TOKEN=$(curl -L -b login_cookies.txt -c login_cookies.txt https://staging.submit-social-housing-data.levellingup.gov.uk$completed_log_link/tenant-code | grep '<meta name="csrf-token"' | sed -n 's/.*content="\([^"]*\)".*/\1/p')
+
+COOKIES=$(awk '/_data_collector_session/ { print $6, $7 }' login_cookies.txt | tr ' ' '=')
+
+echo "lettings_log[tenancycode]=performance_test_tenancy_code&lettings_log[page]=tenant_code&authenticity_token=$TOKEN" > post_data.txt
+
+ab -n 50 -c 50 -l -T application/x-www-form-urlencoded \
+-H "X-CSRF-Token: $TOKEN" \
+-C "$COOKIES" \
+-p post_data.txt \
+"https://staging.submit-social-housing-data.levellingup.gov.uk$completed_log_link/tenant-code" > performance_post_test_results.txt
+
+file="performance_post_test_results.txt"
+failed_requests=$(grep "Failed requests:" "$file" | awk '{print $3}')
+time_per_request_all=$(grep "Time per request:" "$file" | awk 'NR==2{print $4}')
+requests_per_second=$(grep "Requests per second:" "$file" | awk '{print $4}')
+
+
+if [ "$failed_requests" -gt 0 ]; then
+  echo "Update logs: Performance test failed - $failed_requests failed requests"
+  exit 1
+fi
+
+if (( $(echo "$time_per_request_all > 500" | bc -l) )); then
+  echo "Update logs: Performance test failed - Time per request across all concurrent requests is more than 500 ms: $time_per_request_all ms"
+  exit 1
+fi
+
+if (( $(echo "$requests_per_second < 3" | bc -l) )); then
+  echo "Update logs: Performance test failed - Requests per second is less than 3: $requests_per_second"
+  exit 1
+fi
+
+echo "Update logs test passed: No failed requests and request times as expected."
+
+echo "All tests passed"
+exit 0

spec/factories/bulk_upload.rb

@@ -9,6 +9,7 @@ FactoryBot.define do
     sequence(:filename) { |n| "bulk-upload-#{n}.csv" }
     needstype { 1 }
     rent_type_fix_status { BulkUpload.rent_type_fix_statuses.values.sample }
+    organisation_id { user.organisation_id }
 
     trait(:sales) do
       log_type { BulkUpload.log_types[:sales] }

spec/features/organisation_spec.rb

@@ -139,6 +139,10 @@ RSpec.describe "User Features" do
         expect(page).to have_button("Create a new lettings log for this organisation")
       end
 
+      it "shows a upload lettings logs in bulk link" do
+        expect(page).to have_link("Upload lettings logs in bulk")
+      end
+
       context "when creating a log for that organisation" do
         it "pre-fills the value for owning organisation for that log" do
           click_button("Create a new lettings log for this organisation")
@@ -230,6 +234,10 @@ RSpec.describe "User Features" do
         expect(page).to have_button("Create a new sales log for this organisation")
       end
 
+      it "shows a upload sales logs in bulk link" do
+        expect(page).to have_link("Upload sales logs in bulk")
+      end
+
       context "when creating a log for that organisation" do
         it "pre-fills the value for owning organisation for that log" do
           click_button("Create a new sales log for this organisation")

spec/helpers/user_helper_spec.rb

@@ -105,6 +105,32 @@ RSpec.describe UserHelper do
     end
   end
 
+  describe "display_pending_email_change_banner?" do
+    context "when the user doesn't have an unconfirmed email" do
+      let(:user) { FactoryBot.create(:user, :data_provider, unconfirmed_email: nil) }
+
+      it "does not display pending email change banner" do
+        expect(display_pending_email_change_banner?(user)).to be false
+      end
+    end
+
+    context "when the user has the same unconfirmed email as current email" do
+      let(:user) { FactoryBot.create(:user, :data_provider, unconfirmed_email: "updated_email@example.com", email: "updated_email@example.com") }
+
+      it "does not display pending email change banner" do
+        expect(display_pending_email_change_banner?(user)).to be false
+      end
+    end
+
+    context "when the user has a different unconfirmed email" do
+      let(:user) { FactoryBot.create(:user, :data_provider, unconfirmed_email: "updated_email@example.com", email: "old_email@example.com") }
+
+      it "displays pending email change banner" do
+        expect(display_pending_email_change_banner?(user)).to be true
+      end
+    end
+  end
+
   describe "organisation_change_confirmation_warning" do
     context "when user owns logs" do
       before do
@@ -147,4 +173,39 @@ RSpec.describe UserHelper do
       end
     end
   end
+
+  describe "pending_email_change_title_text" do
+    let(:user) { FactoryBot.create(:user, :data_provider, unconfirmed_email: "updated_email@example.com", email: "old_email@example.com") }
+    let(:current_user) { FactoryBot.create(:user, :support) }
+
+    context "when viewing own profile" do
+      it "returns the correct text" do
+        expect(pending_email_change_title_text(user, user)).to eq("You have requested to change your email address to updated_email@example.com.")
+      end
+    end
+
+    context "when viewing another user's profile" do
+      it "returns the correct text" do
+        expect(pending_email_change_title_text(current_user, user)).to eq("There has been a request to change this user’s email address to updated_email@example.com.")
+      end
+    end
+  end
+
+  describe "pending_email_change_banner_text" do
+    context "with provider user" do
+      let(:user) { FactoryBot.create(:user, :data_provider) }
+
+      it "returns the correct text" do
+        expect(pending_email_change_banner_text(user)).to eq("A confirmation link has been sent to the new email address. The current email will continue to work until the change is confirmed.")
+      end
+    end
+
+    context "with support user" do
+      let(:user) { FactoryBot.create(:user, :support) }
+
+      it "returns the correct text" do
+        expect(pending_email_change_banner_text(user)).to eq("A confirmation link has been sent to the new email address. The current email will continue to work until the change is confirmed. Deactivating this user will cancel the email change request.")
+      end
+    end
+  end
 end

spec/services/bulk_upload/lettings/year2024/row_parser_spec.rb

@@ -815,6 +815,23 @@ RSpec.describe BulkUpload::Lettings::Year2024::RowParser do
         end
       end
 
+      context "when blank and bulk upload user is support" do
+        let(:bulk_upload) { create(:bulk_upload, :sales, user: create(:user, :support), year: 2024) }
+
+        let(:attributes) { setup_section_params.merge(bulk_upload:, field_3: nil) }
+
+        it "is not permitted" do
+          parser.valid?
+          expect(parser.errors[:field_3]).to be_present
+          expect(parser.errors[:field_3]).to include("You must answer what is the CORE username of the account this letting log should be assigned to?")
+        end
+
+        it "blocks log creation" do
+          parser.valid?
+          expect(parser).to be_block_log_creation
+        end
+      end
+
       context "when user could not be found" do
         let(:attributes) { { bulk_upload:, field_3: "idonotexist@example.com" } }
 
@@ -1439,6 +1456,43 @@ RSpec.describe BulkUpload::Lettings::Year2024::RowParser do
           expect(parser.errors[:field_10]).to include(/Enter a date when the owning and managing organisation was active/)
         end
       end
+
+      context "when user is an unaffiliated non-support user and bulk upload organisation is affiliated with the owning organisation" do
+        let(:affiliated_org) { create(:organisation, :with_old_visible_id) }
+        let(:unaffiliated_user) { create(:user, organisation: create(:organisation)) }
+        let(:attributes) { { bulk_upload:, field_1: affiliated_org.old_visible_id } }
+        let(:organisation_id) { unaffiliated_user.organisation_id }
+
+        before do
+          create(:organisation_relationship, parent_organisation: owning_org, child_organisation: affiliated_org)
+          bulk_upload.update!(organisation_id:, user: unaffiliated_user)
+        end
+
+        it "blocks log creation and adds an error to field_1" do
+          parser = described_class.new(attributes)
+          parser.valid?
+          expect(parser).to be_block_log_creation
+          expect(parser.errors[:field_1]).to include("You do not have permission to add logs for this owning organisation")
+        end
+      end
+
+      context "when user is an unaffiliated support user and bulk upload organisation is affiliated with the owning organisation" do
+        let(:affiliated_org) { create(:organisation, :with_old_visible_id) }
+        let(:unaffiliated_support_user) { create(:user, :support, organisation: create(:organisation)) }
+        let(:attributes) { { bulk_upload:, field_1: affiliated_org.old_visible_id } }
+        let(:organisation_id) { affiliated_org.id }
+
+        before do
+          create(:organisation_relationship, parent_organisation: owning_org, child_organisation: affiliated_org)
+          bulk_upload.update!(organisation_id:, user: unaffiliated_support_user)
+        end
+
+        it "does not block log creation and does not add an error to field_1" do
+          parser = described_class.new(attributes)
+          parser.valid?
+          expect(parser.errors[:field_1]).not_to include("You do not have permission to add logs for this owning organisation")
+        end
+      end
     end
 
     describe "#field_2" do # managing org

spec/services/bulk_upload/sales/year2024/row_parser_spec.rb

@@ -554,6 +554,43 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
           expect(parser.errors[:field_6]).to include(/Enter a date when the owning organisation was active/)
         end
       end
+
+      context "when user is an unaffiliated non-support user and bulk upload organisation is affiliated with the owning organisation" do
+        let(:affiliated_org) { create(:organisation, :with_old_visible_id) }
+        let(:unaffiliated_user) { create(:user, organisation: create(:organisation)) }
+        let(:attributes) { { bulk_upload:, field_1: affiliated_org.old_visible_id } }
+        let(:organisation_id) { unaffiliated_user.organisation_id }
+
+        before do
+          create(:organisation_relationship, parent_organisation: owning_org, child_organisation: affiliated_org)
+          bulk_upload.update!(organisation_id:, user: unaffiliated_user)
+        end
+
+        it "blocks log creation and adds an error to field_1" do
+          parser = described_class.new(attributes)
+          parser.valid?
+          expect(parser).to be_block_log_creation
+          expect(parser.errors[:field_1]).to include("You do not have permission to add logs for this owning organisation")
+        end
+      end
+
+      context "when user is an unaffiliated support user and bulk upload organisation is affiliated with the owning organisation" do
+        let(:affiliated_org) { create(:organisation, :with_old_visible_id) }
+        let(:unaffiliated_support_user) { create(:user, :support, organisation: create(:organisation)) }
+        let(:attributes) { { bulk_upload:, field_1: affiliated_org.old_visible_id } }
+        let(:organisation_id) { affiliated_org.id }
+
+        before do
+          create(:organisation_relationship, parent_organisation: owning_org, child_organisation: affiliated_org)
+          bulk_upload.update!(organisation_id:, user: unaffiliated_support_user)
+        end
+
+        it "does not block log creation and does not add an error to field_1" do
+          parser = described_class.new(attributes)
+          parser.valid?
+          expect(parser.errors[:field_1]).not_to include("You do not have permission to add logs for this owning organisation")
+        end
+      end
     end
 
     describe "#field_3" do # username for assigned_to
@@ -576,6 +613,23 @@ RSpec.describe BulkUpload::Sales::Year2024::RowParser do
         end
       end
 
+      context "when blank and bulk upload user is support" do
+        let(:bulk_upload) { create(:bulk_upload, :sales, user: create(:user, :support), year: 2024) }
+
+        let(:attributes) { setup_section_params.merge(bulk_upload:, field_3: nil) }
+
+        it "is not permitted" do
+          parser.valid?
+          expect(parser.errors[:field_3]).to be_present
+          expect(parser.errors[:field_3]).to include("You must answer what is the CORE username of the account this sales log should be assigned to?")
+        end
+
+        it "blocks log creation" do
+          parser.valid?
+          expect(parser).to be_block_log_creation
+        end
+      end
+
       context "when user could not be found" do
         let(:attributes) { { bulk_upload:, field_3: "idonotexist@example.com" } }