From 9f2b8c7ab9aee2753b2c98feae1c6ea9d0301560 Mon Sep 17 00:00:00 2001 From: Kat Date: Thu, 7 Sep 2023 09:34:02 +0100 Subject: [PATCH] Add missing sales answers reporting --- app/services/imports/import_report_service.rb | 49 ++++++++++----- .../imports/import_report_service_spec.rb | 60 +++++++++++++++---- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/app/services/imports/import_report_service.rb b/app/services/imports/import_report_service.rb index 7dcde699c..6522cbfeb 100644 --- a/app/services/imports/import_report_service.rb +++ b/app/services/imports/import_report_service.rb @@ -86,24 +86,47 @@ module Imports end def generate_missing_answers_report(report_suffix) - Rails.logger.info("Generating missing imported lettings logs answers report") + create_missing_answers_report(report_suffix, LettingsLog) + create_missing_answers_report(report_suffix, SalesLog) + end + + def create_missing_answers_report(report_suffix, log_class) + class_name = log_class.name.underscore.humanize + Rails.logger.info("Generating missing imported #{class_name}s answers report") + + unanswered_question_counts, missing_answers_example_sets = process_missing_answers(log_class) + + report_name = "MissingAnswersReport#{log_class.name}_#{report_suffix}.csv" + @storage_service.write_file(report_name, BYTE_ORDER_MARK + missing_answers_report(unanswered_question_counts)) + + examples_report_name = "MissingAnswersExamples#{log_class.name}_#{report_suffix}.csv" + @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples(missing_answers_example_sets)) + + @logger.info("Missing #{class_name}s answers report available in s3 import bucket at #{report_name}") + end + + def process_missing_answers(log_class) unanswered_question_counts = {} missing_answers_example_sets = {} - LettingsLog.where.not(old_id: nil).where(status: "in_progress").each do |lettings_log| - applicable_questions = lettings_log.form.subsections.map { |s| s.applicable_questions(lettings_log).select { |q| q.enabled?(lettings_log) } }.flatten - unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(lettings_log) }.map(&:id) - lettings_log.optional_fields).join(", ") + log_class.where.not(old_id: nil).where(status: "in_progress").each do |sales_log| + applicable_questions = sales_log.form.subsections.map { |s| s.applicable_questions(sales_log).select { |q| q.enabled?(sales_log) } }.flatten + unanswered_questions = (applicable_questions.filter { |q| q.unanswered?(sales_log) }.map(&:id) - sales_log.optional_fields).join(", ") if unanswered_question_counts[unanswered_questions].present? unanswered_question_counts[unanswered_questions] += 1 - missing_answers_example_sets[unanswered_questions] << { id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10 + missing_answers_example_sets[unanswered_questions] << { id: sales_log.id, old_form_id: sales_log.old_form_id, owning_organisation_id: sales_log.owning_organisation_id } unless unanswered_question_counts[unanswered_questions] > 10 else unanswered_question_counts[unanswered_questions] = 1 - missing_answers_example_sets[unanswered_questions] = [{ id: lettings_log.id, old_form_id: lettings_log.old_form_id, owning_organisation_id: lettings_log.owning_organisation_id }] + missing_answers_example_sets[unanswered_questions] = [{ id: sales_log.id, old_form_id: sales_log.old_form_id, owning_organisation_id: sales_log.owning_organisation_id }] end end - rep = CSV.generate do |report| + [unanswered_question_counts, missing_answers_example_sets] + end + + def missing_answers_report(unanswered_question_counts) + CSV.generate do |report| headers = ["Missing answers", "Total number of affected logs"] report << headers @@ -111,8 +134,10 @@ module Imports report << [missing_answers, count] end end + end - missing_answers_examples = CSV.generate do |report| + def missing_answers_examples(missing_answers_example_sets) + CSV.generate do |report| headers = ["Missing answers", "Organisation ID", "Log ID", "Old Form ID"] report << headers @@ -122,14 +147,6 @@ module Imports end end end - - report_name = "MissingAnswersReport_#{report_suffix}.csv" - @storage_service.write_file(report_name, BYTE_ORDER_MARK + rep) - - examples_report_name = "MissingAnswersExamples_#{report_suffix}.csv" - @storage_service.write_file(examples_report_name, BYTE_ORDER_MARK + missing_answers_examples) - - @logger.info("Missing answers report available in s3 import bucket at #{report_name}") end end end diff --git a/spec/services/imports/import_report_service_spec.rb b/spec/services/imports/import_report_service_spec.rb index ca6e81bc2..938ab9f1b 100644 --- a/spec/services/imports/import_report_service_spec.rb +++ b/spec/services/imports/import_report_service_spec.rb @@ -100,37 +100,77 @@ RSpec.describe Imports::ImportReportService do end describe "#generate_missing_answers_report" do - context "when there are in progress imported logs" do + context "when there are in progress imported lettings logs" do let(:institutions_csv) { nil } let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") } - let(:expected__answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") } + let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") } before do create_list(:lettings_log, 11, :completed, age1_known: nil) do |log, i| log.old_form_id = "100#{i}" log.old_id = "old_id_age1_known_#{i}" log.save! - expected__answers_examples_content.sub!("{id#{i}}", log.id.to_s) - expected__answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s) + expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s) end create_list(:lettings_log, 4, :completed, beds: nil) do |log, i| log.old_form_id = "200#{i}" log.old_id = "old_id_beds_#{i}" - expected__answers_examples_content.sub!("{id2_#{i}}", log.id.to_s) - expected__answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s) + expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s) log.save! end create(:lettings_log, :completed, age1_known: nil, beds: nil, old_form_id: "300", old_id: "123") do |log| - expected__answers_examples_content.sub!("{id}", log.id.to_s) - expected__answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s) + expected_answers_examples_content.sub!("{id}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s) end create_list(:lettings_log, 2, :completed, age1_known: nil) end it "generates a csv with expected missing fields" do - expect(storage_service).to receive(:write_file).with("MissingAnswersReport_report_suffix.csv", "#{expected_content}") - expect(storage_service).to receive(:write_file).with("MissingAnswersExamples_report_suffix.csv", "#{expected__answers_examples_content}") + expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "#{expected_content}") + expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "#{expected_answers_examples_content}") + expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n") + expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n") + + report_service.generate_missing_answers_report("report_suffix") + end + end + + context "when there are in progress imported sales logs" do + let(:institutions_csv) { nil } + let(:expected_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_report.csv") } + let(:expected_answers_examples_content) { File.read("spec/fixtures/files/imported_lettings_logs_missing_answers_examples.csv") } + + before do + create_list(:sales_log, 11, :completed, age1_known: nil) do |log, i| + log.old_id = "age1_known_#{i}" + log.old_form_id = "100#{i}" + log.save! + expected_answers_examples_content.sub!("{id#{i}}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id#{i}}", log.owning_organisation_id.to_s) + end + create_list(:sales_log, 4, :completed, beds: nil) do |log, i| + log.old_id = "beds_#{i}" + log.old_form_id = "200#{i}" + expected_answers_examples_content.sub!("{id2_#{i}}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id2_#{i}}", log.owning_organisation_id.to_s) + log.save! + end + create(:sales_log, :completed, age1_known: nil, beds: nil, old_id: "beds_and_age", old_form_id: "300") do |log| + expected_answers_examples_content.sub!("{id}", log.id.to_s) + expected_answers_examples_content.sub!("{org_id}", log.owning_organisation_id.to_s) + end + + create_list(:sales_log, 2, :completed, age1_known: nil) + end + + it "generates a csv with expected missing fields" do + expect(storage_service).to receive(:write_file).with("MissingAnswersReportLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Total number of affected logs\n") + expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesLettingsLog_report_suffix.csv", "\uFEFFMissing answers,Organisation ID,Log ID,Old Form ID\n") + expect(storage_service).to receive(:write_file).with("MissingAnswersReportSalesLog_report_suffix.csv", "#{expected_content}") + expect(storage_service).to receive(:write_file).with("MissingAnswersExamplesSalesLog_report_suffix.csv", "#{expected_answers_examples_content}") report_service.generate_missing_answers_report("report_suffix") end