diff --git a/app/services/exports/export_service.rb b/app/services/exports/export_service.rb index fd7aae017..bf98a4edf 100644 --- a/app/services/exports/export_service.rb +++ b/app/services/exports/export_service.rb @@ -12,20 +12,24 @@ module Exports daily_run_number = get_daily_run_number lettings_archives_for_manifest = {} users_archives_for_manifest = {} + organisations_archives_for_manifest = {} if collection.present? case collection when "users" users_archives_for_manifest = get_user_archives(start_time, full_update) + when "organisations" + organisations_archives_for_manifest = get_organisation_archives(start_time, full_update) else lettings_archives_for_manifest = get_lettings_archives(start_time, full_update, collection) end else users_archives_for_manifest = get_user_archives(start_time, full_update) + organisations_archives_for_manifest = get_organisation_archives(start_time, full_update) lettings_archives_for_manifest = get_lettings_archives(start_time, full_update, collection) end - write_master_manifest(daily_run_number, lettings_archives_for_manifest.merge(users_archives_for_manifest)) + write_master_manifest(daily_run_number, lettings_archives_for_manifest.merge(users_archives_for_manifest).merge(organisations_archives_for_manifest)) end private @@ -61,6 +65,11 @@ module Exports users_export_service.export_xml_users(full_update:) end + def get_organisation_archives(start_time, full_update) + organisations_export_service = Exports::OrganisationExportService.new(@storage_service, start_time) + organisations_export_service.export_xml_organisations(full_update:) + end + def get_lettings_archives(start_time, full_update, collection) lettings_export_service = Exports::LettingsLogExportService.new(@storage_service, start_time) lettings_export_service.export_xml_lettings_logs(full_update:, collection_year: collection) diff --git a/app/services/exports/lettings_log_export_constants.rb b/app/services/exports/lettings_log_export_constants.rb index 3dc47736a..6c8cd061d 100644 --- a/app/services/exports/lettings_log_export_constants.rb +++ b/app/services/exports/lettings_log_export_constants.rb @@ -29,7 +29,6 @@ module Exports::LettingsLogExportConstants "has_benefits", "hb", "hbrentshortfall", - "hcnum", "hhmemb", "hhtype", "homeless", @@ -47,9 +46,7 @@ module Exports::LettingsLogExportConstants "layear", "leftreg", "lettype", - "manhcnum", "maningorgid", - "maningorgname", "mantype", "mobstand", "mrcdate", @@ -60,7 +57,6 @@ module Exports::LettingsLogExportConstants "nocharge", "offered", "owningorgid", - "owningorgname", "period", "uprn", "uprn_known", @@ -76,7 +72,6 @@ module Exports::LettingsLogExportConstants "prevloc", "prevten", "propcode", - "providertype", "pscharge", "reason", "reasonother", diff --git a/app/services/exports/lettings_log_export_service.rb b/app/services/exports/lettings_log_export_service.rb index b21099a06..040254f8a 100644 --- a/app/services/exports/lettings_log_export_service.rb +++ b/app/services/exports/lettings_log_export_service.rb @@ -57,13 +57,9 @@ module Exports # Organisation fields if lettings_log.owning_organisation attribute_hash["owningorgid"] = lettings_log.owning_organisation.old_visible_id || (lettings_log.owning_organisation.id + LOG_ID_OFFSET) - attribute_hash["owningorgname"] = lettings_log.owning_organisation.name - attribute_hash["hcnum"] = lettings_log.owning_organisation.housing_registration_no end if lettings_log.managing_organisation attribute_hash["maningorgid"] = lettings_log.managing_organisation.old_visible_id || (lettings_log.managing_organisation.id + LOG_ID_OFFSET) - attribute_hash["maningorgname"] = lettings_log.managing_organisation.name - attribute_hash["manhcnum"] = lettings_log.managing_organisation.housing_registration_no end # Covert date times to ISO 8601 @@ -85,9 +81,9 @@ module Exports end attribute_hash["log_id"] = lettings_log.id - attribute_hash["assigned_to"] = lettings_log.assigned_to&.email - attribute_hash["created_by"] = lettings_log.created_by&.email - attribute_hash["amended_by"] = lettings_log.updated_by&.email + attribute_hash["assigned_to"] = lettings_log.assigned_to_id + attribute_hash["created_by"] = lettings_log.created_by_id + attribute_hash["amended_by"] = lettings_log.updated_by_id attribute_hash["la"] = lettings_log.la attribute_hash["postcode_full"] = lettings_log.postcode_full @@ -164,7 +160,6 @@ module Exports form << doc.create_element(key, value) end end - form << doc.create_element("providertype", lettings_log.owning_organisation&.read_attribute_before_type_cast(:provider_type)) end xml_doc_to_temp_file(doc) diff --git a/app/services/exports/organisation_export_constants.rb b/app/services/exports/organisation_export_constants.rb new file mode 100644 index 000000000..3a1c5fb48 --- /dev/null +++ b/app/services/exports/organisation_export_constants.rb @@ -0,0 +1,27 @@ +module Exports::OrganisationExportConstants + MAX_XML_RECORDS = 10_000 + + EXPORT_FIELDS = Set[ + "id", + "name", + "phone", + "provider_type", + "address_line1", + "address_line2", + "postcode", + "holds_own_stock", + "housing_registration_no", + "active", + "old_org_id", + "old_visible_id", + "merge_date", + "absorbing_organisation_id", + "available_from", + "deleted_at", + "dsa_signed", + "dsa_signed_at", + "dpo_email", + "profit_status", + "group" + ] +end diff --git a/app/services/exports/organisation_export_service.rb b/app/services/exports/organisation_export_service.rb new file mode 100644 index 000000000..71eccf60a --- /dev/null +++ b/app/services/exports/organisation_export_service.rb @@ -0,0 +1,72 @@ +module Exports + class OrganisationExportService < Exports::XmlExportService + include Exports::OrganisationExportConstants + include CollectionTimeHelper + + def export_xml_organisations(full_update: false) + collection = "organisations" + recent_export = Export.where(collection:).order("started_at").last + + base_number = Export.where(empty_export: false, collection:).maximum(:base_number) || 1 + export = build_export_run(collection, base_number, full_update) + archives_for_manifest = write_export_archive(export, collection, recent_export, full_update) + + export.empty_export = archives_for_manifest.empty? + export.save! + + archives_for_manifest + end + + private + + def get_archive_name(collection, base_number, increment) + return unless collection + + base_number_str = "f#{base_number.to_s.rjust(4, '0')}" + increment_str = "inc#{increment.to_s.rjust(4, '0')}" + "#{collection}_2024_2025_apr_mar_#{base_number_str}_#{increment_str}".downcase + end + + def retrieve_resources(recent_export, full_update, _collection) + if !full_update && recent_export + params = { from: recent_export.started_at, to: @start_time } + Organisation.where("(updated_at >= :from AND updated_at <= :to)", params) + else + params = { to: @start_time } + Organisation.where("updated_at <= :to", params) + end + end + + def build_export_xml(organisations) + doc = Nokogiri::XML("") + + organisations.each do |organisation| + attribute_hash = apply_cds_transformation(organisation) + form = doc.create_element("form") + doc.at("forms") << form + attribute_hash.each do |key, value| + if !EXPORT_FIELDS.include?(key) + next + else + form << doc.create_element(key, value) + end + end + end + + xml_doc_to_temp_file(doc) + end + + def apply_cds_transformation(organisation) + attribute_hash = organisation.attributes + attribute_hash["deleted_at"] = organisation.discarded_at + attribute_hash["dsa_signed"] = organisation.data_protection_confirmed? + attribute_hash["dsa_signed_at"] = organisation.data_protection_confirmation&.signed_at + attribute_hash["dpo_email"] = organisation.data_protection_confirmation&.data_protection_officer_email + attribute_hash["provider_type"] = organisation.provider_type_before_type_cast + attribute_hash["profit_status"] = nil # will need update when we add the field to the org + attribute_hash["group"] = nil # will need update when we add the field to the org + + attribute_hash + end + end +end diff --git a/app/services/exports/user_export_service.rb b/app/services/exports/user_export_service.rb index 58464a680..907a1cc86 100644 --- a/app/services/exports/user_export_service.rb +++ b/app/services/exports/user_export_service.rb @@ -4,9 +4,9 @@ module Exports include CollectionTimeHelper def export_xml_users(full_update: false) - recent_export = Export.order("started_at").last - collection = "users" + recent_export = Export.where(collection:).order("started_at").last + base_number = Export.where(empty_export: false, collection:).maximum(:base_number) || 1 export = build_export_run(collection, base_number, full_update) archives_for_manifest = write_export_archive(export, collection, recent_export, full_update) diff --git a/spec/fixtures/exports/general_needs_log.xml b/spec/fixtures/exports/general_needs_log.xml index bacc7e9f0..8a53e0379 100644 --- a/spec/fixtures/exports/general_needs_log.xml +++ b/spec/fixtures/exports/general_needs_log.xml @@ -147,18 +147,13 @@ {id} {owning_org_id} - MHCLG - 1234 {managing_org_id} - MHCLG - 1234 2022-05-01T00:00:00+01:00 2022-05-01T00:00:00+01:00 {log_id} - test1@example.com - test1@example.com + {assigned_to} + {created_by} 2 - 1 diff --git a/spec/fixtures/exports/general_needs_log_23_24.xml b/spec/fixtures/exports/general_needs_log_23_24.xml index 9635cd0e4..3ca4059dd 100644 --- a/spec/fixtures/exports/general_needs_log_23_24.xml +++ b/spec/fixtures/exports/general_needs_log_23_24.xml @@ -148,18 +148,13 @@ {id} {owning_org_id} - MHCLG - 1234 {managing_org_id} - MHCLG - 1234 2023-04-03T00:00:00+01:00 2023-04-03T00:00:00+01:00 {log_id} - test1@example.com - test1@example.com + {assigned_to} + {created_by} 2 - 1 diff --git a/spec/fixtures/exports/general_needs_log_24_25.xml b/spec/fixtures/exports/general_needs_log_24_25.xml index a665a284e..489b096ca 100644 --- a/spec/fixtures/exports/general_needs_log_24_25.xml +++ b/spec/fixtures/exports/general_needs_log_24_25.xml @@ -161,18 +161,13 @@ la as entered {id} {owning_org_id} - MHCLG - 1234 {managing_org_id} - MHCLG - 1234 2024-04-03T00:00:00+01:00 2024-04-03T00:00:00+01:00 {log_id} - test1@example.com - test1@example.com + {assigned_to} + {created_by} 2 - 1 diff --git a/spec/fixtures/exports/organisation.xml b/spec/fixtures/exports/organisation.xml new file mode 100644 index 000000000..8d87da16c --- /dev/null +++ b/spec/fixtures/exports/organisation.xml @@ -0,0 +1,26 @@ + + +
+ {id} + MHCLG + + 1 + 2 Marsham Street + London + SW1P 4DF + true + true + 1234 + + + + + + + true + {dsa_signed_at} + {dpo_email} + + + +
 diff --git a/spec/fixtures/exports/supported_housing_logs.xml b/spec/fixtures/exports/supported_housing_logs.xml index 50649241b..e897b1542 100644 --- a/spec/fixtures/exports/supported_housing_logs.xml +++ b/spec/fixtures/exports/supported_housing_logs.xml @@ -146,17 +146,13 @@ {id} {owning_org_id} - MHCLG - 1234 {managing_org_id} - MHCLG - 1234 2022-05-01T00:00:00+01:00 2022-05-01T00:00:00+01:00 {log_id} - fake@email.com - fake@email.com - other@email.com + {assigned_to} + {created_by} + {amended_by} 7 1 G @@ -175,6 +171,5 @@ {location_id} active 2 - 1 diff --git a/spec/jobs/data_export_xml_job_spec.rb b/spec/jobs/data_export_xml_job_spec.rb index 2eac24218..3712e115c 100644 --- a/spec/jobs/data_export_xml_job_spec.rb +++ b/spec/jobs/data_export_xml_job_spec.rb @@ -5,17 +5,20 @@ describe DataExportXmlJob do let(:env_config_service) { instance_double(Configuration::EnvConfigurationService) } let(:lettings_export_service) { instance_double(Exports::LettingsLogExportService, export_xml_lettings_logs: {}) } let(:user_export_service) { instance_double(Exports::UserExportService, export_xml_users: {}) } + let(:organisation_export_service) { instance_double(Exports::OrganisationExportService, export_xml_organisations: {}) } before do allow(Storage::S3Service).to receive(:new).and_return(storage_service) allow(Configuration::EnvConfigurationService).to receive(:new).and_return(env_config_service) allow(Exports::LettingsLogExportService).to receive(:new).and_return(lettings_export_service) allow(Exports::UserExportService).to receive(:new).and_return(user_export_service) + allow(Exports::OrganisationExportService).to receive(:new).and_return(organisation_export_service) end it "calls the export services" do expect(lettings_export_service).to receive(:export_xml_lettings_logs) expect(user_export_service).to receive(:export_xml_users) + expect(organisation_export_service).to receive(:export_xml_organisations) described_class.perform_now end @@ -24,6 +27,7 @@ describe DataExportXmlJob do it "calls the export service" do expect(lettings_export_service).to receive(:export_xml_lettings_logs).with(full_update: true, collection_year: nil) expect(user_export_service).to receive(:export_xml_users).with(full_update: true) + expect(organisation_export_service).to receive(:export_xml_organisations).with(full_update: true) described_class.perform_now(full_update: true) end diff --git a/spec/services/exports/export_service_spec.rb b/spec/services/exports/export_service_spec.rb index aaab77e62..fb52c5274 100644 --- a/spec/services/exports/export_service_spec.rb +++ b/spec/services/exports/export_service_spec.rb @@ -7,6 +7,8 @@ RSpec.describe Exports::ExportService do let(:expected_master_manifest_filename) { "Manifest_2022_05_01_0001.csv" } let(:start_time) { Time.zone.local(2022, 5, 1) } let(:user) { FactoryBot.create(:user, email: "test1@example.com") } + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: {}) } + let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } before do Timecop.freeze(start_time) @@ -14,6 +16,7 @@ RSpec.describe Exports::ExportService do allow(storage_service).to receive(:write_file) allow(Exports::LettingsLogExportService).to receive(:new).and_return(lettings_logs_export_service) allow(Exports::UserExportService).to receive(:new).and_return(users_export_service) + allow(Exports::OrganisationExportService).to receive(:new).and_return(organisations_export_service) end after do @@ -24,9 +27,7 @@ RSpec.describe Exports::ExportService do context "and no lettings archives get created in lettings logs export" do let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: {}) } - context "and no user archives get created in user export" do - let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } - + context "and no user or organisation archives get created in user export" do it "generates a master manifest with the correct name" do expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) export_service.export_xml @@ -59,14 +60,49 @@ RSpec.describe Exports::ExportService do expect(actual_content).to eq(expected_content) end end + + context "and one organisation archive gets created in organisation export" do + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } + + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_xml + end + + it "generates a master manifest with CSV headers and correct data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_xml + expect(actual_content).to eq(expected_content) + end + end + + context "and user and organisation archive gets created in organisation export" do + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } + let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time }) } + + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_xml + end + + it "generates a master manifest with CSV headers and correct data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_xml + expect(actual_content).to eq(expected_content) + end + end end context "and one lettings archive gets created in lettings logs export" do let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } context "and no user archives get created in user export" do - let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } - it "generates a master manifest with the correct name" do expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) export_service.export_xml @@ -105,8 +141,6 @@ RSpec.describe Exports::ExportService do let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time, "second_file_base_name" => start_time }) } context "and no user archives get created in user export" do - let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } - it "generates a master manifest with the correct name" do expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) export_service.export_xml @@ -139,6 +173,25 @@ RSpec.describe Exports::ExportService do expect(actual_content).to eq(expected_content) end end + + context "and multiple user and organisation archives gets created in user export" do + let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: { "some_user_file_base_name" => start_time, "second_user_file_base_name" => start_time }) } + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time, "second_organisation_file_base_name" => start_time }) } + + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_xml + end + + it "generates a master manifest with CSV headers and correct data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_file_base_name,2022-05-01 00:00:00 +0100,some_file_base_name.zip\nsecond_file_base_name,2022-05-01 00:00:00 +0100,second_file_base_name.zip\nsome_user_file_base_name,2022-05-01 00:00:00 +0100,some_user_file_base_name.zip\nsecond_user_file_base_name,2022-05-01 00:00:00 +0100,second_user_file_base_name.zip\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\nsecond_organisation_file_base_name,2022-05-01 00:00:00 +0100,second_organisation_file_base_name.zip\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_xml + expect(actual_content).to eq(expected_content) + end + end end end @@ -190,8 +243,6 @@ RSpec.describe Exports::ExportService do context "when exporting user collection" do context "and no user archives get created in users export" do - let(:users_export_service) { instance_double("Exports::UserExportService", export_xml_users: {}) } - context "and lettings log archive gets created in lettings logs export" do let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } @@ -233,4 +284,50 @@ RSpec.describe Exports::ExportService do end end end + + context "when exporting organisation collection" do + context "and no organisation archives get created in organisations export" do + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: {}) } + + context "and lettings log archive gets created in lettings logs export" do + let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } + + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_xml(full_update: true, collection: "organisations") + end + + it "does not write lettings log data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_xml(full_update: true, collection: "organisations") + expect(actual_content).to eq(expected_content) + end + end + end + + context "and organisations archive gets created in organisations export" do + let(:lettings_logs_export_service) { instance_double("Exports::LettingsLogExportService", export_xml_lettings_logs: { "some_file_base_name" => start_time }) } + + context "and lettings log archive gets created in lettings log export" do + let(:organisations_export_service) { instance_double("Exports::OrganisationExportService", export_xml_organisations: { "some_organisation_file_base_name" => start_time }) } + + it "generates a master manifest with the correct name" do + expect(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) + export_service.export_xml(full_update: true, collection: "organisations") + end + + it "does not write lettings log data" do + actual_content = nil + expected_content = "zip-name,date-time zipped folder generated,zip-file-uri\nsome_organisation_file_base_name,2022-05-01 00:00:00 +0100,some_organisation_file_base_name.zip\n" + allow(storage_service).to receive(:write_file).with(expected_master_manifest_filename, any_args) { |_, arg2| actual_content = arg2&.string } + + export_service.export_xml(full_update: true, collection: "organisations") + expect(actual_content).to eq(expected_content) + end + end + end + end end diff --git a/spec/services/exports/lettings_log_export_service_spec.rb b/spec/services/exports/lettings_log_export_service_spec.rb index 6f7d88c91..bf2d100ed 100644 --- a/spec/services/exports/lettings_log_export_service_spec.rb +++ b/spec/services/exports/lettings_log_export_service_spec.rb @@ -23,6 +23,9 @@ RSpec.describe Exports::LettingsLogExportService do export_template.sub!(/\{managing_org_id\}/, (lettings_log["managing_organisation_id"] + Exports::LettingsLogExportService::LOG_ID_OFFSET).to_s) export_template.sub!(/\{location_id\}/, (lettings_log["location_id"]).to_s) if lettings_log.needstype == 2 export_template.sub!(/\{scheme_id\}/, (lettings_log["scheme_id"]).to_s) if lettings_log.needstype == 2 + export_template.sub!(/\{assigned_to\}/, lettings_log["assigned_to_id"].to_s) + export_template.sub!(/\{created_by\}/, lettings_log["created_by_id"].to_s) + export_template.sub!(/\{amended_by\}/, lettings_log["updated_by_id"].to_s) export_template.sub!(/\{log_id\}/, lettings_log["id"].to_s) end diff --git a/spec/services/exports/organisation_export_service_spec.rb b/spec/services/exports/organisation_export_service_spec.rb new file mode 100644 index 000000000..4de0e84a8 --- /dev/null +++ b/spec/services/exports/organisation_export_service_spec.rb @@ -0,0 +1,219 @@ +require "rails_helper" + +RSpec.describe Exports::OrganisationExportService do + subject(:export_service) { described_class.new(storage_service, start_time) } + + let(:storage_service) { instance_double(Storage::S3Service) } + + let(:xml_export_file) { File.open("spec/fixtures/exports/organisation.xml", "r:UTF-8") } + let(:local_manifest_file) { File.open("spec/fixtures/exports/manifest.xml", "r:UTF-8") } + + let(:expected_zip_filename) { "organisations_2024_2025_apr_mar_f0001_inc0001.zip" } + let(:expected_data_filename) { "organisations_2024_2025_apr_mar_f0001_inc0001_pt001.xml" } + let(:expected_manifest_filename) { "manifest.xml" } + let(:start_time) { Time.zone.local(2022, 5, 1) } + let(:organisation) { create(:organisation, with_dsa: false) } + + def replace_entity_ids(organisation, export_template) + export_template.sub!(/\{id\}/, organisation["id"].to_s) + export_template.sub!(/\{dsa_signed_at\}/, organisation.data_protection_confirmation&.signed_at.to_s) + export_template.sub!(/\{dpo_email\}/, organisation.data_protection_confirmation&.data_protection_officer_email) + end + + def replace_record_number(export_template, record_number) + export_template.sub!(/\{recno\}/, record_number.to_s) + end + + before do + Timecop.freeze(start_time) + Singleton.__init__(FormHandler) + allow(storage_service).to receive(:write_file) + end + + after do + Timecop.return + end + + context "when exporting daily organisations in XML" do + context "and no organisations are available for export" do + it "returns an empty archives list" do + expect(export_service.export_xml_organisations).to eq({}) + end + end + + context "and one organisation is available for export" do + let!(:organisation) { create(:organisation) } + + it "generates a ZIP export file with the expected filename" do + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) + export_service.export_xml_organisations + end + + it "generates an XML export file with the expected filename within the ZIP file" do + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) + expect(entry).not_to be_nil + expect(entry.name).to eq(expected_data_filename) + end + export_service.export_xml_organisations + end + + it "generates an XML manifest file with the expected content within the ZIP file" do + expected_content = replace_record_number(local_manifest_file.read, 1) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) + expect(entry).not_to be_nil + expect(entry.get_input_stream.read).to eq(expected_content) + end + + export_service.export_xml_organisations + end + + it "generates an XML export file with the expected content within the ZIP file" do + expected_content = replace_entity_ids(organisation, xml_export_file.read) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_data_filename) + expect(entry).not_to be_nil + expect(entry.get_input_stream.read).to eq(expected_content) + end + + export_service.export_xml_organisations + end + + it "returns the list with correct archive" do + expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) + end + end + + context "and multiple organisations are available for export" do + before do + create(:organisation) + create(:organisation) + end + + it "generates an XML manifest file with the expected content within the ZIP file" do + expected_content = replace_record_number(local_manifest_file.read, 2) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) + expect(entry).not_to be_nil + expect(entry.get_input_stream.read).to eq(expected_content) + end + + export_service.export_xml_organisations + end + + it "creates an export record in a database with correct time" do + expect { export_service.export_xml_organisations } + .to change(Export, :count).by(1) + expect(Export.last.started_at).to be_within(2.seconds).of(start_time) + end + + context "when this is the first export (full)" do + it "returns a ZIP archive for the master manifest" do + expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "").gsub(".zip", "") => start_time }) + end + end + + context "and underlying data changes between getting the organisations and writting the manifest" do + def remove_organisations(organisations) + organisations.each(&:destroy) + file = Tempfile.new + doc = Nokogiri::XML("") + doc.write_xml_to(file, encoding: "UTF-8") + file.rewind + file + end + + def create_fake_maifest + file = Tempfile.new + doc = Nokogiri::XML("") + doc.write_xml_to(file, encoding: "UTF-8") + file.rewind + file + end + + it "maintains the same record number" do + # rubocop:disable RSpec/SubjectStub + allow(export_service).to receive(:build_export_xml) do |organisations| + remove_organisations(organisations) + end + allow(export_service).to receive(:build_manifest_xml) do + create_fake_maifest + end + + expect(export_service).to receive(:build_manifest_xml).with(2) + # rubocop:enable RSpec/SubjectStub + export_service.export_xml_organisations + end + end + + context "when this is a second export (partial)" do + before do + start_time = Time.zone.local(2022, 6, 1) + Export.new(started_at: start_time, collection: "organisations").save! # this should be organisation export + end + + it "does not add any entry for the master manifest (no organisations)" do + expect(export_service.export_xml_organisations).to eq({}) + end + end + end + + context "and a previous export has run the same day having organisations" do + before do + create(:organisation) + export_service.export_xml_organisations + end + + context "and we trigger another full update" do + it "increments the base number" do + export_service.export_xml_organisations(full_update: true) + expect(Export.last.base_number).to eq(2) + end + + it "resets the increment number" do + export_service.export_xml_organisations(full_update: true) + expect(Export.last.increment_number).to eq(1) + end + + it "returns a correct archives list for manifest file" do + expect(export_service.export_xml_organisations(full_update: true)).to eq({ "organisations_2024_2025_apr_mar_f0002_inc0001" => start_time }) + end + + it "generates a ZIP export file with the expected filename" do + expect(storage_service).to receive(:write_file).with("organisations_2024_2025_apr_mar_f0002_inc0001.zip", any_args) + export_service.export_xml_organisations(full_update: true) + end + end + end + + context "and a previous export has run having no organisations" do + before { export_service.export_xml_organisations } + + it "doesn't increment the manifest number by 1" do + export_service.export_xml_organisations + + expect(Export.last.increment_number).to eq(1) + end + end + + context "and an organisation has been migrated since the previous partial export" do + before do + create(:organisation, updated_at: Time.zone.local(2022, 4, 27)) + create(:organisation, updated_at: Time.zone.local(2022, 4, 27)) + Export.create!(started_at: Time.zone.local(2022, 4, 26), base_number: 1, increment_number: 1) + end + + it "generates an XML manifest file with the expected content within the ZIP file" do + expected_content = replace_record_number(local_manifest_file.read, 2) + expect(storage_service).to receive(:write_file).with(expected_zip_filename, any_args) do |_, content| + entry = Zip::File.open_buffer(content).find_entry(expected_manifest_filename) + expect(entry).not_to be_nil + expect(entry.get_input_stream.read).to eq(expected_content) + end + + expect(export_service.export_xml_organisations).to eq({ expected_zip_filename.gsub(".zip", "") => start_time }) + end + end + end +end diff --git a/spec/services/exports/user_export_service_spec.rb b/spec/services/exports/user_export_service_spec.rb index e7bcea08b..713d6f907 100644 --- a/spec/services/exports/user_export_service_spec.rb +++ b/spec/services/exports/user_export_service_spec.rb @@ -150,7 +150,7 @@ RSpec.describe Exports::UserExportService do context "when this is a second export (partial)" do before do start_time = Time.zone.local(2022, 6, 1) - Export.new(started_at: start_time).save! # this should be user export + Export.new(started_at: start_time, collection: "users").save! # this should be user export end it "does not add any entry for the master manifest (no users)" do