diff --git a/app/services/imports/lettings_logs_import_service.rb b/app/services/imports/lettings_logs_import_service.rb index 8fe71d3dc..a215e8f5a 100644 --- a/app/services/imports/lettings_logs_import_service.rb +++ b/app/services/imports/lettings_logs_import_service.rb @@ -1,5 +1,5 @@ module Imports - class LettingsLogsImportService < ImportService + class LettingsLogsImportService < LogsImportService def initialize(storage_service, logger = Rails.logger) @logs_with_discrepancies = Set.new @logs_overridden = Set.new @@ -318,43 +318,6 @@ module Imports end end - # Safe: A string that represents only an integer (or empty/nil) - def safe_string_as_integer(xml_doc, attribute) - str = field_value(xml_doc, "xmlns", attribute) - Integer(str, exception: false) - end - - # Safe: A string that represents only a decimal (or empty/nil) - def safe_string_as_decimal(xml_doc, attribute) - str = string_or_nil(xml_doc, attribute) - if str.nil? - nil - else - BigDecimal(str, exception: false) - end - end - - # Unsafe: A string that has more than just the integer value - def unsafe_string_as_integer(xml_doc, attribute) - str = string_or_nil(xml_doc, attribute) - if str.nil? - nil - else - str.to_i - end - end - - def compose_date(xml_doc, day_str, month_str, year_str) - day = Integer(field_value(xml_doc, "xmlns", day_str), exception: false) - month = Integer(field_value(xml_doc, "xmlns", month_str), exception: false) - year = Integer(field_value(xml_doc, "xmlns", year_str), exception: false) - if day.nil? || month.nil? || year.nil? - nil - else - Time.zone.local(year, month, day) - end - end - def get_form_name_component(xml_doc, index) form_name = meta_field_value(xml_doc, "form-name") form_type_components = form_name.split("-") @@ -399,42 +362,6 @@ module Imports end end - def find_organisation_id(xml_doc, id_field) - old_visible_id = string_or_nil(xml_doc, id_field) - organisation = Organisation.find_by(old_visible_id:) - raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil? - - organisation.id - end - - def sex(xml_doc, index) - sex = string_or_nil(xml_doc, "P#{index}Sex") - case sex - when "Male" - "M" - when "Female" - "F" - when "Other", "Non-binary" - "X" - when "Refused" - "R" - end - end - - def relat(xml_doc, index) - relat = string_or_nil(xml_doc, "P#{index}Rel") - case relat - when "Child" - "C" - when "Partner" - "P" - when "Other", "Non-binary" - "X" - when "Refused" - "R" - end - end - def age_known(xml_doc, index, hhmemb) return nil if hhmemb.present? && index > hhmemb @@ -473,16 +400,6 @@ module Imports end end - def compose_postcode(xml_doc, outcode, incode) - outcode_value = string_or_nil(xml_doc, outcode) - incode_value = string_or_nil(xml_doc, incode) - if outcode_value.nil? || incode_value.nil? || !"#{outcode_value} #{incode_value}".match(POSTCODE_REGEXP) - nil - else - "#{outcode_value} #{incode_value}" - end - end - def london_affordable_rent(xml_doc) lar = unsafe_string_as_integer(xml_doc, "LAR") if lar == 1 @@ -502,34 +419,6 @@ module Imports end end - def string_or_nil(xml_doc, attribute) - str = field_value(xml_doc, "xmlns", attribute) - str.presence - end - - def ethnic_group(ethnic) - case ethnic - when 1, 2, 3, 18 - # White - 0 - when 4, 5, 6, 7 - # Mixed - 1 - when 8, 9, 10, 11, 15 - # Asian - 2 - when 12, 13, 14 - # Black - 3 - when 16, 19 - # Others - 4 - when 17 - # Refused - 17 - end - end - # Letters should be lowercase to match case def housing_needs(xml_doc, letter) housing_need = string_or_nil(xml_doc, "Q10-#{letter}") diff --git a/app/services/imports/logs_import_service.rb b/app/services/imports/logs_import_service.rb new file mode 100644 index 000000000..0a3d2df87 --- /dev/null +++ b/app/services/imports/logs_import_service.rb @@ -0,0 +1,127 @@ +module Imports + class LogsImportService < ImportService + private + + # Safe: A string that represents only an integer (or empty/nil) + def safe_string_as_integer(xml_doc, attribute) + str = field_value(xml_doc, "xmlns", attribute) + Integer(str, exception: false) + end + + # Unsafe: A string that has more than just the integer value + def unsafe_string_as_integer(xml_doc, attribute) + str = string_or_nil(xml_doc, attribute) + if str.nil? + nil + else + str.to_i + end + end + + def compose_date(xml_doc, day_str, month_str, year_str) + day = Integer(field_value(xml_doc, "xmlns", day_str), exception: false) + month = Integer(field_value(xml_doc, "xmlns", month_str), exception: false) + year = Integer(field_value(xml_doc, "xmlns", year_str), exception: false) + if day.nil? || month.nil? || year.nil? + nil + else + Time.zone.local(year, month, day) + end + end + + def find_organisation_id(xml_doc, id_field) + old_visible_id = string_or_nil(xml_doc, id_field) + organisation = Organisation.find_by(old_visible_id:) + raise "Organisation not found with legacy ID #{old_visible_id}" if organisation.nil? + + organisation.id + end + + def string_or_nil(xml_doc, attribute) + str = field_value(xml_doc, "xmlns", attribute) + str.presence + end + + def ethnic_group(ethnic) + case ethnic + when 1, 2, 3, 18 + # White + 0 + when 4, 5, 6, 7 + # Mixed + 1 + when 8, 9, 10, 11, 15 + # Asian + 2 + when 12, 13, 14 + # Black + 3 + when 16, 19 + # Others + 4 + when 17 + # Refused + 17 + end + end + + # Safe: A string that represents only a decimal (or empty/nil) + def safe_string_as_decimal(xml_doc, attribute) + str = string_or_nil(xml_doc, attribute) + if str.nil? + nil + else + BigDecimal(str, exception: false) + end + end + + def compose_postcode(xml_doc, outcode, incode) + outcode_value = string_or_nil(xml_doc, outcode) + incode_value = string_or_nil(xml_doc, incode) + if outcode_value.nil? || incode_value.nil? || !"#{outcode_value} #{incode_value}".match(POSTCODE_REGEXP) + nil + else + "#{outcode_value} #{incode_value}" + end + end + + def previous_postcode_known(xml_doc, previous_postcode, prevloc) + previous_postcode_known = string_or_nil(xml_doc, "Q7UnknownPostcode") + if previous_postcode_known == "If postcode not known tick" || (previous_postcode.nil? && prevloc.present?) + 1 + elsif previous_postcode.nil? + nil + else + 0 + end + end + + def sex(xml_doc, index) + sex = string_or_nil(xml_doc, "P#{index}Sex") + case sex + when "Male" + "M" + when "Female" + "F" + when "Other", "Non-binary" + "X" + when "Refused" + "R" + end + end + + def relat(xml_doc, index) + relat = string_or_nil(xml_doc, "P#{index}Rel") + case relat + when "Child" + "C" + when "Partner" + "P" + when "Other", "Non-binary" + "X" + when "Refused", "Buyer prefers not to say" + "R" + end + end + end +end diff --git a/app/services/imports/sales_logs_import_service.rb b/app/services/imports/sales_logs_import_service.rb new file mode 100644 index 000000000..25f8148b8 --- /dev/null +++ b/app/services/imports/sales_logs_import_service.rb @@ -0,0 +1,450 @@ +module Imports + class SalesLogsImportService < LogsImportService + def initialize(storage_service, logger = Rails.logger) + @logs_with_discrepancies = Set.new + @logs_overridden = Set.new + super + end + + def create_logs(folder) + import_from(folder, :create_log) + if @logs_with_discrepancies.count.positive? + @logger.warn("The following sales logs had status discrepancies: [#{@logs_with_discrepancies.join(', ')}]") + end + end + + private + + def create_log(xml_doc) + attributes = {} + + previous_status = meta_field_value(xml_doc, "status") + + # Required fields for status complete or logic to work + # Note: order matters when we derive from previous values (attributes parameter) + + attributes["saledate"] = compose_date(xml_doc, "DAY", "MONTH", "YEAR") + attributes["owning_organisation_id"] = find_organisation_id(xml_doc, "OWNINGORGID") + attributes["type"] = unsafe_string_as_integer(xml_doc, "DerSaleType") + attributes["old_id"] = meta_field_value(xml_doc, "document-id") + attributes["created_at"] = Time.zone.parse(meta_field_value(xml_doc, "created-date")) + attributes["updated_at"] = Time.zone.parse(meta_field_value(xml_doc, "modified-date")) + attributes["purchid"] = string_or_nil(xml_doc, "PurchaserCode") + attributes["ownershipsch"] = unsafe_string_as_integer(xml_doc, "Ownership") + attributes["othtype"] = string_or_nil(xml_doc, "Q38OtherSale") + attributes["jointmore"] = unsafe_string_as_integer(xml_doc, "JointMore") + attributes["jointpur"] = unsafe_string_as_integer(xml_doc, "joint") + attributes["beds"] = safe_string_as_integer(xml_doc, "Q11Bedrooms") + attributes["companybuy"] = unsafe_string_as_integer(xml_doc, "company") if attributes["ownershipsch"] == 3 + attributes["hhmemb"] = safe_string_as_integer(xml_doc, "HHMEMB") + (1..6).each do |index| + attributes["age#{index}"] = safe_string_as_integer(xml_doc, "P#{index}Age") + attributes["sex#{index}"] = sex(xml_doc, index) + attributes["ecstat#{index}"] = unsafe_string_as_integer(xml_doc, "P#{index}Eco") + attributes["age#{index}_known"] = age_known(xml_doc, index, attributes["hhmemb"], attributes["age#{index}"]) + end + (2..6).each do |index| + attributes["relat#{index}"] = relat(xml_doc, index) + attributes["details_known_#{index}"] = details_known(index, attributes) + end + attributes["national"] = unsafe_string_as_integer(xml_doc, "P1Nat") + attributes["othernational"] = nil + attributes["ethnic"] = unsafe_string_as_integer(xml_doc, "P1Eth") + attributes["ethnic_group"] = ethnic_group(attributes["ethnic"]) + attributes["buy1livein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer1") + attributes["buylivein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer") if attributes["ownershipsch"] == 3 + attributes["builtype"] = unsafe_string_as_integer(xml_doc, "Q13BuildingType") + attributes["proptype"] = unsafe_string_as_integer(xml_doc, "Q12PropertyType") + attributes["privacynotice"] = 1 if string_or_nil(xml_doc, "Qdp") == "Yes" + attributes["noint"] = unsafe_string_as_integer(xml_doc, "PartAPurchaser") + attributes["buy2livein"] = unsafe_string_as_integer(xml_doc, "LiveInBuyer2") + attributes["wheel"] = unsafe_string_as_integer(xml_doc, "Q10Wheelchair") + attributes["hholdcount"] = safe_string_as_integer(xml_doc, "LiveInOther") + attributes["la"] = string_or_nil(xml_doc, "Q14ONSLACode") + attributes["income1"] = safe_string_as_integer(xml_doc, "Q2Person1Income") + attributes["income1nk"] = income_known(unsafe_string_as_integer(xml_doc, "P1IncKnown")) + attributes["inc1mort"] = unsafe_string_as_integer(xml_doc, "Q2Person1Mortgage") + attributes["income2"] = safe_string_as_integer(xml_doc, "Q2Person2Income") + attributes["income2nk"] = income_known(unsafe_string_as_integer(xml_doc, "P2IncKnown")) + attributes["savings"] = safe_string_as_integer(xml_doc, "Q3Savings") + attributes["savingsnk"] = savings_known(xml_doc) + attributes["prevown"] = unsafe_string_as_integer(xml_doc, "Q4PrevOwnedProperty") + attributes["mortgage"] = safe_string_as_decimal(xml_doc, "CALCMORT") + attributes["inc2mort"] = unsafe_string_as_integer(xml_doc, "Q2Person2MortApplication") + attributes["hb"] = unsafe_string_as_integer(xml_doc, "Q2a") + attributes["frombeds"] = safe_string_as_integer(xml_doc, "Q20Bedrooms") + attributes["staircase"] = unsafe_string_as_integer(xml_doc, "Q17aStaircase") + attributes["stairbought"] = safe_string_as_integer(xml_doc, "PercentBought") + attributes["stairowned"] = safe_string_as_integer(xml_doc, "PercentOwns") if attributes["staircase"] == 1 + attributes["mrent"] = safe_string_as_decimal(xml_doc, "Q28MonthlyRent") + attributes["exdate"] = compose_date(xml_doc, "EXDAY", "EXMONTH", "EXYEAR") + attributes["exday"] = safe_string_as_integer(xml_doc, "EXDAY") + attributes["exmonth"] = safe_string_as_integer(xml_doc, "EXMONTH") + attributes["exyear"] = safe_string_as_integer(xml_doc, "EXYEAR") + attributes["resale"] = unsafe_string_as_integer(xml_doc, "Q17Resale") + attributes["deposit"] = deposit(xml_doc, attributes) + attributes["cashdis"] = safe_string_as_decimal(xml_doc, "Q27SocialHomeBuy") + attributes["disabled"] = unsafe_string_as_integer(xml_doc, "Disability") + attributes["lanomagr"] = unsafe_string_as_integer(xml_doc, "Q19Rehoused") + attributes["value"] = purchase_price(xml_doc, attributes) + attributes["equity"] = safe_string_as_decimal(xml_doc, "Q23Equity") + attributes["discount"] = safe_string_as_decimal(xml_doc, "Q33Discount") + attributes["grant"] = safe_string_as_decimal(xml_doc, "Q32Reductions") + attributes["pregyrha"] = 1 if string_or_nil(xml_doc, "PREGYRHA") == "Yes" + attributes["pregla"] = 1 if string_or_nil(xml_doc, "PREGLA") == "Yes" + attributes["pregghb"] = 1 if string_or_nil(xml_doc, "PREGHBA") == "Yes" + attributes["pregother"] = 1 if string_or_nil(xml_doc, "PREGOTHER") == "Yes" + attributes["ppostcode_full"] = compose_postcode(xml_doc, "PPOSTC1", "PPOSTC2") + attributes["prevloc"] = string_or_nil(xml_doc, "Q7ONSLACode") + attributes["ppcodenk"] = previous_postcode_known(xml_doc, attributes["ppostcode_full"], attributes["prevloc"]) # Q7UNKNOWNPOSTCODE check mapping + attributes["ppostc1"] = string_or_nil(xml_doc, "PPOSTC1") + attributes["ppostc2"] = string_or_nil(xml_doc, "PPOSTC2") + attributes["previous_la_known"] = nil + attributes["hhregres"] = unsafe_string_as_integer(xml_doc, "ArmedF") + attributes["hhregresstill"] = still_serving(xml_doc) + attributes["proplen"] = safe_string_as_integer(xml_doc, "Q16aProplen2") + attributes["mscharge"] = monthly_charges(xml_doc, attributes) + attributes["mscharge_known"] = 1 if attributes["mscharge"].present? + attributes["prevten"] = unsafe_string_as_integer(xml_doc, "Q6PrevTenure") + attributes["mortgageused"] = unsafe_string_as_integer(xml_doc, "MORTGAGEUSED") + attributes["wchair"] = unsafe_string_as_integer(xml_doc, "Q15Wheelchair") + attributes["armedforcesspouse"] = unsafe_string_as_integer(xml_doc, "ARMEDFORCESSPOUSE") + attributes["hodate"] = compose_date(xml_doc, "HODAY", "HOMONTH", "HOYEAR") + attributes["hoday"] = safe_string_as_integer(xml_doc, "HODAY") + attributes["homonth"] = safe_string_as_integer(xml_doc, "HOMONTH") + attributes["hoyear"] = safe_string_as_integer(xml_doc, "HOYEAR") + attributes["fromprop"] = unsafe_string_as_integer(xml_doc, "Q21PropertyType") + attributes["socprevten"] = unsafe_string_as_integer(xml_doc, "PrevRentType") + attributes["mortgagelender"] = mortgage_lender(xml_doc, attributes) + attributes["mortgagelenderother"] = mortgage_lender_other(xml_doc, attributes) + attributes["mortlen"] = mortgage_length(xml_doc, attributes) + attributes["extrabor"] = borrowing(xml_doc, attributes) + attributes["totadult"] = safe_string_as_integer(xml_doc, "TOTADULT") # would get overridden + attributes["totchild"] = safe_string_as_integer(xml_doc, "TOTCHILD") # would get overridden + attributes["hhtype"] = unsafe_string_as_integer(xml_doc, "HHTYPE") + attributes["pcode1"] = string_or_nil(xml_doc, "PCODE1") + attributes["pcode2"] = string_or_nil(xml_doc, "PCODE2") + attributes["postcode_full"] = compose_postcode(xml_doc, "PCODE1", "PCODE2") + attributes["pcodenk"] = 0 if attributes["postcode_full"].present? # known if given + attributes["soctenant"] = soctenant(attributes) + attributes["ethnic_group2"] = nil # 23/24 variable + attributes["ethnicbuy2"] = nil # 23/24 variable + attributes["prevshared"] = nil # 23/24 variable + attributes["staircasesale"] = nil # 23/24 variable + + # Required for our form invalidated questions (not present in import) + attributes["previous_la_known"] = 1 if attributes["prevloc"].present? && attributes["ppostcode_full"].blank? + attributes["la_known"] = 1 if attributes["la"].present? && attributes["postcode_full"].blank? + + # Sets the log creator + owner_id = meta_field_value(xml_doc, "owner-user-id").strip + if owner_id.present? + user = LegacyUser.find_by(old_user_id: owner_id)&.user + @logger.warn "Missing user! We expected to find a legacy user with old_user_id #{owner_id}" unless user + + attributes["created_by"] = user + end + + set_default_values(attributes) if previous_status.include?("submitted") + sales_log = save_sales_log(attributes, previous_status) + compute_differences(sales_log, attributes) + check_status_completed(sales_log, previous_status) unless @logs_overridden.include?(sales_log.old_id) + end + + def save_sales_log(attributes, previous_status) + sales_log = SalesLog.new(attributes) + begin + sales_log.save! + sales_log + rescue ActiveRecord::RecordNotUnique + legacy_id = attributes["old_id"] + record = SalesLog.find_by(old_id: legacy_id) + @logger.info "Updating sales log #{record.id} with legacy ID #{legacy_id}" + record.update!(attributes) + record + rescue ActiveRecord::RecordInvalid => e + rescue_validation_or_raise(sales_log, attributes, previous_status, e) + end + end + + def rescue_validation_or_raise(sales_log, _attributes, _previous_status, exception) + @logger.error("Log #{sales_log.old_id}: Failed to import") + raise exception + end + + def compute_differences(sales_log, attributes) + differences = [] + attributes.each do |key, value| + sales_log_value = sales_log.send(key.to_sym) + next if fields_not_present_in_softwire_data.include?(key) + + if value != sales_log_value + differences.push("#{key} #{value.inspect} #{sales_log_value.inspect}") + end + end + @logger.warn "Differences found when saving log #{sales_log.old_id}: #{differences}" unless differences.empty? + end + + def fields_not_present_in_softwire_data + %w[created_by + income1_value_check + mortgage_value_check + savings_value_check + deposit_value_check + wheel_value_check + retirement_value_check + extrabor_value_check + deposit_and_mortgage_value_check + shared_ownership_deposit_value_check + grant_value_check + value_value_check + old_persons_shared_ownership_value_check + staircase_bought_value_check + monthly_charges_value_check + hodate_check + saledate_check] + end + + def check_status_completed(sales_log, previous_status) + if previous_status.include?("submitted") && sales_log.status != "completed" + @logger.warn "sales log #{sales_log.id} is not completed. The following answers are missing: #{missing_answers(sales_log).join(', ')}" + @logger.warn "sales log with old id:#{sales_log.old_id} is incomplete but status should be complete" + @logs_with_discrepancies << sales_log.old_id + end + end + + def age_known(_xml_doc, index, hhmemb, age) + return nil if hhmemb.present? && index > hhmemb + + return 0 if age.present? + end + + def details_known(index, attributes) + return nil if attributes["hhmemb"].nil? || index > attributes["hhmemb"] + return nil if attributes["jointpur"] == 1 && index == 2 + + if attributes["age#{index}_known"] != 0 && + attributes["sex#{index}"] == "R" && + attributes["relat#{index}"] == "R" && + attributes["ecstat#{index}"] == 10 + 2 # No + else + 1 # Yes + end + end + + MORTGAGE_LENDER_OPTIONS = { + "atom bank" => 1, + "barclays bank plc" => 2, + "bath building society" => 3, + "buckinghamshire building society" => 4, + "cambridge building society" => 5, + "coventry building society" => 6, + "cumberland building society" => 7, + "darlington building society" => 8, + "dudley building society" => 9, + "ecology building society" => 10, + "halifax" => 11, + "hanley economic building society" => 12, + "hinckley and rugby building society" => 13, + "holmesdale building society" => 14, + "ipswich building society" => 15, + "leeds building society" => 16, + "lloyds bank" => 17, + "mansfield building society" => 18, + "market harborough building society" => 19, + "melton mowbray building society" => 20, + "nationwide building society" => 21, + "natwest" => 22, + "nedbank private wealth" => 23, + "newbury building society" => 24, + "oneSavings bank" => 25, + "parity trust" => 26, + "penrith building society" => 27, + "pepper homeloans" => 28, + "royal bank of scotland" => 29, + "santander" => 30, + "skipton building society" => 31, + "teachers building society" => 32, + "the co-operative bank" => 33, + "tipton & coseley building society" => 34, + "tss" => 35, + "ulster bank" => 36, + "virgin money" => 37, + "west bromwich building society" => 38, + "yorkshire building society" => 39, + "other" => 40, + }.freeze + + # this comes through as a string, need to map to a corresponding integer + def mortgage_lender(xml_doc, attributes) + lender = case attributes["ownershipsch"] + when 1 + string_or_nil(xml_doc, "Q24aMortgageLender") + when 2 + string_or_nil(xml_doc, "Q34a") + when 3 + string_or_nil(xml_doc, "Q41aMortgageLender") + end + return if lender.blank? + + MORTGAGE_LENDER_OPTIONS[lender.downcase] || MORTGAGE_LENDER_OPTIONS["other"] + end + + def mortgage_lender_other(xml_doc, attributes) + return unless attributes["mortgagelender"] == MORTGAGE_LENDER_OPTIONS["other"] + + case attributes["ownershipsch"] + when 1 + string_or_nil(xml_doc, "Q24aMortgageLender") + when 2 + string_or_nil(xml_doc, "Q34a") + when 3 + string_or_nil(xml_doc, "Q41aMortgageLender") + end + end + + def mortgage_length(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + unsafe_string_as_integer(xml_doc, "Q24b") + when 2 + unsafe_string_as_integer(xml_doc, "Q34b") + when 3 + unsafe_string_as_integer(xml_doc, "Q41b") + end + end + + def savings_known(xml_doc) + case unsafe_string_as_integer(xml_doc, "savingsKnown") + when 1 # known + 0 + when 2 # unknown + 1 + end + end + + def soctenant(attributes) + return nil unless attributes["ownershipsch"] == 1 + + if attributes["frombeds"].blank? && attributes["fromprop"].blank? && attributes["socprevten"].blank? + 2 + else + 1 + end + # NO (2) if FROMBEDS, FROMPROP and socprevten are blank, and YES(1) if they are completed + end + + def still_serving(xml_doc) + case unsafe_string_as_integer(xml_doc, "LeftArmedF") + when 4 + 4 + when 5, 6 + 5 + end + end + + def income_known(value) + case value + when 1 # known + 0 + when 2 # unknown + 1 + end + end + + def borrowing(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + unsafe_string_as_integer(xml_doc, "Q25Borrowing") + when 2 + unsafe_string_as_integer(xml_doc, "Q35Borrowing") + when 3 + unsafe_string_as_integer(xml_doc, "Q42Borrowing") + end + end + + def purchase_price(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q22PurchasePrice") + when 2 + safe_string_as_decimal(xml_doc, "Q31PurchasePrice") + when 3 + safe_string_as_decimal(xml_doc, "Q40PurchasePrice") + end + end + + def deposit(xml_doc, attributes) + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q26CashDeposit") + when 2 + safe_string_as_decimal(xml_doc, "Q36CashDeposit") + when 3 + safe_string_as_decimal(xml_doc, "Q43CashDeposit") + end + end + + def monthly_charges(xml_doc, attributes) + safe_string_as_decimal(xml_doc, "Q29MonthlyCharges") + case attributes["ownershipsch"] + when 1 + safe_string_as_decimal(xml_doc, "Q29MonthlyCharges") + when 2 + safe_string_as_decimal(xml_doc, "Q37MonthlyCharges") + end + end + + def set_default_values(attributes) + attributes["mscharge_known"] ||= 0 if attributes["ownershipsch"] == 3 + attributes["mscharge"] ||= 0 if attributes["ownershipsch"] == 3 + attributes["armedforcesspouse"] ||= 7 + attributes["hhregres"] ||= 8 + attributes["disabled"] ||= 3 + attributes["wheel"] ||= 3 + attributes["hb"] ||= 4 + attributes["prevown"] ||= 3 + attributes["savingsnk"] ||= attributes["savings"].present? ? 0 : 1 + # attributes["noint"] = 1 # not interviewed + + # buyer 1 characteristics + attributes["age1_known"] ||= 1 + attributes["sex1"] ||= "R" + attributes["ethnic_group"] ||= 17 + attributes["ethnic"] ||= 17 + attributes["national"] ||= 13 + attributes["ecstat1"] ||= 10 + attributes["income1nk"] ||= attributes["income1"].present? ? 0 : 1 + attributes["hholdcount"] ||= default_household_count(attributes) # just for testing, might need to change + + # buyer 2 characteristics + if attributes["jointpur"] == 1 + attributes["age2_known"] ||= 1 + attributes["sex2"] ||= "R" + attributes["ecstat2"] ||= 10 + attributes["income2nk"] ||= attributes["income2"].present? ? 0 : 1 + end + + # other household members characteristics + (2..attributes["hhmemb"]).each do |index| + attributes["age#{index}_known"] ||= 1 + attributes["sex#{index}"] ||= "R" + attributes["ecstat#{index}"] ||= 10 + attributes["relat#{index}"] ||= "R" + end + end + + def missing_answers(sales_log) + applicable_questions = sales_log.form.subsections.map { |s| s.applicable_questions(sales_log) }.flatten + applicable_questions.filter { |q| q.unanswered?(sales_log) }.map(&:id) + end + + # just for testing, logic might need to change + def default_household_count(attributes) + return 0 if attributes["hhmemb"].zero? || attributes["hhmemb"].blank? + + attributes["jointpur"] == 1 ? attributes["hhmemb"] - 2 : attributes["hhmemb"] - 1 + end + end +end diff --git a/db/migrate/20230215112932_add_old_id_to_sales_logs.rb b/db/migrate/20230215112932_add_old_id_to_sales_logs.rb new file mode 100644 index 000000000..cf61a968c --- /dev/null +++ b/db/migrate/20230215112932_add_old_id_to_sales_logs.rb @@ -0,0 +1,8 @@ +class AddOldIdToSalesLogs < ActiveRecord::Migration[7.0] + def change + change_table :sales_logs, bulk: true do |t| + t.column :old_id, :string + end + add_index :sales_logs, :old_id, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 1747fa6c5..d1668b738 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_02_13_140932) do +ActiveRecord::Schema[7.0].define(version: 2023_02_15_112932) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -522,16 +522,18 @@ ActiveRecord::Schema[7.0].define(version: 2023_02_13_140932) do t.integer "old_persons_shared_ownership_value_check" t.integer "staircase_bought_value_check" t.integer "monthly_charges_value_check" + t.integer "saledate_check" t.integer "details_known_5" t.integer "details_known_6" - t.integer "saledate_check" - t.integer "prevshared" - t.integer "staircasesale" t.integer "ethnic_group2" t.integer "ethnicbuy2" t.integer "proplen_asked" + t.integer "prevshared" + t.integer "staircasesale" + t.string "old_id" t.index ["bulk_upload_id"], name: "index_sales_logs_on_bulk_upload_id" t.index ["created_by_id"], name: "index_sales_logs_on_created_by_id" + t.index ["old_id"], name: "index_sales_logs_on_old_id", unique: true t.index ["owning_organisation_id"], name: "index_sales_logs_on_owning_organisation_id" t.index ["updated_by_id"], name: "index_sales_logs_on_updated_by_id" end diff --git a/lib/tasks/data_import.rake b/lib/tasks/data_import.rake index 0b3881388..0dede82a3 100644 --- a/lib/tasks/data_import.rake +++ b/lib/tasks/data_import.rake @@ -22,6 +22,8 @@ namespace :core do Imports::OrganisationRentPeriodImportService.new(storage_service).create_organisation_rent_periods(path) when "lettings-logs" Imports::LettingsLogsImportService.new(storage_service).create_logs(path) + when "sales-logs" + Imports::SalesLogsImportService.new(storage_service).create_logs(path) else raise "Type #{type} is not supported by data_import" end diff --git a/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml new file mode 100644 index 000000000..575f93887 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/discounted_ownership_sales_log.xml @@ -0,0 +1,351 @@ + + + 2022-CORE-Sales + discounted_ownership_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09z2c55d9cb90d7ba84927e64618 + 7c5bd5fb549c09z2c55d9cb90d7ba84927e64618 + 2023-02-21T11:54:51.786722Z + 2023-02-22T10:59:45.88188Z + submitted-valid + 2022 + Manual Entry + + + + + + + + 2023-02-01 + Discount ownership example + 2 Yes - a discount ownership scheme + + 14 Preserved Right to Buy (PRTB) + + + + + 2 No + + 1 No + + + 3 + 3 House + 1 Purpose built + SW1A 1AA + + + Cheltenham + E07000078 + 3 Don’t know + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 134750 + 1 + 0 + 0 + 0 + 1 +
300203
+ + + + + + + + + + + + + + + + + + + + + + + + + 0 + 1 + 0 + 1 + 0 + 0 + 0 + 1 + 2 + 2023 + + + + + + + GL51 + 9EX + + + GL51 + 9EX + 1 + + + + + 3 Private tenant + GL51 9EX + + Cheltenham + E07000078 + + + Yes + + + + + + + + + + + + + + + + 1 Yes + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + 275000 + + 51 + 1 Yes + 134750 + Halifax + + + 33 + 2 No + 0 + 0.00 + + + + + + + + + + + + 0 + 0 + 0 + + + 9 = other + + + + + 14 Preserved Right to Buy (PRTB) + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml new file mode 100644 index 000000000..4bbdc9981 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/outright_sale_sales_log.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + outright_sale_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T12:09:45.809134Z + 2023-02-22T10:59:01.709949Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-16 + Outright ownership example + 3 No - this is an outright or other sale + + + 10 Outright + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 1 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 2 No + + + 75 + Female + 5 Retired + 1 White: English/Scottish/Welsh/Northern Irish/British + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 2 + 0 + 0 + 0 + 0 +
300202
+ + + + + + + B + + + + B + + + + + + + + + + + + + + 0 + 0 + 1 + 1 + 0 + 0 + 0 + 16 + 1 + 2023 + + + + + + + B95 + 5HZ + SW1A + 1AA + + + + 3 Private tenant + B95 5HZ + + Stratford-on-Avon + E07000221 + + + Yes + + + + + 7 No + + + 2 No + 2 No + + + 2 No + + + + 2 No + + + + 4 Don’t know + + + 2 No + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 300000 + 2 No + + + + + 300000 + + + 1 + 1 + 0 + + + 1 = 1 elder + + + 10 Outright + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml new file mode 100644 index 000000000..6e0c11174 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml new file mode 100644 index 000000000..6334674a2 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log2.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log2 + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml new file mode 100644 index 000000000..b5dabde97 --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log3.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log3 + c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + 134.24 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml new file mode 100644 index 000000000..98342cf3c --- /dev/null +++ b/spec/fixtures/imports/sales_logs/shared_ownership_sales_log4.xml @@ -0,0 +1,333 @@ + + + 2022-CORE-Sales + shared_ownership_sales_log4 + e29c492473446dca4d50224f2bb7cf965a261d6f + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 7c5bd5fb549c09a2c55d7cb90d7ba84927e64618 + 2023-02-21T11:48:28.255968Z + 2023-02-22T11:00:06.575832Z + submitted-valid + 2022 + Manual Entry + + + + + Yes + 2023-01-17 + Shared ownership example + 1 Yes - a shared ownership scheme + 2 Shared Ownership + + + + 2 No + 1 Yes + 2 No + + 2 Yes + + + 2 + 1 Flat or maisonette + 1 Purpose built + SW1A 1AA + Westminster + E09000033 + 3 Don’t know + + + 30 + Male + 1 Full Time - 30 hours or more a week + 2 White: Irish + 18 United Kingdom + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 Yes + + + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 76000 + 1 + 47000 + 0 + 235000 + 0 +
300204
+ + + + + + + + + + + + + + + + + + + + + + + + + 1 + 0 + 0 + 1 + 1 + 2 + 3 + 17 + 1 + 2023 + 6 + 9 + 2022 + 8 + 1 + 2023 + SW14 + 7QP + SW1A + 1AA + + + + 2 Private registered provider (PRP) or housing association tenant + SW14 7QP + + Richmond-upon-Thames + E09000027 + Yes + + + + + + + 8 Don’t know + + + 2 No + 2 No + + + 1 Yes + 47000 + 1 Yes + + + + 4 Don’t know + 1 Yes + 89000 + 1 Yes + + + + + 1 + 2 No + + 30 + 2 No + 2023-01-08 + 2022-09-06 + 2 No + + + + 550000 + 30 + 1 Yes + 76000 + Nationwide + 33 + 2 No + 89000 + + 912.00 + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + 0 + + + 3 = 1 adult + + + 2 Shared Ownership + + + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + 0 + 0 + 0 + 0 + 1 + + + + + E12000007 + 1 + 1 Test + 655 + + diff --git a/spec/lib/tasks/data_import_spec.rb b/spec/lib/tasks/data_import_spec.rb index ad1887d16..462f2740e 100644 --- a/spec/lib/tasks/data_import_spec.rb +++ b/spec/lib/tasks/data_import_spec.rb @@ -109,6 +109,24 @@ describe "rake core:data_import", type: :task do end end + context "when importing sales logs" do + let(:type) { "sales-logs" } + let(:import_service) { instance_double(Imports::SalesLogsImportService) } + let(:fixture_path) { "spec/fixtures/imports/sales_logs" } + + before do + allow(Imports::SalesLogsImportService).to receive(:new).and_return(import_service) + end + + it "creates sales logs from the given XML file" do + expect(Storage::S3Service).to receive(:new).with(paas_config_service, instance_name) + expect(Imports::SalesLogsImportService).to receive(:new).with(storage_service) + expect(import_service).to receive(:create_logs).with(fixture_path) + + task.invoke(type, fixture_path) + end + end + context "when importing scheme data" do let(:type) { "scheme" } let(:import_service) { instance_double(Imports::SchemeImportService) } diff --git a/spec/services/imports/sales_logs_import_service_spec.rb b/spec/services/imports/sales_logs_import_service_spec.rb new file mode 100644 index 000000000..b556d3e68 --- /dev/null +++ b/spec/services/imports/sales_logs_import_service_spec.rb @@ -0,0 +1,595 @@ +require "rails_helper" + +RSpec.describe Imports::SalesLogsImportService do + subject(:sales_log_service) { described_class.new(storage_service, logger) } + + let(:storage_service) { instance_double(Storage::S3Service) } + let(:logger) { instance_double(ActiveSupport::Logger) } + + let(:fixture_directory) { "spec/fixtures/imports/sales_logs" } + + let(:organisation) { FactoryBot.create(:organisation, old_visible_id: "1", provider_type: "PRP") } + let(:managing_organisation) { FactoryBot.create(:organisation, old_visible_id: "2", provider_type: "PRP") } + let(:remote_folder) { "sales_logs" } + + def open_file(directory, filename) + File.open("#{directory}/#{filename}.xml") + end + + before do + { "GL519EX" => "E07000078", + "SW1A2AA" => "E09000033", + "SW1A1AA" => "E09000033", + "SW147QP" => "E09000027", + "B955HZ" => "E07000221" }.each do |postcode, district_code| + WebMock.stub_request(:get, /api.postcodes.io\/postcodes\/#{postcode}/).to_return(status: 200, body: "{\"status\":200,\"result\":{\"admin_district\":\"#{district_code}\",\"codes\":{\"admin_district\":\"#{district_code}\"}}}", headers: {}) + end + + allow(Organisation).to receive(:find_by).and_return(nil) + allow(Organisation).to receive(:find_by).with(old_visible_id: organisation.old_visible_id).and_return(organisation) + allow(Organisation).to receive(:find_by).with(old_visible_id: managing_organisation.old_visible_id).and_return(managing_organisation) + + # Created by users + FactoryBot.create(:user, old_user_id: "c3061a2e6ea0b702e6f6210d5c52d2a92612d2aa", organisation:) + FactoryBot.create(:user, old_user_id: "e29c492473446dca4d50224f2bb7cf965a261d6f", organisation:) + end + + context "when importing sales logs" do + before do + # Stub the S3 file listing and download + allow(storage_service).to receive(:list_files) + .and_return(%W[#{remote_folder}/shared_ownership_sales_log.xml #{remote_folder}/shared_ownership_sales_log2.xml #{remote_folder}/outright_sale_sales_log.xml #{remote_folder}/discounted_ownership_sales_log.xml]) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log"), open_file(fixture_directory, "shared_ownership_sales_log")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log2.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log2"), open_file(fixture_directory, "shared_ownership_sales_log2")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/outright_sale_sales_log.xml") + .and_return(open_file(fixture_directory, "outright_sale_sales_log"), open_file(fixture_directory, "outright_sale_sales_log")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/discounted_ownership_sales_log.xml") + .and_return(open_file(fixture_directory, "discounted_ownership_sales_log"), open_file(fixture_directory, "discounted_ownership_sales_log")) + end + + it "successfully creates all sales logs" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.create_logs(remote_folder) } + .to change(SalesLog, :count).by(4) + end + + it "only updates existing sales logs" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).to receive(:info).with(/Updating sales log/).exactly(4).times + expect { 2.times { sales_log_service.create_logs(remote_folder) } } + .to change(SalesLog, :count).by(4) + end + + context "when there are status discrepancies" do + let(:sales_log_file) { open_file(fixture_directory, "shared_ownership_sales_log3") } + let(:sales_log_xml) { Nokogiri::XML(sales_log_file) } + + before do + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log3.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log3"), open_file(fixture_directory, "shared_ownership_sales_log3")) + allow(storage_service).to receive(:get_file_io) + .with("#{remote_folder}/shared_ownership_sales_log4.xml") + .and_return(open_file(fixture_directory, "shared_ownership_sales_log4"), open_file(fixture_directory, "shared_ownership_sales_log4")) + end + + it "the logger logs a warning with the sales log's old id/filename" do + expect(logger).to receive(:warn).with(/is not completed/).once + expect(logger).to receive(:warn).with(/sales log with old id:shared_ownership_sales_log3 is incomplete but status should be complete/).once + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "on completion the ids of all logs with status discrepancies are logged in a warning" do + allow(storage_service).to receive(:list_files) + .and_return(%W[#{remote_folder}/shared_ownership_sales_log3.xml #{remote_folder}/shared_ownership_sales_log4.xml]) + expect(logger).to receive(:warn).with(/is not completed/).twice + expect(logger).to receive(:warn).with(/is incomplete but status should be complete/).twice + expect(logger).to receive(:warn).with(/The following sales logs had status discrepancies: \[shared_ownership_sales_log3, shared_ownership_sales_log4\]/) + + sales_log_service.create_logs(remote_folder) + end + end + end + + context "when importing a specific log" do + let(:sales_log_file) { open_file(fixture_directory, sales_log_id) } + let(:sales_log_xml) { Nokogiri::XML(sales_log_file) } + + context "and the organisation legacy ID does not exist" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + before { sales_log_xml.at_xpath("//xmlns:OWNINGORGID").content = 99_999 } + + it "raises an exception" do + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to raise_error(RuntimeError, "Organisation not found with legacy ID 99999") + end + end + + context "when the mortgage lender is set to an existing option" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q34a").content = "halifax" + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets mortgage lender" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.mortgagelender).to be(11) + end + end + + context "when the mortgage lender is set to a non existing option" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q34a").content = "something else" + allow(logger).to receive(:warn).and_return(nil) + end + + it "correctly sets mortgage lender and mortgage lender other" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.mortgagelender).to be(40) + expect(sales_log&.mortgagelenderother).to eq("something else") + end + end + + context "with shared ownership type" do + let(:sales_log_id) { "shared_ownership_sales_log" } + + it "successfully creates a completed shared ownership log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "with discounted ownership type" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + it "successfully creates a completed discounted ownership log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "with outright sale type" do + let(:sales_log_id) { "outright_sale_sales_log" } + + it "successfully creates a completed outright sale log" do + expect(logger).not_to receive(:error) + expect(logger).not_to receive(:warn) + expect(logger).not_to receive(:info) + expect { sales_log_service.send(:create_log, sales_log_xml) } + .to change(SalesLog, :count).by(1) + end + end + + context "when inferring default answers for completed sales logs" do + context "when the armedforcesspouse is not answered" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:ARMEDFORCESSPOUSE").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets armedforcesspouse to don't know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.armedforcesspouse).to be(7) + end + end + + context "when the savings not known is not answered and savings is not given" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:savingsKnown").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets savingsnk to not know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.savingsnk).to be(1) + end + end + + context "when the savings not known is not answered and savings is given" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:Q3Savings").content = "10000" + sales_log_xml.at_xpath("//xmlns:savingsKnown").content = "" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets savingsnk to know" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.savingsnk).to be(0) + end + end + + context "and it's an outright sale" do + let(:sales_log_id) { "outright_sale_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "infers mscharge_known as no" do + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log.mscharge_known).to eq(0) + end + end + + context "when inferring age known" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Age").content = "" + sales_log_xml.at_xpath("//xmlns:P2Age").content = "" + sales_log_xml.at_xpath("//xmlns:P3Age").content = "22" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets age known to no if age not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.age1_known).to be(1) # unknown + expect(sales_log&.age2_known).to be(1) # unknown + end + + it "sets age known to yes if age answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.age3_known).to be(0) # known + end + end + + context "when inferring gender" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Sex").content = "" + sales_log_xml.at_xpath("//xmlns:P2Sex").content = "" + sales_log_xml.at_xpath("//xmlns:P3Sex").content = "Female" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets gender to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.sex1).to eq("R") + expect(sales_log&.sex2).to eq("R") + end + + it "sets the gender correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.sex3).to eq("F") + end + end + + context "when inferring ethnic group" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "1" + sales_log_xml.at_xpath("//xmlns:P1Eth").content = "" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets ethnic group to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ethnic_group).to eq(17) + end + end + + context "when inferring nationality" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "1" + sales_log_xml.at_xpath("//xmlns:P1Nat").content = "" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets nationality to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.national).to eq(13) + end + end + + context "when inferring economic status" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P1Eco").content = "" + sales_log_xml.at_xpath("//xmlns:P2Eco").content = "" + sales_log_xml.at_xpath("//xmlns:P3Eco").content = "3" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets economic status to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ecstat1).to eq(10) + expect(sales_log&.ecstat2).to eq(10) + end + + it "sets the economic status correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.ecstat3).to eq(3) + end + end + + context "when inferring relationship" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:P2Rel").content = "" + sales_log_xml.at_xpath("//xmlns:P3Rel").content = "Partner" + allow(logger).to receive(:warn).and_return(nil) + + sales_log_service.send(:create_log, sales_log_xml) + end + + it "sets relationship to prefers not to say if not answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.relat2).to eq("R") + end + + it "sets the relationship correctly if answered" do + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.relat3).to eq("P") + end + end + + context "when inferring armed forces" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hhregres to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:ArmedF").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hhregres).to eq(8) + end + + it "sets hhregres correctly if answered" do + sales_log_xml.at_xpath("//xmlns:ArmedF").content = "7 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hhregres).to eq(7) + end + end + + context "when inferring disability" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets disabled to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Disability").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.disabled).to eq(3) + end + + it "sets disabled correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Disability").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.disabled).to eq(2) + end + end + + context "when inferring wheelchair" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets wheel to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q10Wheelchair").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.wheel).to eq(3) + end + + it "sets wheel correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q10Wheelchair").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.wheel).to eq(2) + end + end + + context "when inferring housing benefit" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hb to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q2a").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hb).to eq(4) + end + + it "sets hb correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q2a").content = "2 Housing Benefit" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hb).to eq(2) + end + end + + context "when inferring income not known" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets income to not known if not answered and income is not given" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(1) + expect(sales_log&.income2nk).to eq(1) + end + + it "sets income to known if not answered but the income is given" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person1Income").content = "30000" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "" + sales_log_xml.at_xpath("//xmlns:Q2Person2Income").content = "40000" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(0) + expect(sales_log&.income2nk).to eq(0) + end + + it "sets income known correctly if answered" do + sales_log_xml.at_xpath("//xmlns:P1IncKnown").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:P2IncKnown").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.income1nk).to eq(0) + expect(sales_log&.income2nk).to eq(1) + end + end + + context "when inferring prevown" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets prevown to don't know if not answered" do + sales_log_xml.at_xpath("//xmlns:Q4PrevOwnedProperty").content = "" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.prevown).to eq(3) + end + + it "sets prevown correctly if answered" do + sales_log_xml.at_xpath("//xmlns:Q4PrevOwnedProperty").content = "2 No" + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.prevown).to eq(2) + end + end + + context "when inferring household count" do + let(:sales_log_id) { "discounted_ownership_sales_log" } + + before do + allow(logger).to receive(:warn).and_return(nil) + end + + it "sets hholdcount to hhmemb - 1 if not answered and not joint purchase" do + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:joint").content = "2 No" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(2) + end + + it "sets hholdcount to hhmemb - 2 if not answered and joint purchase" do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "3" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(1) + end + + it "sets hholdcount to 0 if HHMEMB is 0" do + sales_log_xml.at_xpath("//xmlns:joint").content = "1 Yes" + sales_log_xml.at_xpath("//xmlns:JointMore").content = "2 No" + sales_log_xml.at_xpath("//xmlns:HHMEMB").content = "0" + sales_log_xml.at_xpath("//xmlns:LiveInOther").content = "" + + sales_log_service.send(:create_log, sales_log_xml) + + sales_log = SalesLog.find_by(old_id: sales_log_id) + expect(sales_log&.hholdcount).to eq(0) + end + end + end + end +end