From 02c3d00bf688b6f752b21c583e8ca2820ec1fac9 Mon Sep 17 00:00:00 2001 From: Samuel Young Date: Mon, 20 Apr 2026 16:09:27 +0100 Subject: [PATCH] CLDC-4436: Add a script to recalculate LA of logs in a year (#3315) * CLDC-4436: Add a script to recalculate LA of logs in a year * fixup! CLDC-4436: Add a script to recalculate LA of logs in a year dont filter by BU, if this is nil we ought to try to correct it only set back completed logs * fixup! CLDC-4436: Add a script to recalculate LA of logs in a year report count both before and after processing * fixup! CLDC-4436: Add a script to recalculate LA of logs in a year fix SalesLog reference * CLDC-4436: Upgrade log LA fix script for every log it changes to in progress after setting LA, print this to the console this output can be copied and written to a file local. the two subsequent tasks will convert these to CSV files that can be shared with LAs combines them into a single namespace * CLDC-4436: Put final files into a folder for easy access * CLDC-4436: Print relation IDs as well * fixup! CLDC-4436: Upgrade log LA fix script * CLDC-4436: Add separate cols for la ecode and la name * CLDC-4436: Allow 30 logs in an org * fixup! CLDC-4436: Upgrade log LA fix script fix rows for sales too Co-authored-by: Oscar Richardson <116292912+oscar-richardson-softwire@users.noreply.github.com> --------- Co-authored-by: Oscar Richardson <116292912+oscar-richardson-softwire@users.noreply.github.com> --- lib/tasks/log_la_fix.rake | 129 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 lib/tasks/log_la_fix.rake diff --git a/lib/tasks/log_la_fix.rake b/lib/tasks/log_la_fix.rake new file mode 100644 index 000000000..1a13602be --- /dev/null +++ b/lib/tasks/log_la_fix.rake @@ -0,0 +1,129 @@ +namespace :log_la_fix do + desc "For all logs missing an LA that could have one, call the postcode changed method to request a new one from postcodes API. For logs where an LA still cannot be found, this will set them back to in progress." + task :search_for_la_on_logs_with_nil_la, [:year] => :environment do |_task, args| + include CollectionTimeHelper + + year = args[:year]&.to_i || current_collection_start_year + + lettings_logs = LettingsLog.filter_by_year(year).where(la: nil, needstype: 1, status: "completed") + sales_logs = SalesLog.filter_by_year(year).where(la: nil, status: "completed") + lettings_logs_count = lettings_logs.count + sales_logs_count = sales_logs.count + + puts "Checking LA on #{lettings_logs_count} lettings logs in #{year}" + + i = 0 + lettings_logs.find_each do |log| + next unless log.valid? + + log.process_postcode_changes! + unless log.save + puts "Failed to save lettings log #{log.id}" + puts "Errors: #{log.errors.full_messages}" + end + + if log.la.nil? && log.status == "in_progress" + puts "#lettings##{log.id},\"#{log.tenancycode}\",\"#{log.propcode}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.startdate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\"" + end + + i += 1 + if (i % 100).zero? + puts "Processed #{i} lettings logs" + end + end + + puts "Done #{lettings_logs_count} lettings logs" + + puts "Checking LA on #{sales_logs_count} sales logs in #{year}" + + i = 0 + sales_logs.find_each do |log| + next unless log.valid? + + log.process_postcode_changes! + unless log.save + puts "Failed to save sales log #{log.id}" + puts "Errors: #{log.errors.full_messages}" + end + + if log.la.nil? && log.status == "in_progress" + puts "#sales##{log.id},\"#{log.purchid}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.saledate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\"" + end + + i += 1 + if (i % 100).zero? + puts "Processed #{i} sales logs" + end + end + + puts "Done #{sales_logs_count} sales logs" + + puts "Done" + end + + desc "Parse the output of search_for_la_on_logs_with_nil_la into separate lettings and sales CSV files" + task parse_logs_moved_to_incomplete_with_no_la: :environment do + require "csv" + + file = "output.txt" + + lettings_headers = %w[id tenancycode propcode owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to startdate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name] + sales_headers = %w[id purchid owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to saledate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name] + + lettings_csv = CSV.open("lettings_logs_moved_to_incomplete_with_no_la.csv", "w") + sales_csv = CSV.open("sales_logs_moved_to_incomplete_with_no_la.csv", "w") + + lettings_csv << lettings_headers + sales_csv << sales_headers + + File.readlines(file).each do |line| + line = line.strip + if line.start_with?("#lettings#") + row = CSV.parse_line(line.delete_prefix("#lettings#")) + lettings_csv << row + elsif line.start_with?("#sales#") + row = CSV.parse_line(line.delete_prefix("#sales#")) + sales_csv << row + end + end + + lettings_csv.close + sales_csv.close + + puts "Written lettings_logs_moved_to_incomplete_with_no_la.csv" + puts "Written sales_logs_moved_to_incomplete_with_no_la.csv" + end + + desc "Split lettings and sales CSVs by managing organisation into separate files per org" + task split_logs_by_managing_org: :environment do + require "csv" + + %w[lettings sales].each do |log_type| + input_file = "#{log_type}_logs_moved_to_incomplete_with_no_la.csv" + + rows_by_org = Hash.new { |h, k| h[k] = [] } + table = CSV.read(input_file, headers: true) + + table.each do |row| + org_name = row["managing_organisation"] + rows_by_org[org_name] << row + end + + rows_by_org.each do |org_name, rows| + if rows.size < 30 + puts "Skipping #{org_name} (#{rows.size} rows)" + next + end + + FileUtils.mkdir_p("log_output") + sanitised_name = org_name.parameterize(separator: "_") + output_file = "log_output/#{sanitised_name}_#{log_type}_logs_moved_to_incomplete_with_no_la.csv" + CSV.open(output_file, "w") do |csv| + csv << table.headers + rows.each { |row| csv << row } + end + puts "Written #{output_file} (#{rows.size} rows)" + end + end + end +end