CLDC-4436: Add a script to recalculate LA of logs in a year (#3315)

* CLDC-4436: Add a script to recalculate LA of logs in a year

* fixup! CLDC-4436: Add a script to recalculate LA of logs in a year

dont filter by BU, if this is nil we ought to try to correct it

only set back completed logs

* fixup! CLDC-4436: Add a script to recalculate LA of logs in a year

report count both before and after processing

* fixup! CLDC-4436: Add a script to recalculate LA of logs in a year

fix SalesLog reference

* CLDC-4436: Upgrade log LA fix script

for every log it changes to in progress after setting LA, print this to the console

this output can be copied and written to a file local. the two subsequent tasks will convert these to CSV files that can be shared with LAs

combines them into a single namespace

* CLDC-4436: Put final files into a folder for easy access

* CLDC-4436: Print relation IDs as well

* fixup! CLDC-4436: Upgrade log LA fix script

* CLDC-4436: Add separate cols for la ecode and la name

* CLDC-4436: Allow 30 logs in an org

* fixup! CLDC-4436: Upgrade log LA fix script

fix rows for sales too

Co-authored-by: Oscar Richardson <116292912+oscar-richardson-softwire@users.noreply.github.com>

---------

Co-authored-by: Oscar Richardson <116292912+oscar-richardson-softwire@users.noreply.github.com>

lib/tasks/log_la_fix.rake

@@ -0,0 +1,129 @@
+namespace :log_la_fix do
+  desc "For all logs missing an LA that could have one, call the postcode changed method to request a new one from postcodes API. For logs where an LA still cannot be found, this will set them back to in progress."
+  task :search_for_la_on_logs_with_nil_la, [:year] => :environment do |_task, args|
+    include CollectionTimeHelper
+
+    year = args[:year]&.to_i || current_collection_start_year
+
+    lettings_logs = LettingsLog.filter_by_year(year).where(la: nil, needstype: 1, status: "completed")
+    sales_logs = SalesLog.filter_by_year(year).where(la: nil, status: "completed")
+    lettings_logs_count = lettings_logs.count
+    sales_logs_count = sales_logs.count
+
+    puts "Checking LA on #{lettings_logs_count} lettings logs in #{year}"
+
+    i = 0
+    lettings_logs.find_each do |log|
+      next unless log.valid?
+
+      log.process_postcode_changes!
+      unless log.save
+        puts "Failed to save lettings log #{log.id}"
+        puts "Errors: #{log.errors.full_messages}"
+      end
+
+      if log.la.nil? && log.status == "in_progress"
+        puts "#lettings##{log.id},\"#{log.tenancycode}\",\"#{log.propcode}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.startdate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\""
+      end
+
+      i += 1
+      if (i % 100).zero?
+        puts "Processed #{i} lettings logs"
+      end
+    end
+
+    puts "Done #{lettings_logs_count} lettings logs"
+
+    puts "Checking LA on #{sales_logs_count} sales logs in #{year}"
+
+    i = 0
+    sales_logs.find_each do |log|
+      next unless log.valid?
+
+      log.process_postcode_changes!
+      unless log.save
+        puts "Failed to save sales log #{log.id}"
+        puts "Errors: #{log.errors.full_messages}"
+      end
+
+      if log.la.nil? && log.status == "in_progress"
+        puts "#sales##{log.id},\"#{log.purchid}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.saledate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\""
+      end
+
+      i += 1
+      if (i % 100).zero?
+        puts "Processed #{i} sales logs"
+      end
+    end
+
+    puts "Done #{sales_logs_count} sales logs"
+
+    puts "Done"
+  end
+
+  desc "Parse the output of search_for_la_on_logs_with_nil_la into separate lettings and sales CSV files"
+  task parse_logs_moved_to_incomplete_with_no_la: :environment do
+    require "csv"
+
+    file = "output.txt"
+
+    lettings_headers = %w[id tenancycode propcode owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to startdate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name]
+    sales_headers = %w[id purchid owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to saledate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name]
+
+    lettings_csv = CSV.open("lettings_logs_moved_to_incomplete_with_no_la.csv", "w")
+    sales_csv = CSV.open("sales_logs_moved_to_incomplete_with_no_la.csv", "w")
+
+    lettings_csv << lettings_headers
+    sales_csv << sales_headers
+
+    File.readlines(file).each do |line|
+      line = line.strip
+      if line.start_with?("#lettings#")
+        row = CSV.parse_line(line.delete_prefix("#lettings#"))
+        lettings_csv << row
+      elsif line.start_with?("#sales#")
+        row = CSV.parse_line(line.delete_prefix("#sales#"))
+        sales_csv << row
+      end
+    end
+
+    lettings_csv.close
+    sales_csv.close
+
+    puts "Written lettings_logs_moved_to_incomplete_with_no_la.csv"
+    puts "Written sales_logs_moved_to_incomplete_with_no_la.csv"
+  end
+
+  desc "Split lettings and sales CSVs by managing organisation into separate files per org"
+  task split_logs_by_managing_org: :environment do
+    require "csv"
+
+    %w[lettings sales].each do |log_type|
+      input_file = "#{log_type}_logs_moved_to_incomplete_with_no_la.csv"
+
+      rows_by_org = Hash.new { |h, k| h[k] = [] }
+      table = CSV.read(input_file, headers: true)
+
+      table.each do |row|
+        org_name = row["managing_organisation"]
+        rows_by_org[org_name] << row
+      end
+
+      rows_by_org.each do |org_name, rows|
+        if rows.size < 30
+          puts "Skipping #{org_name} (#{rows.size} rows)"
+          next
+        end
+
+        FileUtils.mkdir_p("log_output")
+        sanitised_name = org_name.parameterize(separator: "_")
+        output_file = "log_output/#{sanitised_name}_#{log_type}_logs_moved_to_incomplete_with_no_la.csv"
+        CSV.open(output_file, "w") do |csv|
+          csv << table.headers
+          rows.each { |row| csv << row }
+        end
+        puts "Written #{output_file} (#{rows.size} rows)"
+      end
+    end
+  end
+end