submit-social-housing-lettings-and-sales-data/lib/tasks/log_la_fix.rake

namespace :log_la_fix do
  desc "For all logs missing an LA that could have one, call the postcode changed method to request a new one from postcodes API. For logs where an LA still cannot be found, this will set them back to in progress."
  task :search_for_la_on_logs_with_nil_la, [:year] => :environment do |_task, args|
    include CollectionTimeHelper

    year = args[:year]&.to_i || current_collection_start_year

    lettings_logs = LettingsLog.filter_by_year(year).where(la: nil, needstype: 1, status: "completed")
    sales_logs = SalesLog.filter_by_year(year).where(la: nil, status: "completed")
    lettings_logs_count = lettings_logs.count
    sales_logs_count = sales_logs.count

    puts "Checking LA on #{lettings_logs_count} lettings logs in #{year}"

    i = 0
    lettings_logs.find_each do |log|
      next unless log.valid?

      log.process_postcode_changes!
      unless log.save
        puts "Failed to save lettings log #{log.id}"
        puts "Errors: #{log.errors.full_messages}"
      end

      if log.la.nil? && log.status == "in_progress"
        puts "#lettings##{log.id},\"#{log.tenancycode}\",\"#{log.propcode}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.startdate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\""
      end

      i += 1
      if (i % 100).zero?
        puts "Processed #{i} lettings logs"
      end
    end

    puts "Done #{lettings_logs_count} lettings logs"

    puts "Checking LA on #{sales_logs_count} sales logs in #{year}"

    i = 0
    sales_logs.find_each do |log|
      next unless log.valid?

      log.process_postcode_changes!
      unless log.save
        puts "Failed to save sales log #{log.id}"
        puts "Errors: #{log.errors.full_messages}"
      end

      if log.la.nil? && log.status == "in_progress"
        puts "#sales##{log.id},\"#{log.purchid}\",#{log.owning_organisation_id},\"#{log.owning_organisation&.name}\",#{log.managing_organisation_id},\"#{log.managing_organisation&.name}\",#{log.assigned_to_id},\"#{log.assigned_to&.name}\",#{log.saledate},\"#{log.address_line1}\",\"#{log.address_line2}\",\"#{log.town_or_city}\",\"#{log.county}\",\"#{log.postcode_full}\",\"\",\"\""
      end

      i += 1
      if (i % 100).zero?
        puts "Processed #{i} sales logs"
      end
    end

    puts "Done #{sales_logs_count} sales logs"

    puts "Done"
  end

  desc "Parse the output of search_for_la_on_logs_with_nil_la into separate lettings and sales CSV files"
  task parse_logs_moved_to_incomplete_with_no_la: :environment do
    require "csv"

    file = "output.txt"

    lettings_headers = %w[id tenancycode propcode owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to startdate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name]
    sales_headers = %w[id purchid owning_organisation_id owning_organisation managing_organisation_id managing_organisation assigned_to_id assigned_to saledate address_line1 address_line2 town_or_city county postcode_full la_ecode la_name]

    lettings_csv = CSV.open("lettings_logs_moved_to_incomplete_with_no_la.csv", "w")
    sales_csv = CSV.open("sales_logs_moved_to_incomplete_with_no_la.csv", "w")

    lettings_csv << lettings_headers
    sales_csv << sales_headers

    File.readlines(file).each do |line|
      line = line.strip
      if line.start_with?("#lettings#")
        row = CSV.parse_line(line.delete_prefix("#lettings#"), liberal_parsing: true)
        lettings_csv << row
      elsif line.start_with?("#sales#")
        row = CSV.parse_line(line.delete_prefix("#sales#"), liberal_parsing: true)
        sales_csv << row
      end
    end

    lettings_csv.close
    sales_csv.close

    puts "Written lettings_logs_moved_to_incomplete_with_no_la.csv"
    puts "Written sales_logs_moved_to_incomplete_with_no_la.csv"
  end

  desc "Split lettings and sales CSVs by managing organisation into separate files per org"
  task split_logs_by_managing_org: :environment do
    require "csv"

    %w[lettings sales].each do |log_type|
      input_file = "#{log_type}_logs_moved_to_incomplete_with_no_la.csv"

      rows_by_org = Hash.new { |h, k| h[k] = [] }
      table = CSV.read(input_file, headers: true)

      table.each do |row|
        org_name = row["managing_organisation"]
        rows_by_org[org_name] << row
      end

      rows_by_org.each do |org_name, rows|
        if rows.size < 30
          puts "Skipping #{org_name} (#{rows.size} rows)"
          next
        end

        FileUtils.mkdir_p("log_output")
        sanitised_name = org_name.parameterize(separator: "_")
        output_file = "log_output/#{sanitised_name}_#{log_type}_logs_moved_to_incomplete_with_no_la.csv"
        CSV.open(output_file, "w") do |csv|
          csv << table.headers
          rows.each { |row| csv << row }
        end
        puts "Written #{output_file} (#{rows.size} rows)"
      end
    end
  end
end